From 868081727ef624f557f2549e3e7798d175f6a837 Mon Sep 17 00:00:00 2001 From: Johannes 'josch' Schauer Date: Mon, 9 Dec 2019 10:40:51 +0100 Subject: [PATCH] add special hooks copy-in, copy-out, tar-in, tar-out, upload and download --- coverage.sh | 143 ++++++++++- mmdebstrap | 711 +++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 825 insertions(+), 29 deletions(-) diff --git a/coverage.sh b/coverage.sh index ee70c7c..8e7fbc0 100755 --- a/coverage.sh +++ b/coverage.sh @@ -52,7 +52,7 @@ if [ ! -e shared/mmdebstrap ] || [ mmdebstrap -nt shared/mmdebstrap ]; then fi starttime= -total=115 +total=119 i=1 print_header() { @@ -1242,6 +1242,147 @@ else ./run_null.sh SUDO fi +# test special hooks +for mode in root unshare fakechroot proot; do + print_header "mode=$mode,variant=apt: test special hooks with $mode mode" + if [ "$mode" = "unshare" ] && [ "$HAVE_UNSHARE" != "yes" ]; then + echo "HAVE_UNSHARE != yes -- Skipping test..." + continue + fi + if [ "$mode" = "proot" ] && [ "$HAVE_PROOT" != "yes" ]; then + echo "HAVE_PROOT != yes -- Skipping test..." + continue + fi + cat << END > shared/test.sh +#!/bin/sh +set -eu +export LC_ALL=C.UTF-8 +[ "\$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1 && adduser --gecos user --disabled-password user +[ "$mode" = unshare ] && sysctl -w kernel.unprivileged_userns_clone=1 +prefix= +[ "\$(id -u)" -eq 0 ] && [ "$mode" != "root" ] && prefix="runuser -u user --" +[ "$mode" = "fakechroot" ] && prefix="\$prefix fakechroot fakeroot" +symlinktarget=/real +case $mode in fakechroot|proot) symlinktarget='\$1/real';; esac +echo copy-in-setup > /tmp/copy-in-setup +echo copy-in-essential > /tmp/copy-in-essential +echo copy-in-customize > /tmp/copy-in-customize +echo tar-in-setup > /tmp/tar-in-setup +echo tar-in-essential > /tmp/tar-in-essential +echo tar-in-customize > /tmp/tar-in-customize +tar -C /tmp -cf /tmp/tar-in-setup.tar tar-in-setup +tar -C /tmp -cf /tmp/tar-in-essential.tar tar-in-essential +tar -C /tmp -cf /tmp/tar-in-customize.tar tar-in-customize +rm /tmp/tar-in-setup +rm /tmp/tar-in-essential +rm /tmp/tar-in-customize +echo upload-in-setup > /tmp/upload-in-setup +echo upload-in-essential > /tmp/upload-in-essential +echo upload-in-customize > /tmp/upload-in-customize +\$prefix $CMD --mode=$mode --variant=apt \ + --setup-hook='mkdir "\$1/real"' \ + --setup-hook='copy-in /tmp/copy-in-setup /real' \ + --setup-hook='echo copy-in-setup | cmp "\$1/real/copy-in-setup" -' \ + --setup-hook='rm "\$1/real/copy-in-setup"' \ + --setup-hook='echo copy-out-setup > "\$1/real/copy-out-setup"' \ + --setup-hook='copy-out /real/copy-out-setup /tmp' \ + --setup-hook='rm "\$1/real/copy-out-setup"' \ + --setup-hook='tar-in /tmp/tar-in-setup.tar /real' \ + --setup-hook='echo tar-in-setup | cmp "\$1/real/tar-in-setup" -' \ + --setup-hook='tar-out /real/tar-in-setup /tmp/tar-out-setup.tar' \ + --setup-hook='rm "\$1"/real/tar-in-setup' \ + --setup-hook='upload /tmp/upload-in-setup /real/upload' \ + --setup-hook='echo upload-in-setup | cmp "\$1/real/upload" -' \ + --setup-hook='download /real/upload /tmp/download-in-setup' \ + --setup-hook='rm "\$1/real/upload"' \ + --essential-hook='ln -s "'"\$symlinktarget"'" "\$1/symlink"' \ + --essential-hook='copy-in /tmp/copy-in-essential /symlink' \ + --essential-hook='echo copy-in-essential | cmp "\$1/real/copy-in-essential" -' \ + --essential-hook='rm "\$1/real/copy-in-essential"' \ + --essential-hook='echo copy-out-essential > "\$1/real/copy-out-essential"' \ + --essential-hook='copy-out /symlink/copy-out-essential /tmp' \ + --essential-hook='rm "\$1/real/copy-out-essential"' \ + --essential-hook='tar-in /tmp/tar-in-essential.tar /symlink' \ + --essential-hook='echo tar-in-essential | cmp "\$1/real/tar-in-essential" -' \ + --essential-hook='tar-out /symlink/tar-in-essential /tmp/tar-out-essential.tar' \ + --essential-hook='rm "\$1"/real/tar-in-essential' \ + --essential-hook='upload /tmp/upload-in-essential /symlink/upload' \ + --essential-hook='echo upload-in-essential | cmp "\$1/real/upload" -' \ + --essential-hook='download /symlink/upload /tmp/download-in-essential' \ + --essential-hook='rm "\$1/real/upload"' \ + --customize-hook='copy-in /tmp/copy-in-customize /symlink' \ + --customize-hook='echo copy-in-customize | cmp "\$1/real/copy-in-customize" -' \ + --customize-hook='rm "\$1/real/copy-in-customize"' \ + --customize-hook='echo copy-out-customize > "\$1/real/copy-out-customize"' \ + --customize-hook='copy-out /symlink/copy-out-customize /tmp' \ + --customize-hook='rm "\$1/real/copy-out-customize"' \ + --customize-hook='tar-in /tmp/tar-in-customize.tar /symlink' \ + --customize-hook='echo tar-in-customize | cmp "\$1/real/tar-in-customize" -' \ + --customize-hook='tar-out /symlink/tar-in-customize /tmp/tar-out-customize.tar' \ + --customize-hook='rm "\$1"/real/tar-in-customize' \ + --customize-hook='upload /tmp/upload-in-customize /symlink/upload' \ + --customize-hook='echo upload-in-customize | cmp "\$1/real/upload" -' \ + --customize-hook='download /symlink/upload /tmp/download-in-customize' \ + --customize-hook='rm "\$1/real/upload"' \ + --customize-hook='rmdir "\$1/real"' \ + --customize-hook='rm "\$1/symlink"' \ + $DEFAULT_DIST /tmp/debian-chroot.tar $mirror +for n in setup essential customize; do + ret=0 + cmp /tmp/tar-in-\$n.tar /tmp/tar-out-\$n.tar || ret=\$? + if [ "\$ret" -ne 0 ]; then + if type diffoscope >/dev/null; then + diffoscope /tmp/tar-in-\$n.tar /tmp/tar-out-\$n.tar + continue + else + echo "no diffoscope installed" >&2 + fi + if type base64 >/dev/null; then + base64 /tmp/tar-in-\$n.tar + base64 /tmp/tar-out-\$n.tar + continue + else + echo "no base64 installed" >&2 + fi + if type xxd >/dev/null; then + xxd /tmp/tar-in-\$n.tar + xxd /tmp/tar-out-\$n.tar + continue + else + echo "no xxd installed" >&2 + fi + exit 1 + fi +done +echo copy-out-setup | cmp /tmp/copy-out-setup - +echo copy-out-essential | cmp /tmp/copy-out-essential - +echo copy-out-customize | cmp /tmp/copy-out-customize - +echo upload-in-setup | cmp /tmp/download-in-setup - +echo upload-in-essential | cmp /tmp/download-in-essential - +echo upload-in-customize | cmp /tmp/download-in-customize - +# in fakechroot mode, we use a fake ldconfig, so we have to +# artificially add some files +{ tar -tf /tmp/debian-chroot.tar; + [ "$mode" = "fakechroot" ] && printf "./etc/ld.so.cache\n./var/cache/ldconfig/\n"; + [ "$mode" = "fakechroot" ] && [ "$variant" != "essential" ] && printf "./etc/.pwd.lock\n"; +} | sort | diff -u tar1.txt - +rm /tmp/debian-chroot.tar \ + /tmp/copy-in-setup /tmp/copy-in-essential /tmp/copy-in-customize \ + /tmp/copy-out-setup /tmp/copy-out-essential /tmp/copy-out-customize \ + /tmp/tar-in-setup.tar /tmp/tar-in-essential.tar /tmp/tar-in-customize.tar \ + /tmp/tar-out-setup.tar /tmp/tar-out-essential.tar /tmp/tar-out-customize.tar \ + /tmp/upload-in-setup /tmp/upload-in-essential /tmp/upload-in-customize \ + /tmp/download-in-setup /tmp/download-in-essential /tmp/download-in-customize +END + if [ "$HAVE_QEMU" = "yes" ]; then + ./run_qemu.sh + elif [ "$mode" = "root" ]; then + ./run_null.sh SUDO + else + ./run_null.sh + fi +done + print_header "mode=root,variant=apt: debootstrap no-op options" cat << END > shared/test.sh #!/bin/sh diff --git a/mmdebstrap b/mmdebstrap index f5f8a95..b9b7af5 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -31,6 +31,7 @@ use Pod::Usage; use File::Copy; use File::Path qw(make_path remove_tree); use File::Temp qw(tempfile tempdir); +use File::Basename; use Cwd qw(abs_path); require "syscall.ph"; use Fcntl qw(S_IFCHR S_IFBLK FD_CLOEXEC F_GETFD F_SETFD); @@ -38,6 +39,7 @@ use List::Util qw(any none); use POSIX qw(SIGINT SIGHUP SIGPIPE SIGTERM SIG_BLOCK SIG_UNBLOCK); use Carp; use Term::ANSIColor; +use Socket; # from sched.h use constant { @@ -951,7 +953,31 @@ sub run_hooks($$) { my $runner = sub { foreach my $script (@{$options->{"${name}_hook"}}) { - if ( -x $script || $script !~ m/[^\w@\%+=:,.\/-]/a) { + if ($script =~ /^(copy-in|copy-out|tar-in|tar-out|upload|download) /) { + info "running special hook: $script"; + if (any { $_ eq $options->{variant} } ('extract', 'custom') + and any { $_ eq $options->{mode} } ('fakechroot', 'proot') + and $name ne 'setup') { + info "the copy-in, copy-out, tar-in and tar-out commands in fakechroot mode or proot mode might fail in extract and custom variants because there might be no tar inside the chroot"; + } + + my $pid = fork() // error "fork() failed: $!"; + if ($pid == 0) { + # whatever the script writes on stdout is sent to the + # socket + # whatever is written to the socket, send to stdin + open(STDOUT, '>&', $options->{hooksock}) or error "cannot open STDOUT: $!"; + open(STDIN, '<&', $options->{hooksock}) or error "cannot open STDIN: $!"; + + # we execute ourselves under sh to avoid having to + # implement a clever parser of the quoting used in $script + # for the filenames + exec 'sh', '-c', "$PROGRAM_NAME --hook-helper \"\$1\" \"\$2\" \"\$3\" \"\$4\" \"\$5\" $script", + 'exec', $options->{root}, $options->{mode}, $name, (defined $options->{qemu} ? "qemu-$options->{qemu}" : 'env', $verbosity_level); + } + waitpid($pid, 0); + $? == 0 or error "special hook failed with exit code $?"; + } elsif ( -x $script || $script !~ m/[^\w@\%+=:,.\/-]/a) { info "running --$name-hook directly: $script $options->{root}"; # execute it directly if it's an executable file # or if it there are no shell metacharacters @@ -987,6 +1013,8 @@ sub setup { debug "$key: $options->{$key}"; } elsif (ref $value eq 'ARRAY') { debug "$key: [" . (join ', ', @{$value}) . "]"; + } elsif (ref $value eq 'GLOB') { + debug "$key: GLOB"; } else { error "unknown type for key $key: " . (ref $value); } @@ -1843,9 +1871,253 @@ sub setup { } } +# messages from process inside unshared namespace to the outside +# openw -- open file for writing +# untar -- extract tar into directory +# write -- write data to last opened file or tar process +# close -- finish file writing or tar extraction +# adios -- last message and tear-down +# messages from process outside unshared namespace to the inside +# okthx -- success +sub checkokthx { + my $fh = shift; + my $ret = read ($fh, my $buf, 2+5) // error "cannot read from socket: $!"; + if ($ret == 0) { error "received eof on socket"; } + my ($len, $msg) = unpack("nA5", $buf); + if ($msg ne "okthx") { error "expected okthx but got: $msg"; } + if ($len != 0) { error "expected no payload but got $len bytes"; } +} + sub main() { umask 022; + if (scalar @ARGV >= 7 && $ARGV[0] eq "--hook-helper") { + my $root = $ARGV[1]; + my $mode = $ARGV[2]; + my $hook = $ARGV[3]; + my $qemu = $ARGV[4]; + $verbosity_level = $ARGV[5]; + my $command = $ARGV[6]; + + # unless we are in the setup hook (where there is no tar inside the + # chroot) we need to run tar on the inside because otherwise, possible + # absolute symlinks in the path given via --directory are not + # correctly resolved + # + # FIXME: the issue above can be fixed by a function that is able to + # resolve absolute symlinks even inside the chroot directory to a full + # path that is valid on the outside -- fakechroot and proot have their + # own reasons, see below + my @cmdprefix = (); + my @tarcmd = ('tar'); + if ($hook eq 'setup') { + if ($mode eq 'proot') { + # since we cannot run tar inside the chroot under proot during + # the setup hook because the chroot is empty, we have to run + # tar from the outside, which leads to all files being owned + # by the user running mmdebstrap. To let the ownership + # information not be completely off, we force all files be + # owned by the root user. + push @tarcmd, '--owner=root', '--group=root'; + } + } elsif (any { $_ eq $hook} ('essential', 'customize')) { + if ($mode eq 'fakechroot') { + # Fakechroot requires tar to run inside the chroot or + # otherwise absolute symlinks will include the path to the + # root directory + push @cmdprefix, '/usr/sbin/chroot', $root; + } elsif ($mode eq 'proot') { + # proot requires tar to run inside proot or otherwise + # permissions will be completely off + push @cmdprefix, 'proot', '--root-id', "--rootfs=$root", '--cwd=/', "--qemu=$qemu"; + } elsif (any { $_ eq $mode } ('root', 'chrootless', 'unshare')) { + push @cmdprefix, '/usr/sbin/chroot', $root; + } else { + error "unknown mode: $mode"; + } + } else { + error "unknown hook: $hook"; + } + + if (any { $_ eq $command} ('copy-in', 'tar-in', 'upload')) { + if (scalar @ARGV < 9) { + error "copy-in and tar-in need at least one path on the outside and the output path inside the chroot" + } + my $outpath = $ARGV[-1]; + for (my $i = 7; $i < $#ARGV; $i++) { + # the right argument for tar's --directory argument depends on + # whether tar is called from inside the chroot or from the + # outside + my $directory; + if ($hook eq 'setup') { + $directory = "$root/$outpath"; + } elsif (any { $_ eq $hook} ('essential', 'customize')) { + $directory = $outpath; + } else { + error "unknown hook: $hook"; + } + + # FIXME: here we would like to check if the path inside the + # chroot given by $directory actually exists but we cannot + # because we are missing a function that can resolve even + # paths including absolute symlinks to paths that are valid + # outside the chroot + + my $fh; + if ($command eq 'upload') { + # open the requested file for writing + open $fh, '|-', @cmdprefix, 'sh', '-c', 'cat > "$1"', 'exec', $directory // error "failed to fork(): $!"; + } else { + # open a tar process that extracts the tarfile that we supply + # it with on stdin to the output directory inside the chroot + open $fh, '|-', @cmdprefix, @tarcmd, '--directory', $directory, '--extract', '--file', '-' // error "failed to fork(): $!"; + } + + if ($command eq 'copy-in') { + # instruct the parent process to create a tarball of the + # requested path outside the chroot + debug "sending mktar"; + print STDOUT (pack("n", length $ARGV[$i]) . "mktar" . $ARGV[$i]); + } else { + # instruct parent process to open a tarball of the + # requested path outside the chroot for reading + debug "sending openr"; + print STDOUT (pack("n", length $ARGV[$i]) . "openr" . $ARGV[$i]); + } + STDOUT->flush(); + debug "waiting for okthx"; + checkokthx \*STDIN; + + # handle "write" messages from the parent process and feed + # their payload into the tar process until a "close" message + # is encountered + while(1) { + # receive the next message + my $ret = read (STDIN, my $buf, 2+5) // error "cannot read from socket: $!"; + if ($ret == 0) { + error "received eof on socket"; + } + my ($len, $msg) = unpack("nA5", $buf); + debug "received message: $msg"; + if ($msg eq "close") { + # finish the loop + if ($len != 0) { + error "expected no payload but got $len bytes"; + } + debug "sending okthx"; + print STDOUT (pack("n", 0) . "okthx") or error "cannot write to socket: $!"; + STDOUT->flush(); + last; + } elsif ($msg ne "write") { + # we should not receive this message at this point + print STDOUT (pack("n", 0) . "error") or error "cannot write to socket: $!"; + STDOUT->flush(); + error "expected write but got: $msg"; + } + # read the payload + my $content; + { + my $ret = read (STDIN, $content, $len) // error "error cannot read from socket: $!"; + if ($ret == 0) { + error "received eof on socket"; + } + } + # write the payload to the tar process + print $fh $content or error "cannot write to tar process: $!"; + debug "sending okthx"; + print STDOUT (pack("n", 0) . "okthx") or error "cannot write to socket: $!"; + STDOUT->flush(); + } + close $fh; + if ($command ne 'upload' and $? != 0) { + error "tar failed"; + } + } + } elsif (any { $_ eq $command} ('copy-out', 'tar-out', 'download')) { + if (scalar @ARGV < 9) { + error "copy-out needs at least one path inside the chroot and the output path on the outside" + } + my $outpath = $ARGV[-1]; + for (my $i = 7; $i < $#ARGV; $i++) { + # the right argument for tar's --directory argument depends on + # whether tar is called from inside the chroot or from the + # outside + my $directory; + if ($hook eq 'setup') { + $directory = "$root/$ARGV[$i]"; + } elsif (any { $_ eq $hook} ('essential', 'customize')) { + $directory = $ARGV[$i]; + } else { + error "unknown hook: $hook"; + } + + # FIXME: here we would like to check if the path inside the + # chroot given by $directory actually exists but we cannot + # because we are missing a function that can resolve even + # paths including absolute symlinks to paths that are valid + # outside the chroot + + my $fh; + if ($command eq 'download') { + # open the requested file for reading + open $fh, '-|', @cmdprefix, 'sh', '-c', 'cat "$1"', 'exec', $directory // error "failed to fork(): $!"; + } else { + # Open a tar process that creates a tarfile of everything in + # the requested directory inside the chroot and writes it to + # stdout. To emulate the behaviour of cp, change to the + # dirname of the requested path first. + open $fh, '-|', @cmdprefix, @tarcmd, '--directory', dirname($directory), '--create', '--file', '-', basename($directory) // error "failed to fork(): $!"; + } + + if ($command eq 'copy-out') { + # instruct the parent process to extract a tarball to a + # certain path outside the chroot + debug "sending untar"; + print STDOUT (pack("n", length $outpath) . "untar" . $outpath); + } else { + # instruct parent process to open a tarball of the + # requested path outside the chroot for writing + debug "sending openw"; + print STDOUT (pack("n", length $outpath) . "openw" . $outpath); + } + STDOUT->flush(); + debug "waiting for okthx"; + checkokthx \*STDIN; + + # read from the tar process and send as payload to the parent + # process + while (1) { + # read from tar + my $ret = read ($fh, my $cont, 4096) // error "cannot read from pipe: $!"; + if ($ret == 0) { last; } + debug "sending write"; + # send to parent + print STDOUT pack("n", $ret) . "write" . $cont; + STDOUT->flush(); + debug "waiting for okthx"; + checkokthx \*STDIN; + if ($ret < 4096) { last; } + } + + # signal to the parent process that we are done + debug "sending close"; + print STDOUT pack("n", 0) . "close"; + STDOUT->flush(); + debug "waiting for okthx"; + checkokthx \*STDIN; + + close $fh; + if ($command ne 'download' and $? != 0) { + error "tar failed"; + } + } + } else { + error "unknown command: $command"; + } + + exit 0; + } + my $mtime = time; if (exists $ENV{SOURCE_DATE_EPOCH}) { $mtime = $ENV{SOURCE_DATE_EPOCH}+0; @@ -2572,7 +2844,13 @@ sub main() { POSIX::sigprocmask(SIG_BLOCK, $sigset) or error "Can't block signals: $!"; my $pid; + + # a pipe to transfer the final tarball from the child to the parent pipe my $rfh, my $wfh; + + # instead of two pipe calls, creating four file handles, we use socketpair + socketpair my $childsock, my $parentsock, AF_UNIX, SOCK_STREAM, PF_UNSPEC or error "socketpair failed: $!"; + $options->{hooksock} = $childsock; if ($options->{mode} eq 'unshare') { $pid = get_unshare_cmd { # child @@ -2586,9 +2864,15 @@ sub main() { close $rfh; open(STDOUT, '>&', STDERR); + close $parentsock; setup($options); + print $childsock (pack('n', 0) . 'adios'); + $childsock->flush(); + + close $childsock; + if ($options->{maketar}) { info "creating tarball..."; @@ -2622,9 +2906,15 @@ sub main() { close $rfh; open(STDOUT, '>&', STDERR); + close $parentsock; setup($options); + print $childsock (pack('n', 0) . 'adios'); + $childsock->flush(); + + close $childsock; + if ($options->{maketar}) { info "creating tarball..."; @@ -2692,6 +2982,301 @@ sub main() { POSIX::sigprocmask(SIG_UNBLOCK, $sigset) or error "Can't unblock signals: $!"; close $wfh; + close $childsock; + + debug "starting to listen for hooks"; + # handle special hook commands via parentsock + # we use eval() so that error() doesn't take this process down and + # thus leaves the setup() process without a parent + eval { + while (1) { + # get the next message + my $msg = "error"; + my $len = -1; + { + debug "reading from parentsock"; + my $ret = read ($parentsock, my $buf, 2+5) // error "cannot read from socket: $!"; + debug "finished reading from parentsock"; + if ($ret == 0) { + error "received eof on socket"; + } + ($len, $msg) = unpack("nA5", $buf); + } + if ($msg eq "adios") { + # setup finished, so we break out of the loop + if ($len != 0) { + error "expected no payload but got $len bytes"; + } + last; + } elsif ($msg eq "openr") { + # handle the openr message + debug "received message: openr"; + my $infile; + { + my $ret = read ($parentsock, $infile, $len) // error "cannot read from socket: $!"; + if ($ret == 0) { + error "received eof on socket"; + } + } + # make sure that the requested path exists outside the chroot + if (! -e $infile) { + print $parentsock (pack("n", 0) . "error") or error "cannot write to socket: $!"; + $parentsock->flush(); + error "$infile does not exist"; + } + debug "sending okthx"; + print $parentsock (pack("n", 0) . "okthx") or error "cannot write to socket: $!"; + $parentsock->flush(); + + open my $fh, '<', $infile or error "failed to open $infile for reading: $!"; + + # read from the file and send as payload to the child process + while (1) { + # read from file + my $ret = read ($fh, my $cont, 4096) // error "cannot read from pipe: $!"; + if ($ret == 0) { last; } + debug "sending write"; + # send to child + print $parentsock pack("n", $ret) . "write" . $cont; + $parentsock->flush(); + debug "waiting for okthx"; + checkokthx $parentsock; + if ($ret < 4096) { last; } + } + + # signal to the child process that we are done + debug "sending close"; + print $parentsock pack("n", 0) . "close"; + $parentsock->flush(); + debug "waiting for okthx"; + checkokthx $parentsock; + + close $fh; + } elsif ($msg eq "openw") { + debug "received message: openw"; + # payload is the output directory + my $outfile; + { + my $ret = read ($parentsock, $outfile, $len) // error "cannot read from socket: $!"; + if ($ret == 0) { + error "received eof on socket"; + } + } + # make sure that the directory exists + my $outdir = dirname($outfile); + if (-e $outdir) { + if (! -d $outdir) { + print $parentsock (pack("n", 0) . "error") or error "cannot write to socket: $!"; + $parentsock->flush(); + error "$outdir already exists but is not a directory"; + } + } else { + my $num_created = make_path $outdir, {error => \my $err}; + if ($err && @$err) { + print $parentsock (pack("n", 0) . "error") or error "cannot write to socket: $!"; + $parentsock->flush(); + error (join "; ", (map {"cannot create " . (join ": ", %{$_})} @$err)); + } elsif ($num_created == 0) { + print $parentsock (pack("n", 0) . "error") or error "cannot write to socket: $!"; + $parentsock->flush(); + error "cannot create $outdir"; + } + } + debug "sending okthx"; + print $parentsock (pack("n", 0) . "okthx") or error "cannot write to socket: $!"; + $parentsock->flush(); + + # now we expect one or more "write" messages containing the + # tarball to write + open my $fh, '>', $outfile or error "failed to open $outfile for writing: $!"; + + # handle "write" messages from the child process and feed + # their payload into the file handle until a "close" message + # is encountered + while(1) { + # receive the next message + my $ret = read ($parentsock, my $buf, 2+5) // error "cannot read from socket: $!"; + if ($ret == 0) { + error "received eof on socket"; + } + my ($len, $msg) = unpack("nA5", $buf); + debug "received message: $msg"; + if ($msg eq "close") { + # finish the loop + if ($len != 0) { + error "expected no payload but got $len bytes"; + } + debug "sending okthx"; + print $parentsock (pack("n", 0) . "okthx") or error "cannot write to socket: $!"; + $parentsock->flush(); + last; + } elsif ($msg ne "write") { + # we should not receive this message at this point + print $parentsock (pack("n", 0) . "error") or error "cannot write to socket: $!"; + $parentsock->flush(); + error "expected write but got: $msg"; + } + # read the payload + my $content; + { + my $ret = read ($parentsock, $content, $len) // error "error cannot read from socket: $!"; + if ($ret == 0) { + error "received eof on socket"; + } + } + # write the payload to the file handle + print $fh $content or error "cannot write to file handle: $!"; + debug "sending okthx"; + print $parentsock (pack("n", 0) . "okthx") or error "cannot write to socket: $!"; + $parentsock->flush(); + } + close $fh; + } elsif ($msg eq "mktar") { + # handle the mktar message + debug "received message: mktar"; + my $indir; + { + my $ret = read ($parentsock, $indir, $len) // error "cannot read from socket: $!"; + if ($ret == 0) { + error "received eof on socket"; + } + } + # make sure that the requested path exists outside the chroot + if (! -e $indir) { + print $parentsock (pack("n", 0) . "error") or error "cannot write to socket: $!"; + $parentsock->flush(); + error "$indir does not exist"; + } + debug "sending okthx"; + print $parentsock (pack("n", 0) . "okthx") or error "cannot write to socket: $!"; + $parentsock->flush(); + + # Open a tar process creating a tarfile of the instructed + # path. To emulate the behaviour of cp, change to the + # dirname of the requested path first. + open my $fh, '-|', 'tar', '--directory', dirname($indir), '--create', '--file', '-', basename($indir) // error "failed to fork(): $!"; + + # read from the tar process and send as payload to the child + # process + while (1) { + # read from tar + my $ret = read ($fh, my $cont, 4096) // error "cannot read from pipe: $!"; + if ($ret == 0) { last; } + debug "sending write"; + # send to child + print $parentsock pack("n", $ret) . "write" . $cont; + $parentsock->flush(); + debug "waiting for okthx"; + checkokthx $parentsock; + if ($ret < 4096) { last; } + } + + # signal to the child process that we are done + debug "sending close"; + print $parentsock pack("n", 0) . "close"; + $parentsock->flush(); + debug "waiting for okthx"; + checkokthx $parentsock; + + close $fh; + if ($? != 0) { + error "tar failed"; + } + } elsif ($msg eq "untar") { + debug "received message: untar"; + # payload is the output directory + my $outdir; + { + my $ret = read ($parentsock, $outdir, $len) // error "cannot read from socket: $!"; + if ($ret == 0) { + error "received eof on socket"; + } + } + # make sure that the directory exists + if (-e $outdir) { + if (! -d $outdir) { + print $parentsock (pack("n", 0) . "error") or error "cannot write to socket: $!"; + $parentsock->flush(); + error "$outdir already exists but is not a directory"; + } + } else { + my $num_created = make_path $outdir, {error => \my $err}; + if ($err && @$err) { + print $parentsock (pack("n", 0) . "error") or error "cannot write to socket: $!"; + $parentsock->flush(); + error (join "; ", (map {"cannot create " . (join ": ", %{$_})} @$err)); + } elsif ($num_created == 0) { + print $parentsock (pack("n", 0) . "error") or error "cannot write to socket: $!"; + $parentsock->flush(); + error "cannot create $outdir"; + } + } + debug "sending okthx"; + print $parentsock (pack("n", 0) . "okthx") or error "cannot write to socket: $!"; + $parentsock->flush(); + + # now we expect one or more "write" messages containing the + # tarball to unpack + open my $fh, '|-', 'tar', '--directory', $outdir, '--extract', '--file', '-' // error "failed to fork(): $!"; + + # handle "write" messages from the child process and feed + # their payload into the tar process until a "close" message + # is encountered + while(1) { + # receive the next message + my $ret = read ($parentsock, my $buf, 2+5) // error "cannot read from socket: $!"; + if ($ret == 0) { + error "received eof on socket"; + } + my ($len, $msg) = unpack("nA5", $buf); + debug "received message: $msg"; + if ($msg eq "close") { + # finish the loop + if ($len != 0) { + error "expected no payload but got $len bytes"; + } + debug "sending okthx"; + print $parentsock (pack("n", 0) . "okthx") or error "cannot write to socket: $!"; + $parentsock->flush(); + last; + } elsif ($msg ne "write") { + # we should not receive this message at this point + print $parentsock (pack("n", 0) . "error") or error "cannot write to socket: $!"; + $parentsock->flush(); + error "expected write but got: $msg"; + } + # read the payload + my $content; + { + my $ret = read ($parentsock, $content, $len) // error "error cannot read from socket: $!"; + if ($ret == 0) { + error "received eof on socket"; + } + } + # write the payload to the tar process + print $fh $content or error "cannot write to tar process: $!"; + debug "sending okthx"; + print $parentsock (pack("n", 0) . "okthx") or error "cannot write to socket: $!"; + $parentsock->flush(); + } + close $fh; + if ($? != 0) { + error "tar failed"; + } + } else { + error "unknown message: $msg"; + } + } + }; + if ($@) { + # we cannot die here because that would leave the other thread + # running without a parent + warning "listening on child socket failed: $@"; + $exitstatus = 1; + } + debug "finish to listen for hooks"; + + close $parentsock; if ($options->{maketar}) { # we use eval() so that error() doesn't take this process down and @@ -3041,15 +3626,8 @@ equivalent: Execute arbitrary Is right after initial setup (directory creation, configuration of apt and dpkg, ...) but before any packages are downloaded or installed. At that point, the chroot directory does not contain any -executables and thus cannot be chroot-ed into. The option can be specified -multiple times and the commands are executed in the order in which they are -given on the command line. If I is an existing executable file or if -I does not contain any shell metacharacters, then I is -directly exec-ed with the path to the chroot directory passed as the first -argument. Otherwise, I is executed under I and the chroot -directory can be accessed via I<$1>. All environment variables used by -B (like C, C, C and C) -are preserved. +executables and thus cannot be chroot-ed into. See section B for more +information. Example: Setup merged-/usr via symlinks @@ -3067,15 +3645,8 @@ Example: Setup chroot for installing a sub-essential busybox-based chroot with Execute arbitrary Is after the Essential:yes packages have been installed but before installing the remaining packages. The hook is not -executed for the B and B variants. The option can be -specified multiple times and the commands are executed in the order in which -they are given on the command line. If I is an existing executable -file or if I does not contain any shell metacharacters, then -I is directly exec-ed with the path to the chroot directory passed as -the first argument. Otherwise, I is executed under I and the -chroot directory can be accessed via I<$1>. All environment variables used by -B (like C, C, C and C) -are preserved. +executed for the B and B variants. See section B for +more information. Example: Enable unattended upgrades @@ -3089,15 +3660,8 @@ Example: Select Europe/Berlin as the timezone =item B<--customize-hook>=I Execute arbitrary Is after the chroot is set up and all packages got -installed but before final cleanup actions are carried out. The option can be -specified multiple times and the commands are executed in the order in which -they are given on the command line. If I is an existing executable -file or if I does not contain any shell metacharacters, then -I is directly exec-ed with the path to the chroot directory passed as -the first argument. Otherwise, I is executed under I and the -chroot directory can be accessed via I<$1>. All environment variables used by -B (like C, C, C and C) -are preserved. +installed but before final cleanup actions are carried out. See section +B for more information. Example: Preparing a chroot for use with autopkgtest @@ -3239,6 +3803,84 @@ The B set plus all packages with Priority:standard. =back +=begin comment + +=head1 HOOKS + +This section describes properties of the hook options B<--setup-hook>, +B<--essential-hook> and B<--customize-hook> which are common to all three of +them. Any information specific to each hook is documented under the specific +hook options in the section B. + +The options can be specified multiple times and the commands are executed in +the order in which they are given on the command line. There are three +different types of hook option arguments. If the argument passed to the hook +option starts with C, C, C, C, C +or C followed by a space, then the hook is interpreted as a special +hook. Otherwise, if I is an existing executable file from C<$PATH> or +if I does not contain any shell metacharacters, then I is +directly exec-ed with the path to the chroot directory passed as the first +argument. Otherwise, I is executed under I and the chroot +directory can be accessed via I<$1>. All environment variables used by +B (like C, C, C and C) +are preserved. + +The paths inside the chroot are relative to the root directory of the chroot. +The path on the outside is relative to current directory of the original +B invocation. The path inside the chroot must already exist. Paths +outside the chroot are created as necessary. + +To be able to resolve even absolute symlinks, C or C and C are +executed inside the chroot for the B and B hooks. This +means that the special hooks might fail for the B and B +variants if no C or C and C is available inside the chroot. +Since nothing is yet installed at the time of the B hook, no absolute +symlinks in paths inside the chroot are supported during that hook. + +=over 8 + +=item B I [I ...] I + +Recursively copies one or more files and directories out of the chroot into, +placing them into I outside of the chroot. + +=item B I [I ...] I + +Recursively copies one or more files and directories into the chroot into, +placing them into I inside of the chroot. + +=item B I I + +Unpacks a tarball I from outside the chroot into a certain +location I inside the chroot. + +=item B I I + +Packs the path I from inside the chroot into a tarball, placing it +into a certain location I outside the chroot. + +=item B I I + +Copy the file given by I from inside the chroot to outside the +chroot as I. In contrast to B, this command only +handles files and not directories. To copy a directory recursively out of the +chroot, use B or B. Its advantage is, that by being able to +specify the full path on the outside, including the filename, the file on the +outside can have a different name from the file on the inside. + +=item B I I + +Copy the file given by I from outside the chroot to inside the +chroot as I. In contrast to B, this command only +handles files and not directories. To copy a directory recursively into the +chroot, use B or B. Its advantage is, that by being able to +specify the full path on the inside, including the filename, the file on the +inside can have a different name from the file on the outside. + +=back + +=end comment + =head1 EXAMPLES Use like debootstrap: @@ -3419,6 +4061,19 @@ B treats any warning from "apt-get update" as an error. Fixing this will require apt to provide a machine readable status interface. See Debian bugs #778357, #776152, #696335, and #745735. +=begin comment + +Special hooks either require C or C and C be present inside the +chroot or otherwise there is no support for absolute symlinks. This limitation +can be dropped for the B and B modes as well as for the +B hook in all modes if code is added that supports resolving paths with +absolute symlinks even inside the chroot directory reliably. Due to how +symlinks are handled by B and B, the requirement of having +C or C and C inside the chroot can never be dropped for these +modes. + +=end comment + =head1 SEE ALSO debootstrap(8)