From a3afe24fc0b1096fa0547503229f93487b7990d9 Mon Sep 17 00:00:00 2001 From: Johannes 'josch' Schauer Date: Sun, 20 Jan 2019 10:41:29 +0100 Subject: [PATCH] Instead of using gzip compression support, run compressor ourselves - otherwise ./dev tar cannot be concatenated with the rest - test compressor early - better document the TARGET option --- coverage.sh | 8 +-- mmdebstrap | 149 +++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 121 insertions(+), 36 deletions(-) diff --git a/coverage.sh b/coverage.sh index 6711989..80d63d0 100755 --- a/coverage.sh +++ b/coverage.sh @@ -251,17 +251,17 @@ else ./run_null.sh SUDO fi -print_header "mode=unshare,variant=apt: create tarball" +print_header "mode=unshare,variant=apt: create gzip compressed tarball" cat << END > shared/test.sh #!/bin/sh set -eu export LC_ALL=C.UTF-8 adduser --gecos user --disabled-password user sysctl -w kernel.unprivileged_userns_clone=1 -runuser -u user -- $CMD --mode=unshare --variant=apt unstable /tmp/unstable-chroot.tar $mirror -tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt +runuser -u user -- $CMD --mode=unshare --variant=apt unstable /tmp/unstable-chroot.tar.gz $mirror +tar -tf /tmp/unstable-chroot.tar.gz | sort > tar2.txt diff -u tar1.txt tar2.txt -rm /tmp/unstable-chroot.tar +rm /tmp/unstable-chroot.tar.gz END if [ "$HAVE_QEMU" = "yes" ]; then ./run_qemu.sh diff --git a/mmdebstrap b/mmdebstrap index 7050c46..fb9815c 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -130,26 +130,32 @@ sub error { # tar cannot figure out the decompression program when receiving data on # standard input, thus we do it ourselves. This is copied from tar's # src/suffix.c -sub get_tar_compress_options($) { +sub get_tar_compressor($) { my $filename = shift; - if ($filename =~ /\.(gz|tgz|taz)$/) { - return ('--gzip'); + if ($filename eq '-') { + return undef + } elsif ($filename =~ /\.tar$/) { + return undef + } elsif ($filename =~ /\.(gz|tgz|taz)$/) { + return 'gzip'; } elsif ($filename =~ /\.(Z|taZ)$/) { - return ('--compress'); + return 'compress'; } elsif ($filename =~ /\.(bz2|tbz|tbz2|tz2)$/) { - return ('--bzip2'); + return 'bzip2'; } elsif ($filename =~ /\.lz$/) { - return ('--lzip'); + return 'lzip'; } elsif ($filename =~ /\.(lzma|tlz)$/) { - return ('--lzma'); + return 'lzma'; } elsif ($filename =~ /\.lzo$/) { - return ('--lzop'); + return 'lzop'; } elsif ($filename =~ /\.lz4$/) { - return ('--use-compress-program', 'lz4'); + return 'lz4'; } elsif ($filename =~ /\.(xz|txz)$/) { - return ('--xz'); + return 'xz'; + } elsif ($filename =~ /\.zst$/) { + return 'zstd'; } - return (); + return undef } sub test_unshare($) { @@ -1931,11 +1937,11 @@ sub main() { error "refusing to use the filesystem root as output directory"; } - my @tar_compress_opts = get_tar_compress_options($options->{target}); + my $tar_compressor = get_tar_compressor($options->{target}); # figure out whether a tarball has to be created in the end $options->{maketar} = 0; - if (scalar @tar_compress_opts > 0 or $options->{target} =~ /\.tar$/ or $options->{target} eq '-') { + if (defined $tar_compressor or $options->{target} =~ /\.tar$/ or $options->{target} eq '-') { $options->{maketar} = 1; if (any { $_ eq $options->{variant} } ('extract', 'custom') and $options->{mode} eq 'fakechroot') { info "creating a tarball in fakechroot mode might fail in extract and custom variants because there might be no tar inside the chroot"; @@ -1945,6 +1951,19 @@ sub main() { open my $fh, '>', $options->{target} or error "cannot open $options->{target} for writing: $!"; close $fh; } + # check if the compressor is installed + if (defined $tar_compressor) { + my $pid = fork(); + if ($pid == 0) { + open(STDOUT, '>', '/dev/null') or error "cannot open /dev/null for writing: $!"; + open(STDIN, '<', '/dev/null') or error "cannot open /dev/null for reading: $!"; + exec $tar_compressor or error "cannot exec $tar_compressor: $!"; + } + waitpid $pid, 0; + if ($? != 0) { + error "failed to start $tar_compressor"; + } + } } if ($options->{maketar}) { @@ -2074,7 +2093,6 @@ sub main() { my $exitstatus = 0; my @taropts = ('--sort=name', "--mtime=\@$mtime", '--clamp-mtime', '--numeric-owner', '--one-file-system', '-c', '--exclude=./dev'); - push @taropts, @tar_compress_opts; # disable signals so that we can fork and change behaviour of the signal # handler in the parent and child without getting interrupted @@ -2196,18 +2214,47 @@ sub main() { close $wfh; if ($options->{maketar}) { - # we cannot die here because that would leave the other thread running - # without a parent - if ($options->{target} ne '-') { - if(!copy($rfh, $options->{target})) { - warning "cannot copy to $options->{target}: $!"; - $exitstatus = 1; - } - } else { - if (!copy($rfh, *STDOUT)) { - warning "cannot copy to standard output: $!"; - $exitstatus = 1; + # we use eval() so that error() doesn't take this process down and + # thus leaves the setup() process without a parent + eval { + if ($options->{target} eq '-') { + if (!copy($rfh, *STDOUT)) { + error "cannot copy to standard output: $!"; + } + } else { + if (defined $tar_compressor) { + POSIX::sigprocmask(SIG_BLOCK, $sigset) or error "Can't block signals: $!"; + my $cpid = fork(); + if ($cpid == 0) { + # child: default signal handlers + $SIG{'INT'} = 'DEFAULT'; + $SIG{'HUP'} = 'DEFAULT'; + $SIG{'PIPE'} = 'DEFAULT'; + $SIG{'TERM'} = 'DEFAULT'; + + # unblock all delayed signals (and possibly handle them) + POSIX::sigprocmask(SIG_UNBLOCK, $sigset) or error "Can't unblock signals: $!"; + open(STDOUT, '>', $options->{target}) or error "cannot open $options->{target} for writing: $!"; + open(STDIN, '<&', $rfh) or error "cannot open file handle for reading: $!"; + exec $tar_compressor or error "cannot exec $tar_compressor: $!"; + } + POSIX::sigprocmask(SIG_UNBLOCK, $sigset) or error "Can't unblock signals: $!"; + waitpid $cpid, 0; + if ($? != 0) { + error "failed to start $tar_compressor"; + } + } else { + if(!copy($rfh, $options->{target})) { + error "cannot copy to $options->{target}: $!"; + } + } } + }; + if ($@) { + # we cannot die here because that would leave the other thread + # running without a parent + warning "run_chroot failed: $@"; + $exitstatus = 1; } } close($rfh); @@ -2302,13 +2349,24 @@ specified and are appended to the chroot's sources.list in the given order. If any mirror contains a https URI, then the packages apt-transport-https and ca-certificates will be installed inside the chroot. -The I argument can either be a directory or a tarball filename. If -I is a directory, then it must not exist beforehand. A tarball -filename is detected by the filename extension of I. Choosing a -directory only makes sense with the B mode because otherwise the -contents of the chroot will not be owned by the superuser. If no I was -specified or if I is C<->, then a tarball of the chroot is written to -standard output. +The optional I argument can either be the path to a directory, the +path to a tarball filename or C<->. If I ends with C<.tar>, or with +any of the filename extensions listed in the section B, then +I will be interpreted as a path to a tarball filename. If I is +the path to a tarball filename or if I is C<-> or if no I was +specified, B will create a temporary chroot directory in +C<$TMPDIR> or F. If I is the path to a tarball filename, +B will create a tarball of that directory and store it as +I, optionally applying a compression algorithm as indicated by its +filename extension. If I is C<-> or if no I was specified, +then an uncompressed tarball of that directory will be sent to standard +output. If I does not end in C<.tar> or with any of the filename +extensions listed in the section B, then I will be +interpreted as the path to a directory. If the directory already exists, it +must either be empty or only contain an empty C directory. If a +directory is chosen as output in any other mode than B, then its +contents will have wrong ownership information and special device files will +be missing. The I may be a valid release code name (eg, sid, stretch, jessie) or a symbolic name (eg, unstable, testing, stable, oldstable). Any suite name that @@ -2642,6 +2700,33 @@ Limitations in comparison to debootstrap: =back +=head1 COMPRESSION + +B will choose a suitable compressor for the output tarball +depending on the filename extension. The following mapping from filename +extension to compressor applies: + + extension compressor + -------------------- + .tar none + .gz gzip + .tgz gzip + .taz gzip + .Z compress + .taZ compress + .bz2 bzip2 + .tbz bzip2 + .tbz2 bzip2 + .tz2 bzip2 + .lz lzip + .lzma lzma + .tlz lzma + .lzo lzop + .lz4 lz4 + .xz xz + .txz xz + .zst zstd + =head1 SEE ALSO debootstrap(8)