Instead of using gzip compression support, run compressor ourselves

- otherwise ./dev tar cannot be concatenated with the rest
 - test compressor early
 - better document the TARGET option
This commit is contained in:
Johannes 'josch' Schauer 2019-01-20 10:41:29 +01:00
parent 84b0b5061b
commit a3afe24fc0
Signed by untrusted user: josch
GPG key ID: F2CBA5C78FBD83E1
2 changed files with 121 additions and 36 deletions

View file

@ -251,17 +251,17 @@ else
./run_null.sh SUDO ./run_null.sh SUDO
fi fi
print_header "mode=unshare,variant=apt: create tarball" print_header "mode=unshare,variant=apt: create gzip compressed tarball"
cat << END > shared/test.sh cat << END > shared/test.sh
#!/bin/sh #!/bin/sh
set -eu set -eu
export LC_ALL=C.UTF-8 export LC_ALL=C.UTF-8
adduser --gecos user --disabled-password user adduser --gecos user --disabled-password user
sysctl -w kernel.unprivileged_userns_clone=1 sysctl -w kernel.unprivileged_userns_clone=1
runuser -u user -- $CMD --mode=unshare --variant=apt unstable /tmp/unstable-chroot.tar $mirror runuser -u user -- $CMD --mode=unshare --variant=apt unstable /tmp/unstable-chroot.tar.gz $mirror
tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt tar -tf /tmp/unstable-chroot.tar.gz | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm /tmp/unstable-chroot.tar rm /tmp/unstable-chroot.tar.gz
END END
if [ "$HAVE_QEMU" = "yes" ]; then if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh ./run_qemu.sh

View file

@ -130,26 +130,32 @@ sub error {
# tar cannot figure out the decompression program when receiving data on # tar cannot figure out the decompression program when receiving data on
# standard input, thus we do it ourselves. This is copied from tar's # standard input, thus we do it ourselves. This is copied from tar's
# src/suffix.c # src/suffix.c
sub get_tar_compress_options($) { sub get_tar_compressor($) {
my $filename = shift; my $filename = shift;
if ($filename =~ /\.(gz|tgz|taz)$/) { if ($filename eq '-') {
return ('--gzip'); return undef
} elsif ($filename =~ /\.tar$/) {
return undef
} elsif ($filename =~ /\.(gz|tgz|taz)$/) {
return 'gzip';
} elsif ($filename =~ /\.(Z|taZ)$/) { } elsif ($filename =~ /\.(Z|taZ)$/) {
return ('--compress'); return 'compress';
} elsif ($filename =~ /\.(bz2|tbz|tbz2|tz2)$/) { } elsif ($filename =~ /\.(bz2|tbz|tbz2|tz2)$/) {
return ('--bzip2'); return 'bzip2';
} elsif ($filename =~ /\.lz$/) { } elsif ($filename =~ /\.lz$/) {
return ('--lzip'); return 'lzip';
} elsif ($filename =~ /\.(lzma|tlz)$/) { } elsif ($filename =~ /\.(lzma|tlz)$/) {
return ('--lzma'); return 'lzma';
} elsif ($filename =~ /\.lzo$/) { } elsif ($filename =~ /\.lzo$/) {
return ('--lzop'); return 'lzop';
} elsif ($filename =~ /\.lz4$/) { } elsif ($filename =~ /\.lz4$/) {
return ('--use-compress-program', 'lz4'); return 'lz4';
} elsif ($filename =~ /\.(xz|txz)$/) { } elsif ($filename =~ /\.(xz|txz)$/) {
return ('--xz'); return 'xz';
} elsif ($filename =~ /\.zst$/) {
return 'zstd';
} }
return (); return undef
} }
sub test_unshare($) { sub test_unshare($) {
@ -1931,11 +1937,11 @@ sub main() {
error "refusing to use the filesystem root as output directory"; error "refusing to use the filesystem root as output directory";
} }
my @tar_compress_opts = get_tar_compress_options($options->{target}); my $tar_compressor = get_tar_compressor($options->{target});
# figure out whether a tarball has to be created in the end # figure out whether a tarball has to be created in the end
$options->{maketar} = 0; $options->{maketar} = 0;
if (scalar @tar_compress_opts > 0 or $options->{target} =~ /\.tar$/ or $options->{target} eq '-') { if (defined $tar_compressor or $options->{target} =~ /\.tar$/ or $options->{target} eq '-') {
$options->{maketar} = 1; $options->{maketar} = 1;
if (any { $_ eq $options->{variant} } ('extract', 'custom') and $options->{mode} eq 'fakechroot') { if (any { $_ eq $options->{variant} } ('extract', 'custom') and $options->{mode} eq 'fakechroot') {
info "creating a tarball in fakechroot mode might fail in extract and custom variants because there might be no tar inside the chroot"; info "creating a tarball in fakechroot mode might fail in extract and custom variants because there might be no tar inside the chroot";
@ -1945,6 +1951,19 @@ sub main() {
open my $fh, '>', $options->{target} or error "cannot open $options->{target} for writing: $!"; open my $fh, '>', $options->{target} or error "cannot open $options->{target} for writing: $!";
close $fh; close $fh;
} }
# check if the compressor is installed
if (defined $tar_compressor) {
my $pid = fork();
if ($pid == 0) {
open(STDOUT, '>', '/dev/null') or error "cannot open /dev/null for writing: $!";
open(STDIN, '<', '/dev/null') or error "cannot open /dev/null for reading: $!";
exec $tar_compressor or error "cannot exec $tar_compressor: $!";
}
waitpid $pid, 0;
if ($? != 0) {
error "failed to start $tar_compressor";
}
}
} }
if ($options->{maketar}) { if ($options->{maketar}) {
@ -2074,7 +2093,6 @@ sub main() {
my $exitstatus = 0; my $exitstatus = 0;
my @taropts = ('--sort=name', "--mtime=\@$mtime", '--clamp-mtime', '--numeric-owner', '--one-file-system', '-c', '--exclude=./dev'); my @taropts = ('--sort=name', "--mtime=\@$mtime", '--clamp-mtime', '--numeric-owner', '--one-file-system', '-c', '--exclude=./dev');
push @taropts, @tar_compress_opts;
# disable signals so that we can fork and change behaviour of the signal # disable signals so that we can fork and change behaviour of the signal
# handler in the parent and child without getting interrupted # handler in the parent and child without getting interrupted
@ -2196,18 +2214,47 @@ sub main() {
close $wfh; close $wfh;
if ($options->{maketar}) { if ($options->{maketar}) {
# we cannot die here because that would leave the other thread running # we use eval() so that error() doesn't take this process down and
# without a parent # thus leaves the setup() process without a parent
if ($options->{target} ne '-') { eval {
if(!copy($rfh, $options->{target})) { if ($options->{target} eq '-') {
warning "cannot copy to $options->{target}: $!"; if (!copy($rfh, *STDOUT)) {
$exitstatus = 1; error "cannot copy to standard output: $!";
} }
} else { } else {
if (!copy($rfh, *STDOUT)) { if (defined $tar_compressor) {
warning "cannot copy to standard output: $!"; POSIX::sigprocmask(SIG_BLOCK, $sigset) or error "Can't block signals: $!";
$exitstatus = 1; my $cpid = fork();
if ($cpid == 0) {
# child: default signal handlers
$SIG{'INT'} = 'DEFAULT';
$SIG{'HUP'} = 'DEFAULT';
$SIG{'PIPE'} = 'DEFAULT';
$SIG{'TERM'} = 'DEFAULT';
# unblock all delayed signals (and possibly handle them)
POSIX::sigprocmask(SIG_UNBLOCK, $sigset) or error "Can't unblock signals: $!";
open(STDOUT, '>', $options->{target}) or error "cannot open $options->{target} for writing: $!";
open(STDIN, '<&', $rfh) or error "cannot open file handle for reading: $!";
exec $tar_compressor or error "cannot exec $tar_compressor: $!";
} }
POSIX::sigprocmask(SIG_UNBLOCK, $sigset) or error "Can't unblock signals: $!";
waitpid $cpid, 0;
if ($? != 0) {
error "failed to start $tar_compressor";
}
} else {
if(!copy($rfh, $options->{target})) {
error "cannot copy to $options->{target}: $!";
}
}
}
};
if ($@) {
# we cannot die here because that would leave the other thread
# running without a parent
warning "run_chroot failed: $@";
$exitstatus = 1;
} }
} }
close($rfh); close($rfh);
@ -2302,13 +2349,24 @@ specified and are appended to the chroot's sources.list in the given order. If
any mirror contains a https URI, then the packages apt-transport-https and any mirror contains a https URI, then the packages apt-transport-https and
ca-certificates will be installed inside the chroot. ca-certificates will be installed inside the chroot.
The I<TARGET> argument can either be a directory or a tarball filename. If The optional I<TARGET> argument can either be the path to a directory, the
I<TARGET> is a directory, then it must not exist beforehand. A tarball path to a tarball filename or C<->. If I<TARGET> ends with C<.tar>, or with
filename is detected by the filename extension of I<TARGET>. Choosing a any of the filename extensions listed in the section B<COMPRESSION>, then
directory only makes sense with the B<sudo> mode because otherwise the I<TARGET> will be interpreted as a path to a tarball filename. If I<TARGET> is
contents of the chroot will not be owned by the superuser. If no I<TARGET> was the path to a tarball filename or if I<TARGET> is C<-> or if no I<TARGET> was
specified or if I<TARGET> is C<->, then a tarball of the chroot is written to specified, B<mmdebstrap> will create a temporary chroot directory in
standard output. C<$TMPDIR> or F</tmp>. If I<TARGET> is the path to a tarball filename,
B<mmdebstrap> will create a tarball of that directory and store it as
I<TARGET>, optionally applying a compression algorithm as indicated by its
filename extension. If I<TARGET> is C<-> or if no I<TARGET> was specified,
then an uncompressed tarball of that directory will be sent to standard
output. If I<TARGET> does not end in C<.tar> or with any of the filename
extensions listed in the section B<COMPRESSION>, then I<TARGET> will be
interpreted as the path to a directory. If the directory already exists, it
must either be empty or only contain an empty C<lost+found> directory. If a
directory is chosen as output in any other mode than B<sudo>, then its
contents will have wrong ownership information and special device files will
be missing.
The I<SUITE> may be a valid release code name (eg, sid, stretch, jessie) or a The I<SUITE> may be a valid release code name (eg, sid, stretch, jessie) or a
symbolic name (eg, unstable, testing, stable, oldstable). Any suite name that symbolic name (eg, unstable, testing, stable, oldstable). Any suite name that
@ -2642,6 +2700,33 @@ Limitations in comparison to debootstrap:
=back =back
=head1 COMPRESSION
B<mmdebstrap> will choose a suitable compressor for the output tarball
depending on the filename extension. The following mapping from filename
extension to compressor applies:
extension compressor
--------------------
.tar none
.gz gzip
.tgz gzip
.taz gzip
.Z compress
.taZ compress
.bz2 bzip2
.tbz bzip2
.tbz2 bzip2
.tz2 bzip2
.lz lzip
.lzma lzma
.tlz lzma
.lzo lzop
.lz4 lz4
.xz xz
.txz xz
.zst zstd
=head1 SEE ALSO =head1 SEE ALSO
debootstrap(8) debootstrap(8)