From 895c388ede0f0f7946e0c790e163a159c40d3b7a Mon Sep 17 00:00:00 2001 From: Johannes 'josch' Schauer Date: Thu, 9 Apr 2020 18:33:05 +0200 Subject: [PATCH] add --format option and ext2 image output --- coverage.sh | 74 ++++++++++- make_mirror.sh | 2 +- mmdebstrap | 343 ++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 339 insertions(+), 80 deletions(-) diff --git a/coverage.sh b/coverage.sh index e0175bd..5fa2806 100755 --- a/coverage.sh +++ b/coverage.sh @@ -72,7 +72,7 @@ if [ ! -e shared/taridshift ] || [ taridshift -nt shared/taridshift ]; then fi starttime= -total=141 +total=146 skipped=0 runtests=0 i=1 @@ -711,6 +711,31 @@ else runtests=$((runtests+1)) fi +print_header "mode=$defaultmode,variant=apt: directory ending in .tar" +cat << END > shared/test.sh +#!/bin/sh +set -eu +export LC_ALL=C.UTF-8 +$CMD --mode=$defaultmode --variant=apt --format=directory $DEFAULT_DIST /tmp/debian-chroot.tar $mirror +ftype=\$(stat -c %F /tmp/debian-chroot.tar) +if [ "\$ftype" != directory ]; then + echo "expected directory but got: \$ftype" >&2 + exit 1 +fi +tar -C /tmp/debian-chroot.tar --one-file-system -c . | tar -t | sort | diff -u tar1.txt - +rm -r /tmp/debian-chroot.tar +END +if [ "$HAVE_QEMU" = "yes" ]; then + ./run_qemu.sh + runtests=$((runtests+1)) +elif [ "$defaultmode" = "root" ]; then + ./run_null.sh SUDO + runtests=$((runtests+1)) +else + ./run_null.sh + runtests=$((runtests+1)) +fi + print_header "mode=$defaultmode,variant=apt: test squashfs image" cat << END > shared/test.sh #!/bin/sh @@ -737,6 +762,50 @@ else runtests=$((runtests+1)) fi +for mode in root unshare fakechroot proot; do + print_header "mode=$mode,variant=apt: test ext2 image" + cat << END > shared/test.sh +#!/bin/sh +set -eu +export LC_ALL=C.UTF-8 +if [ "\$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then + if [ ! -e /mmdebstrap-testenv ]; then + echo "this test modifies the system and should only be run inside a container" >&2 + exit 1 + fi + adduser --gecos user --disabled-password user +fi +if [ "$mode" = unshare ]; then + if [ ! -e /mmdebstrap-testenv ]; then + echo "this test modifies the system and should only be run inside a container" >&2 + exit 1 + fi + sysctl -w kernel.unprivileged_userns_clone=1 +fi +prefix= +[ "\$(id -u)" -eq 0 ] && [ "$mode" != "root" ] && prefix="runuser -u user --" +[ "$mode" = "fakechroot" ] && prefix="\$prefix fakechroot fakeroot" +\$prefix $CMD --mode=$mode --variant=apt $DEFAULT_DIST /tmp/debian-chroot.ext2 $mirror +mount /tmp/debian-chroot.ext2 /mnt +rmdir /mnt/lost+found +# in fakechroot mode, we use a fake ldconfig, so we have to +# artificially add some files +{ tar -C /mnt -c . | tar -t; + [ "$mode" = "fakechroot" ] && printf "./etc/ld.so.cache\n./var/cache/ldconfig/\n"; + [ "$mode" = "fakechroot" ] && printf "./etc/.pwd.lock\n"; +} | sort | diff -u tar1.txt - +umount /mnt +rm /tmp/debian-chroot.ext2 +END + if [ "$HAVE_QEMU" = "yes" ]; then + ./run_qemu.sh + runtests=$((runtests+1)) + else + echo "HAVE_QEMU != yes -- Skipping test..." >&2 + skipped=$((skipped+1)) + fi +done + print_header "mode=auto,variant=apt: test auto-mode without unshare capabilities" cat << END > shared/test.sh #!/bin/sh @@ -2017,8 +2086,9 @@ set -eu export LC_ALL=C.UTF-8 $CMD --mode=root --variant=apt --logfile=log $DEFAULT_DIST /tmp/debian-chroot $mirror # we check the full log to also prevent debug printfs to accidentally make it into a commit -cat << LOG | diff - log +cat << LOG | diff -u - log I: chroot architecture $HOSTARCH is equal to the host's architecture +I: automatically chosen format: directory I: running apt-get update... I: downloading packages with apt... I: extracting archives... diff --git a/make_mirror.sh b/make_mirror.sh index 925333c..61a19f7 100755 --- a/make_mirror.sh +++ b/make_mirror.sh @@ -424,7 +424,7 @@ if [ "$HAVE_QEMU" = "yes" ]; then tmpdir="$(mktemp -d)" trap "cleanuptmpdir; cleanup_newcachedir" EXIT INT TERM - pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,debootstrap,procps,apt-cudf,aspcud,squashfs-tools-ng,python3,libcap2-bin,gpg + pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,debootstrap,procps,apt-cudf,aspcud,squashfs-tools-ng,genext2fs,python3,libcap2-bin,gpg if [ "$HAVE_PROOT" = "yes" ]; then pkgs="$pkgs,proot" fi diff --git a/mmdebstrap b/mmdebstrap index 03a95da..679f72b 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -2771,6 +2771,27 @@ sub guess_sources_format { return; } +sub approx_disk_usage { + my $directory = shift; + info "approximating disk usage..."; + open my $fh, '-|', 'du', '--block-size', '1024', + '--summarize', '--one-file-system', + $directory // error "failed to fork(): $!"; + chomp( + my $du = do { local $/; <$fh> } + ); + close $fh; + if (($? != 0) or (!$du)) { + error "du failed: $?"; + } + if ($du =~ /^(\d+)\t/) { + return int($1 * 1.1); + } else { + error "unexpected du output: $du"; + } + return; +} + sub main() { umask 022; @@ -2840,6 +2861,7 @@ sub main() { dryrun => 0, }; my $logfile = undef; + my $format = 'auto'; Getopt::Long::Configure('default', 'bundling', 'auto_abbrev', 'ignore_case_always'); GetOptions( @@ -2878,6 +2900,7 @@ sub main() { 'q|quiet' => sub { $verbosity_level = 0; }, 'v|verbose' => sub { $verbosity_level = 2; }, 'd|debug' => sub { $verbosity_level = 3; }, + 'format=s' => \$format, 'logfile=s' => \$logfile, # no-op options so that mmdebstrap can be used with # sbuild-createchroot --debootstrap=mmdebstrap @@ -2955,6 +2978,19 @@ sub main() { error "invalid mode. Choose from " . (join ', ', @valid_modes); } + # sqfs is an alias for squashfs + if ($format eq 'sqfs') { + $format = 'squasfs'; + } + # dir is an alias for directory + if ($format eq 'dir') { + $format = 'directory'; + } + my @valid_formats = ('auto', 'directory', 'tar', 'squashfs', 'ext2'); + if (none { $_ eq $format } @valid_formats) { + error "invalid format. Choose from " . (join ', ', @valid_formats); + } + my $check_fakechroot_running = sub { # test if we are inside fakechroot already # We fork a child process because setting FAKECHROOT_DETECT seems to @@ -3658,62 +3694,88 @@ sub main() { my $tar_compressor = get_tar_compressor($options->{target}); - # figure out whether a tarball has to be created in the end - $options->{maketar} = 0; - $options->{makesqfs} = 0; - if ( - defined $tar_compressor - or $options->{target} =~ /\.tar$/ - or $options->{target} eq '-' - or -p $options->{target} # named pipe (fifo) - or -c $options->{target} # character special like /dev/null - ) { - $options->{maketar} = 1; - # check if the compressor is installed - if (defined $tar_compressor) { + # figure out the right format + if ($format eq 'auto') { + if ($options->{target} ne '-' and -d $options->{target}) { + $format = 'directory'; + } elsif ( + defined $tar_compressor + or $options->{target} =~ /\.tar$/ + or $options->{target} eq '-' + or -p $options->{target} # named pipe (fifo) + or -c $options->{target} # character special like /dev/null + ) { + $format = 'tar'; + # check if the compressor is installed + if (defined $tar_compressor) { + my $pid = fork() // error "fork() failed: $!"; + if ($pid == 0) { + open(STDOUT, '>', '/dev/null') + or error "cannot open /dev/null for writing: $!"; + open(STDIN, '<', '/dev/null') + or error "cannot open /dev/null for reading: $!"; + exec { $tar_compressor->[0] } @{$tar_compressor} + or error("cannot exec " + . (join " ", @{$tar_compressor}) + . ": $!"); + } + waitpid $pid, 0; + if ($? != 0) { + error("failed to start " . (join " ", @{$tar_compressor})); + } + } + } elsif ($options->{target} =~ /\.(squashfs|sqfs)$/) { + $format = 'squashfs'; + # check if tar2sqfs is installed my $pid = fork() // error "fork() failed: $!"; if ($pid == 0) { open(STDOUT, '>', '/dev/null') or error "cannot open /dev/null for writing: $!"; open(STDIN, '<', '/dev/null') or error "cannot open /dev/null for reading: $!"; - exec { $tar_compressor->[0] } @{$tar_compressor} - or error( - "cannot exec " . (join " ", @{$tar_compressor}) . ": $!"); + exec('tar2sqfs', '--version') + or error("cannot exec tar2sqfs --version: $!"); } waitpid $pid, 0; if ($? != 0) { - error("failed to start " . (join " ", @{$tar_compressor})); + error("failed to start tar2sqfs --version"); } + } elsif ($options->{target} =~ /\.ext2$/) { + $format = 'ext2'; + # check if the installed version of genext2fs supports tarballs on + # stdin + (undef, my $filename) + = tempfile("mmdebstrap.ext2.XXXXXXXXXXXX", OPEN => 0); + open my $fh, '|-', 'genext2fs', '-B', '1024', '-b', '8', '-N', + '11', + $filename // error "failed to fork(): $!"; + # write 10240 null-bytes to genext2fs -- this represents an empty + # tar archive + print $fh ("\0" x 10240) + or error "cannot write to genext2fs process"; + close $fh; + my $exitstatus = $?; + unlink $filename // die "cannot unlink $filename"; + if ($exitstatus != 0) { + error "genext2fs failed with exit status: $exitstatus"; + } + } else { + $format = 'directory'; } - } elsif ($options->{target} =~ /\.(squashfs|sqfs)$/) { - $options->{makesqfs} = 1; - # check if tar2sqfs is installed - my $pid = fork() // error "fork() failed: $!"; - if ($pid == 0) { - open(STDOUT, '>', '/dev/null') - or error "cannot open /dev/null for writing: $!"; - open(STDIN, '<', '/dev/null') - or error "cannot open /dev/null for reading: $!"; - exec('tar2sqfs', '--version') - or error("cannot exec tar2sqfs --version: $!"); - } - waitpid $pid, 0; - if ($? != 0) { - error("failed to start tar2sqfs --version"); - } - } elsif ($options->{target} =~ /\.ext2$/) { - error "genext2fs does not yet support tarballs as input. See " - . "https://github.com/bestouff/genext2fs/issues/10 for more " - . "information"; + info "automatically chosen format: $format"; } - if ($options->{maketar} or $options->{makesqfs}) { + if ($options->{target} eq '-' and $format ne 'tar') { + error "the $format format is unable to write to standard output"; + } + + if (any { $_ eq $format } ('tar', 'squashfs', 'ext2')) { if ( any { $_ eq $options->{variant} } ('extract', 'custom') and any { $_ eq $options->{mode} } ('fakechroot', 'proot')) { - info "creating a tarball or squashfs image in fakechroot mode or" - . " proot mode might fail in extract and custom variants because" - . " there might be no tar inside the chroot"; + info "creating a tarball or squashfs image or ext2 image in" + . " fakechroot mode or proot mode might fail in extract and" + . " custom variants because there might be no tar inside the" + . " chroot"; } # try to fail early if target tarball or squashfs image cannot be # opened for writing @@ -3739,7 +3801,7 @@ sub main() { if (any { $_ eq $options->{mode} } ('unshare', 'root')) { chmod 0755, $options->{root} or error "cannot chmod root: $!"; } - } else { + } elsif ($format eq 'directory') { # user does not seem to have specified a tarball as output, thus work # directly in the supplied directory $options->{root} = $options->{target}; @@ -3789,6 +3851,8 @@ sub main() { error "cannot create $options->{root}"; } } + } else { + error "unknown format: $format"; } # check for double quotes because apt doesn't allow to escape them and @@ -3844,7 +3908,7 @@ sub main() { my $devtar = ''; # We always craft the /dev entries ourselves if a tarball is to be created - if ($options->{maketar} or $options->{makesqfs}) { + if (any { $_ eq $format } ('tar', 'squashfs', 'ext2')) { foreach my $file (@devfiles) { my ($fname, $mode, $type, $linkname, $devmajor, $devminor) = @{$file}; @@ -3871,8 +3935,13 @@ sub main() { = sprintf("%06o\0", unpack("%16C*", $entry)); $devtar .= $entry; } + } elsif ($format eq 'directory') { + # nothing to do + } else { + error "unknown format: $format"; } + my $numblocks = 0; my $exitstatus = 0; my @taropts = ( '--sort=name', @@ -3920,12 +3989,24 @@ sub main() { setup($options); + if (!$options->{dryrun} && $format eq 'ext2') { + my $numblocks = approx_disk_usage($options->{root}); + debug "sending nblks"; + print $childsock ( + pack("n", length "$numblocks") . "nblks$numblocks"); + $childsock->flush(); + debug "waiting for okthx"; + checkokthx $childsock; + } + print $childsock (pack('n', 0) . 'adios'); $childsock->flush(); close $childsock; - if ($options->{maketar} or $options->{makesqfs}) { + if ($options->{dryrun}) { + info "simulate creating tarball..."; + } elsif (any { $_ eq $format } ('tar', 'squashfs', 'ext2')) { info "creating tarball..."; # redirect tar output to the writing end of the pipe so @@ -3942,6 +4023,10 @@ sub main() { or error "tar failed: $?"; info "done"; + } elsif ($format eq 'directory') { + # nothing to do + } else { + error "unknown format: $format"; } exit 0; @@ -3969,6 +4054,15 @@ sub main() { setup($options); + if (!$options->{dryrun} && $format eq 'ext2') { + my $numblocks = approx_disk_usage($options->{root}); + print $childsock ( + pack("n", length "$numblocks") . "nblks$numblocks"); + $childsock->flush(); + debug "waiting for okthx"; + checkokthx $childsock; + } + print $childsock (pack('n', 0) . 'adios'); $childsock->flush(); @@ -3976,7 +4070,7 @@ sub main() { if ($options->{dryrun}) { info "simulate creating tarball..."; - } elsif ($options->{maketar} or $options->{makesqfs}) { + } elsif (any { $_ eq $format } ('tar', 'squashfs', 'ext2')) { info "creating tarball..."; # redirect tar output to the writing end of the pipe so that @@ -4028,6 +4122,10 @@ sub main() { } info "done"; + } elsif ($format eq 'directory') { + # nothing to do + } else { + error "unknown format: $format"; } exit 0; @@ -4360,6 +4458,23 @@ sub main() { if ($? != 0) { error "tar failed"; } + } elsif ($msg eq "nblks") { + # handle the nblks message + debug "received message: nblks"; + { + my $ret = read($parentsock, $numblocks, $len) + // error "cannot read from socket: $!"; + if ($ret == 0) { + error "received eof on socket"; + } + } + if ($numblocks !~ /^\d+$/) { + error "invalid number of blocks: $numblocks"; + } + debug "sending okthx"; + print $parentsock (pack("n", 0) . "okthx") + or error "cannot write to socket: $!"; + $parentsock->flush(); } else { error "unknown message: $msg"; } @@ -4381,7 +4496,9 @@ sub main() { if ($options->{dryrun}) { # nothing to do - } elsif ($options->{maketar} or $options->{makesqfs}) { + } elsif ($format eq 'directory') { + # nothing to do + } elsif (any { $_ eq $format } ('tar', 'squashfs', 'ext2')) { # we use eval() so that error() doesn't take this process down and # thus leaves the setup() process without a parent eval { @@ -4390,17 +4507,27 @@ sub main() { error "cannot copy to standard output: $!"; } } else { - if ($options->{makesqfs} or defined $tar_compressor) { + if ( $format eq 'squashfs' + or $format eq 'ext2' + or defined $tar_compressor) { my @argv = (); - if ($options->{makesqfs}) { + if ($format eq 'squashfs') { push @argv, 'tar2sqfs', '--quiet', '--no-skip', '--force', '--exportable', '--compressor', 'xz', '--block-size', '1048576', $options->{target}; - } else { + } elsif ($format eq 'ext2') { + if ($numblocks <= 0) { + error "invalid number of blocks: $numblocks"; + } + push @argv, 'genext2fs', '-B', 1024, '-b', $numblocks, + '-N', '0', $options->{target}; + } elsif ($format eq 'tar') { push @argv, @{$tar_compressor}; + } else { + error "unknown format: $format"; } POSIX::sigprocmask(SIG_BLOCK, $sigset) or error "Can't block signals: $!"; @@ -4417,13 +4544,16 @@ sub main() { POSIX::sigprocmask(SIG_UNBLOCK, $sigset) or error "Can't unblock signals: $!"; - if ($options->{makesqfs}) { + # redirect stdout to file or /dev/null + if ($format eq 'squashfs' or $format eq 'ext2') { open(STDOUT, '>', '/dev/null') or error "cannot open /dev/null for writing: $!"; - } else { + } elsif ($format eq 'tar') { open(STDOUT, '>', $options->{target}) or error "cannot open $options->{target} for writing: $!"; + } else { + error "unknown format: $format"; } open(STDIN, '<&', $rfh) or error "cannot open file handle for reading: $!"; @@ -4452,6 +4582,8 @@ sub main() { warning "creating tarball failed: $@"; $exitstatus = 1; } + } else { + error "unknown format: $format"; } close($rfh); waitpid $pid, 0; @@ -4462,8 +4594,12 @@ sub main() { # change signal handler message $waiting_for = "cleanup"; - if (($options->{maketar} or $options->{makesqfs}) - and -e $options->{root}) { + if ($format eq 'directory') { + # nothing to do + } elsif (any { $_ eq $format } ('tar', 'squashfs', 'ext2')) { + if (!-e $options->{root}) { + error "$options->{root} does not exist"; + } info "removing tempdir $options->{root}..."; if ($options->{mode} eq 'unshare') { # We don't have permissions to remove the directory outside @@ -4527,6 +4663,8 @@ sub main() { } else { error "unknown mode: $options->{mode}"; } + } else { + error "unknown format: $format"; } if ($got_signal) { @@ -4575,30 +4713,11 @@ installed inside the chroot. If any mirror contains a tor+xxx URI, then the apt-transport-tor package will be installed inside the chroot. The optional I argument can either be the path to a directory, the path -to a tarball filename, the path to a squashfs image, a FIFO, a character -special device, or C<->. If I ends with C<.tar>, or with any of the -filename extensions listed in the section B, or if I is a -FIFO or a character special device, then I will be interpreted as a -path to a tarball filename. If I ends with C<.squashfs> or C<.sqfs>, -then I will be interpreted as a path to a squashfs image. If I -is the path to a tarball filename or a squashfs image or if I is C<-> -or if no I was specified, B will create a temporary chroot -directory in C<$TMPDIR> or F. If I is the path to a tarball -filename, B will create a tarball of that directory and store it as -I, optionally applying a compression algorithm as indicated by its -filename extension. If I is C<-> or if no I was specified, then -an uncompressed tarball of that directory will be sent to standard output. When -B creates a tarball it also stores extended attributes. To preserve -the extended attributes, you have to pass B<--xattrs --xattrs-include='*'> to -tar when extracting the tarball. If I is the path to a squashfs image, -B will create an xz compressed image with a blocksize of 1048576 -bytes. If I does neither end with C<.tar> nor with any of the filename -extensions listed in the section B, nor with C<.squashfs> or -C<.sqfs>, then I will be interpreted as the path to a directory. If the -directory already exists, it must either be empty or only contain an empty -C directory. If a directory is chosen as output in any other mode -than B, then its contents will have wrong ownership information and -special device files will be missing. +to a tarball filename, the path to a squashfs image, the path to an ext2 image, +a FIFO, a character special device, or C<->. Without the B<--format> option, +I will be used to choose the format. See the section B for +more information. If no I was specified or if I is C<->, an +uncompressed tarball will be sent to standard output. The I may be a valid release code name (eg, sid, stretch, jessie) or a symbolic name (eg, unstable, testing, stable, oldstable). Any suite name that @@ -4651,6 +4770,12 @@ B, B, B, B, B, B, B and B. The default mode is B. See the section B for more information. +=item B<--format>=I + +Choose the output format. Valid format Is are B, B, +B, B, and B. The default format is B. See the +section B for more information. + =item B<--aptopt>=I