From dbdf3f34c6d1d26e1744257e846a18b2d59ff975 Mon Sep 17 00:00:00 2001 From: Johannes 'josch' Schauer Date: Tue, 7 Jan 2020 17:40:13 +0100 Subject: [PATCH] add support for generating squashfs images using tar2sqfs --- coverage.sh | 25 +++++++++++- make_mirror.sh | 2 +- mmdebstrap | 104 +++++++++++++++++++++++++++++++++---------------- 3 files changed, 96 insertions(+), 35 deletions(-) diff --git a/coverage.sh b/coverage.sh index 43157a0..f87361f 100755 --- a/coverage.sh +++ b/coverage.sh @@ -49,7 +49,7 @@ if [ ! -e shared/mmdebstrap ] || [ mmdebstrap -nt shared/mmdebstrap ]; then fi starttime= -total=122 +total=123 skipped=0 runtests=0 i=1 @@ -547,6 +547,29 @@ else runtests=$((runtests+1)) fi +print_header "mode=$defaultmode,variant=apt: test squashfs image" +cat << END > shared/test.sh +#!/bin/sh +set -eu +export LC_ALL=C.UTF-8 +$CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST /tmp/debian-chroot.squashfs $mirror +printf 'hsqs' | cmp --bytes=4 /tmp/debian-chroot.squashfs - +# workaround for https://github.com/AgentD/squashfs-tools-ng/issues/37 +sed 's#\\([^.]\\)/\$#\\1#' tar1.txt | sort > /tmp/tar1noslash.txt +sqfs2tar --no-skip --root-becomes . /tmp/debian-chroot.squashfs | tar -t | sort | diff -u /tmp/tar1noslash.txt - +rm /tmp/debian-chroot.squashfs /tmp/tar1noslash.txt +END +if [ "$HAVE_QEMU" = "yes" ]; then + ./run_qemu.sh + runtests=$((runtests+1)) +elif [ "$defaultmode" = "root" ]; then + ./run_null.sh SUDO + runtests=$((runtests+1)) +else + ./run_null.sh + runtests=$((runtests+1)) +fi + print_header "mode=auto,variant=apt: test auto-mode without unshare capabilities" cat << END > shared/test.sh #!/bin/sh diff --git a/make_mirror.sh b/make_mirror.sh index f5ac97a..5f8f5a5 100755 --- a/make_mirror.sh +++ b/make_mirror.sh @@ -407,7 +407,7 @@ if [ "$HAVE_QEMU" = "yes" ]; then tmpdir="$(mktemp -d)" trap "cleanuptmpdir; cleanup_newcachedir" EXIT INT TERM - pkgs=perl-doc,linux-image-amd64,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,proot,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,debootstrap,procps,apt-cudf,aspcud + pkgs=perl-doc,linux-image-amd64,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,proot,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,debootstrap,procps,apt-cudf,aspcud,squashfs-tools-ng if [ "$HOSTARCH" = amd64 ] && [ "$RUN_MA_SAME_TESTS" = "yes" ]; then arches=amd64,armhf pkgs="$pkgs,libfakechroot:armhf,libfakeroot:armhf" diff --git a/mmdebstrap b/mmdebstrap index 1f6c135..b7607bd 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -2707,16 +2707,9 @@ sub main() { # figure out whether a tarball has to be created in the end $options->{maketar} = 0; + $options->{makesqfs} = 0; if (defined $tar_compressor or $options->{target} =~ /\.tar$/ or $options->{target} eq '-') { $options->{maketar} = 1; - if (any { $_ eq $options->{variant} } ('extract', 'custom') and any { $_ eq $options->{mode} } ('fakechroot', 'proot')) { - info "creating a tarball in fakechroot mode or proot mode might fail in extract and custom variants because there might be no tar inside the chroot"; - } - # try to fail early if target tarball cannot be opened for writing - if ($options->{target} ne '-') { - open my $fh, '>', $options->{target} or error "cannot open $options->{target} for writing: $!"; - close $fh; - } # check if the compressor is installed if (defined $tar_compressor) { my $pid = fork() // error "fork() failed: $!"; @@ -2730,9 +2723,31 @@ sub main() { error ("failed to start " . (join " ", @{$tar_compressor})); } } + } elsif ($options->{target} =~ /\.(squashfs|sqfs)$/) { + $options->{makesqfs} = 1; + # check if tar2sqfs is installed + my $pid = fork() // error "fork() failed: $!"; + if ($pid == 0) { + open(STDOUT, '>', '/dev/null') or error "cannot open /dev/null for writing: $!"; + open(STDIN, '<', '/dev/null') or error "cannot open /dev/null for reading: $!"; + exec ('tar2sqfs', '--version') or error ("cannot exec tar2sqfs --version: $!"); + } + waitpid $pid, 0; + if ($? != 0) { + error ("failed to start tar2sqfs --version"); + } } - if ($options->{maketar}) { + if ($options->{maketar} or $options->{makesqfs}) { + if (any { $_ eq $options->{variant} } ('extract', 'custom') and any { $_ eq $options->{mode} } ('fakechroot', 'proot')) { + info "creating a tarball or squashfs image in fakechroot mode or proot mode might fail in extract and custom variants because there might be no tar inside the chroot"; + } + # try to fail early if target tarball or squashfs image cannot be + # opened for writing + if ($options->{target} ne '-') { + open my $fh, '>', $options->{target} or error "cannot open $options->{target} for writing: $!"; + close $fh; + } # since the output is a tarball, we create the rootfs in a temporary # directory $options->{root} = tempdir( @@ -2837,7 +2852,7 @@ sub main() { my $devtar = ''; # We always craft the /dev entries ourselves if a tarball is to be created - if ($options->{maketar}) { + if ($options->{maketar} or $options->{makesqfs}) { foreach my $file (@devfiles) { my ($fname, $mode, $type, $linkname, $devmajor, $devminor) = @{$file}; my $entry = pack('a100 a8 a8 a8 a12 a12 A8 a1 a100 a8 a32 a32 a8 a8 a155 x12', @@ -2901,7 +2916,7 @@ sub main() { close $childsock; - if ($options->{maketar}) { + if ($options->{maketar} or $options->{makesqfs}) { info "creating tarball..."; # redirect tar output to the writing end of the pipe so that the @@ -2943,7 +2958,7 @@ sub main() { close $childsock; - if ($options->{maketar}) { + if ($options->{maketar} or $options->{makesqfs}) { info "creating tarball..."; # redirect tar output to the writing end of the pipe so that the @@ -3306,7 +3321,7 @@ sub main() { close $parentsock; - if ($options->{maketar}) { + if ($options->{maketar} or $options->{makesqfs}) { # we use eval() so that error() doesn't take this process down and # thus leaves the setup() process without a parent eval { @@ -3315,7 +3330,17 @@ sub main() { error "cannot copy to standard output: $!"; } } else { - if (defined $tar_compressor) { + if ($options->{makesqfs} or defined $tar_compressor) { + my @argv = (); + if ($options->{makesqfs}) { + push @argv, 'tar2sqfs', + '--quiet', '--no-skip', '--force', '--exportable', + '--compressor', 'xz', + '--block-size', '1048576', + $options->{target}; + } else { + push @argv, @{$tar_compressor}; + } POSIX::sigprocmask(SIG_BLOCK, $sigset) or error "Can't block signals: $!"; my $cpid = fork() // error "fork() failed: $!"; if ($cpid == 0) { @@ -3327,14 +3352,19 @@ sub main() { # unblock all delayed signals (and possibly handle them) POSIX::sigprocmask(SIG_UNBLOCK, $sigset) or error "Can't unblock signals: $!"; - open(STDOUT, '>', $options->{target}) or error "cannot open $options->{target} for writing: $!"; + + if ($options->{makesqfs}) { + open(STDOUT, '>', '/dev/null') or error "cannot open /dev/null for writing: $!"; + } else { + open(STDOUT, '>', $options->{target}) or error "cannot open $options->{target} for writing: $!"; + } open(STDIN, '<&', $rfh) or error "cannot open file handle for reading: $!"; - exec { $tar_compressor->[0] } @{$tar_compressor} or error ("cannot exec " . (join " ", @{$tar_compressor}) . ": $!"); + exec { $argv[0] } @argv or error ("cannot exec " . (join " ", @argv) . ": $!"); } POSIX::sigprocmask(SIG_UNBLOCK, $sigset) or error "Can't unblock signals: $!"; waitpid $cpid, 0; if ($? != 0) { - error ("failed to start " . (join " ", @{$tar_compressor})); + error ("failed to start " . (join " ", @argv)); } } else { if(!copy($rfh, $options->{target})) { @@ -3359,7 +3389,7 @@ sub main() { # change signal handler message $waiting_for = "cleanup"; - if ($options->{maketar} and -e $options->{root}) { + if (($options->{maketar} or $options->{makesqfs}) and -e $options->{root}) { info "removing tempdir $options->{root}..."; if ($options->{mode} eq 'unshare') { # We don't have permissions to remove the directory outside @@ -3447,25 +3477,29 @@ installed inside the chroot. If any mirror contains a tor+xxx URI, then the apt-transport-tor package will be installed inside the chroot. The optional I argument can either be the path to a directory, the -path to a tarball filename or C<->. If I ends with C<.tar>, or with -any of the filename extensions listed in the section B, then -I will be interpreted as a path to a tarball filename. If I is -the path to a tarball filename or if I is C<-> or if no I was -specified, B will create a temporary chroot directory in -C<$TMPDIR> or F. If I is the path to a tarball filename, -B will create a tarball of that directory and store it as -I, optionally applying a compression algorithm as indicated by its +path to a tarball filename, the path to a squashfs image or C<->. If I +ends with C<.tar>, or with any of the filename extensions listed in the +section B, then I will be interpreted as a path to a +tarball filename. If I ends with C<.squashfs> or C<.sqfs>, then +I will be interpreted as a path to a squashfs image. If I is +the path to a tarball filename or a squashfs image or if I is C<-> or +if no I was specified, B will create a temporary chroot +directory in C<$TMPDIR> or F. If I is the path to a tarball +filename, B will create a tarball of that directory and store it +as I, optionally applying a compression algorithm as indicated by its filename extension. If I is C<-> or if no I was specified, then an uncompressed tarball of that directory will be sent to standard output. When B creates a tarball it also stores extended attributes. To preserve the extended attributes, you have to pass B<--xattrs ---xattrs-include='*'> to tar when extracting the tarball. If I does -not end in C<.tar> or with any of the filename extensions listed in the -section B, then I will be interpreted as the path to a -directory. If the directory already exists, it must either be empty or only -contain an empty C directory. If a directory is chosen as output -in any other mode than B, then its contents will have wrong ownership -information and special device files will be missing. +--xattrs-include='*'> to tar when extracting the tarball. If I is the +path to a squashfs image, B will create an xz compressed image +with a blocksize of 1048576 bytes. If I does neither end with C<.tar> +nor with any of the filename extensions listed in the section B, +nor with C<.squashfs> or C<.sqfs>, then I will be interpreted as the +path to a directory. If the directory already exists, it must either be empty +or only contain an empty C directory. If a directory is chosen as +output in any other mode than B, then its contents will have wrong +ownership information and special device files will be missing. The I may be a valid release code name (eg, sid, stretch, jessie) or a symbolic name (eg, unstable, testing, stable, oldstable). Any suite name that @@ -3938,6 +3972,10 @@ create device nodes: $ mmdebstrap unstable | tar --delete ./dev > unstable-chroot.tar +Instead of a tarball, a squashfs image can be created: + + $ mmdebstrap unstable unstable-chroot.squashfs + By default, debootstrapping a stable distribution will add mirrors for security and updates to the sources.list.