release 0.8.1

* remove_tree() requires the CWD to be accessible or fails with
   cannot chdir to $CWD from $DIR_TO_DELETE: Permission denied, aborting.
 * CWD is not always accessible -- example: run mmdebstrap from a
   directory only accessible by the current user (like a tempdir) in
   unshare mode
 * find from findutils *also* requires CWD to be accessible but it's
   easier to temporarily change CWD in a subprocess because using
   there is no utility in perl core that changes CWD temporarily and
   cleans up after itself
 * we need to use find from findutils instead of rm in unshare mode
   because the root directory itself might not be removable by the
   unshared user so we only want to remove its subdirectories

CHANGELOG.md

@@ -1,3 +1,12 @@
+0.8.1 (2021-10-07)
+------------------
+
+ - enforce dpkg >= 1.20.0 and apt >= 2.3.7
+ - allow working directory be not world readable
+ - do not run xz and zstd with --threads=0 since this is a bad default for
+   machines with more than 100 cores
+ - bit-by-bit identical chrootless mode
+
 0.8.0 (2021-09-21)
 ------------------
 

coverage.sh

@@ -127,7 +127,7 @@ if [ ! -e shared/hooks/eatmydata/customize.sh ] || [ hooks/eatmydata/customize.s
 	fi
 fi
 starttime=
-total=212
+total=213
 skipped=0
 runtests=0
 i=1
@@ -1055,6 +1055,39 @@ else
 	runtests=$((runtests+1))
 fi
 
+print_header "mode=unshare,variant=apt: CWD directory not accessible by unshared user"
+cat << END > shared/test.sh
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+adduser --gecos user --disabled-password user
+sysctl -w kernel.unprivileged_userns_clone=1
+mkdir /tmp/debian-chroot
+chmod 700 /tmp/debian-chroot
+chown user:user /tmp/debian-chroot
+if [ "$CMD" = "./mmdebstrap" ]; then
+	CMD=\$(realpath --canonicalize-existing ./mmdebstrap)
+elif [ "$CMD" = "perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap" ]; then
+	CMD="perl -MDevel::Cover=-silent,-nogcov \$(realpath --canonicalize-existing ./mmdebstrap)"
+else
+	CMD="$CMD"
+fi
+env --chdir=/tmp/debian-chroot runuser -u user -- \$CMD --mode=unshare --variant=apt $DEFAULT_DIST /tmp/debian-chroot.tar $mirror
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+rm /tmp/debian-chroot.tar
+END
+if [ "$HAVE_QEMU" = "yes" ]; then
+	./run_qemu.sh
+	runtests=$((runtests+1))
+else
+	echo "HAVE_QEMU != yes -- Skipping test..." >&2
+	skipped=$((skipped+1))
+fi
+
 print_header "mode=unshare,variant=apt: create gzip compressed tarball"
 cat << END > shared/test.sh
 #!/bin/sh

mmdebstrap

@@ -23,13 +23,13 @@
 use strict;
 use warnings;
 
-our $VERSION = '0.8.0';
+our $VERSION = '0.8.1';
 
 use English;
 use Getopt::Long;
 use Pod::Usage;
 use File::Copy;
-use File::Path qw(make_path remove_tree);
+use File::Path qw(make_path);
 use File::Temp qw(tempfile tempdir);
 use File::Basename;
 use File::Find;
@@ -1550,41 +1550,6 @@ sub setup {
 sub run_setup() {
     my $options = shift;
 
-    my $dpkgversion = version->new(0);
-    {
-        my $pid = open my $fh, '-|' // error "failed to fork(): $!";
-        if ($pid == 0) {
-            # redirect stderr to /dev/null to hide error messages from dpkg
-            # versions before 1.20.0
-            open(STDERR, '>', '/dev/null')
-              or error "cannot open /dev/null for writing: $!";
-            exec 'dpkg', '--robot', '--version';
-        }
-        chomp(
-            my $content = do { local $/; <$fh> }
-        );
-        close $fh;
-        # the --robot option was introduced in 1.20.0 but until 1.20.2 the
-        # output contained a string after the version, separated by a
-        # whitespace -- since then, it's only the version
-        if ($? == 0 and $content =~ /^([0-9.]+)( .*)?$/) {
-            # dpkg is new enough for the --robot option
-            $dpkgversion = version->new($1);
-        }
-    }
-    my $aptversion = version->new(0);
-    {
-        my $pid = open my $fh, '-|', 'apt-get',
-          '--version' // error "failed to fork(): $!";
-        chomp(my $firstline = <$fh>);
-        close $fh;
-        if (    $? == 0
-            and $firstline =~ /^apt ([0-9]+\.[0-9]+\.[0-9]+) \([a-z0-9-]+\)$/)
-        {
-            $aptversion = version->new($1);
-        }
-    }
-
     {
         my @directories = (
             '/etc/apt/apt.conf.d',        '/etc/apt/sources.list.d',
@@ -1595,8 +1560,7 @@ sub run_setup() {
         push @directories, '/var/lib/dpkg';
         # since we do not know the dpkg version inside the chroot at this
         # point, we can only omit it in chrootless mode
-        if (   $options->{mode} ne 'chrootless'
+        if ($options->{mode} ne 'chrootless'
-            or $dpkgversion < "1.20.0"
             or scalar @{ $options->{dpkgopts} } > 0) {
             push @directories, '/etc/dpkg/dpkg.cfg.d/';
         }
@@ -1609,8 +1573,7 @@ sub run_setup() {
             push @directories, '/var/log/apt';
             # since we do not know the dpkg version inside the chroot at this
             # point, we can only omit it in chrootless mode
-            if (   $options->{mode} ne 'chrootless'
+            if ($options->{mode} ne 'chrootless') {
-                or $dpkgversion < "1.20.0") {
                 push @directories, '/var/lib/dpkg/triggers',
                   '/var/lib/dpkg/info', '/var/lib/dpkg/alternatives',
                   '/var/lib/dpkg/updates';
@@ -1688,24 +1651,6 @@ sub run_setup() {
         # options.
         print $conf "pkgCacheGen::ForceEssential \",\";\n";
     }
-    if ($options->{dryrun}
-        and ($aptversion < "2.1.16")) {
-        # Without this option, apt will fail with:
-        # E: Could not configure 'libc6:amd64'.
-        # E: Could not perform immediate configuration on 'libgcc1:amd64'.
-        #
-        # See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=953260
-        #
-        # For some while there were other immediate configuration problems
-        # which were fixed by changing the dependencies in src:glibc:
-        # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=972552
-        #
-        # Immediate configuration only was fixed in apt later which then also
-        # fixed these bugs:
-        # https://bugs.debian.org/973325
-        # https://bugs.debian.org/973305
-        print $conf "APT::Immediate-Configure false;\n";
-    }
     close $conf;
 
     # We put certain configuration items in their own configuration file
@@ -1728,31 +1673,6 @@ sub run_setup() {
         close $fh;
     }
 
-    # /var/lib/dpkg/available is required to exist or otherwise package
-    # removals will fail
-    # since we do not know the dpkg version inside the chroot at this point, we
-    # can only omit it in chrootless mode
-    # since we want to produce equivalent output independent on the mode, we
-    # only check for the dpkg version
-    if ($dpkgversion < "1.20.0") {
-        open my $fh, '>', "$options->{root}/var/lib/dpkg/available"
-          or error "failed to open(): $!";
-        close $fh;
-    }
-
-    # /var/lib/dpkg/cmethopt is used by dselect
-    # see #930788
-    # since we do not know the dpkg version inside the chroot at this point, we
-    # can only omit it in chrootless mode
-    # since we want to produce equivalent output independent on the mode, we
-    # only check for the dpkg version
-    if ($dpkgversion < "1.20.0") {
-        open my $fh, '>', "$options->{root}/var/lib/dpkg/cmethopt"
-          or error "failed to open(): $!";
-        print $fh "apt apt\n";
-        close $fh;
-    }
-
     # we create /var/lib/dpkg/arch inside the chroot either if there is more
     # than the native architecture in the chroot or if chrootless mode is
     # used to create a chroot of a different architecture than the native
@@ -2041,27 +1961,10 @@ sub run_setup() {
 sub run_update() {
     my $options = shift;
 
-    my $aptversion = version->new(0);
-    {
-        my $pid = open my $fh, '-|', 'apt-get',
-          '--version' // error "failed to fork(): $!";
-        chomp(my $firstline = <$fh>);
-        close $fh;
-        if (    $? == 0
-            and $firstline =~ /^apt ([0-9]+\.[0-9]+\.[0-9]+) \([a-z0-9-]+\)$/)
-        {
-            $aptversion = version->new($1);
-        }
-    }
     my $aptopts = {
-        ARGV  => ['apt-get', 'update'],
+        ARGV  => ['apt-get', 'update', '--error-on=any'],
         CHDIR => $options->{root},
     };
-    if ($aptversion < "2.1.16") {
-        $aptopts->{FIND_APT_WARNINGS} = 1;
-    } else {
-        push @{ $aptopts->{ARGV} }, '--error-on=any';
-    }
 
     info "running apt-get update...";
     run_apt_progress($aptopts);
@@ -3104,15 +3007,14 @@ sub run_cleanup() {
         # apt since 1.6 creates the auxfiles directory. If apt inside the
         # chroot is older than that, then it will not know how to clean it.
         if (-e "$options->{root}/var/lib/apt/lists/auxfiles") {
-            remove_tree("$options->{root}/var/lib/apt/lists/auxfiles",
+            0 == system(
-                { error => \my $err });
+                'rm',
-            if (@$err) {
+                '--interactive=never',
-                for my $diag (@$err) {
+                '--recursive',
-                    my ($file, $message) = %$diag;
+                '--preserve-root',
-                    if ($file eq '') { warning "general error: $message"; }
+                '--one-file-system',
-                    else { warning "problem unlinking $file: $message"; }
+                "$options->{root}/var/lib/apt/lists/auxfiles"
-                }
+            ) or error "rm failed: $?";
-            }
         }
     }
 
@@ -3182,15 +3084,11 @@ sub run_cleanup() {
                 next if $entry eq ".";
                 next if $entry eq "..";
                 warning "deleting files in /tmp: $entry";
-                remove_tree("$options->{root}/tmp/$entry",
+                0 == system(
-                    { error => \my $err });
+                    'rm',                '--interactive=never',
-                if (@$err) {
+                    '--recursive',       '--preserve-root',
-                    for my $diag (@$err) {
+                    '--one-file-system', "$options->{root}/tmp/$entry"
-                        my ($file, $message) = %$diag;
+                ) or error "rm failed: $?";
-                        if ($file eq '') { warning "general error: $message"; }
-                        else { warning "problem unlinking $file: $message"; }
-                    }
-                }
             }
             closedir($dh);
         }
@@ -4566,8 +4464,11 @@ sub main() {
         $ENV{PATH} = "/usr/sbin:/usr/bin:/sbin:/bin";
     }
 
-    foreach my $tool ('dpkg', 'dpkg-deb', 'apt-get', 'apt-cache', 'apt-config',
+    foreach my $tool (
-        'tar') {
+        'dpkg',       'dpkg-deb', 'apt-get', 'apt-cache',
+        'apt-config', 'tar',      'rm',      'find',
+        'env'
+    ) {
         my $found = 0;
         foreach my $path (split /:/, $ENV{PATH}) {
             if (-f "$path/$tool" && -x _ ) {
@@ -4580,6 +4481,49 @@ sub main() {
         }
     }
 
+    {
+        my $dpkgversion = version->new(0);
+        my $pid         = open my $fh, '-|' // error "failed to fork(): $!";
+        if ($pid == 0) {
+            # redirect stderr to /dev/null to hide error messages from dpkg
+            # versions before 1.20.0
+            open(STDERR, '>', '/dev/null')
+              or error "cannot open /dev/null for writing: $!";
+            exec 'dpkg', '--robot', '--version';
+        }
+        chomp(
+            my $content = do { local $/; <$fh> }
+        );
+        close $fh;
+        # the --robot option was introduced in 1.20.0 but until 1.20.2 the
+        # output contained a string after the version, separated by a
+        # whitespace -- since then, it's only the version
+        if ($? == 0 and $content =~ /^([0-9.]+)( .*)?$/) {
+            # dpkg is new enough for the --robot option
+            $dpkgversion = version->new($1);
+        }
+        if ($dpkgversion < "1.20.0") {
+            error "need dpkg >= 1.20.0 but have $dpkgversion";
+        }
+    }
+
+    {
+        my $aptversion = version->new(0);
+        my $pid        = open my $fh, '-|', 'apt-get',
+          '--version' // error "failed to fork(): $!";
+        chomp(
+            my $content = do { local $/; <$fh> }
+        );
+        close $fh;
+        if (    $? == 0
+            and $content =~ /^apt ([0-9]+\.[0-9]+\.[0-9]+) \([a-z0-9-]+\)$/m) {
+            $aptversion = version->new($1);
+        }
+        if ($aptversion < "2.3.7") {
+            error "need apt >= 2.3.7 but have $aptversion";
+        }
+    }
+
     my $check_fakechroot_running = sub {
         # test if we are inside fakechroot already
         # We fork a child process because setting FAKECHROOT_DETECT seems to
@@ -5952,31 +5896,18 @@ sub main() {
             # no risk of removing anything important.
             $pid = get_unshare_cmd(
                 sub {
-                    # File::Path will produce the error "cannot stat initial
+                    # change CWD to chroot directory because find tries to
-                    # working directory" if the working directory cannot be
+                    # chdir to the current directory which might not be
-                    # accessed by the unprivileged unshared user. Thus, we
+                    # accessible by the unshared user:
-                    # first navigate to the parent of the root directory.
+                    # find: Failed to restore initial working directory
-                    chdir "$options->{root}/.."
+                    0 == system('env', "--chdir=$options->{root}", 'find',
-                      or error "unable to chdir() to parent directory of"
+                        $options->{root}, '-mount',
-                      . " $options->{root}: $!";
+                        '-mindepth', '1', '-delete')
-                    remove_tree($options->{root}, { error => \my $err });
+                      or error "rm failed: $?";
-                    if (@$err) {
+                    # ignore failure in case the unshared user doesn't have the
-                        for my $diag (@$err) {
+                    # required permissions -- we attempt again later if
-                            my ($file, $message) = %$diag;
+                    # necessary
-                            if ($file eq '') {
+                    rmdir "$options->{root}";
-                                warning "general error: $message";
-                            } else {
-                                # the unshared process might not have
-                                # sufficient permissions to remove the root
-                                # directory. We attempt its removal later, so
-                                # don't warn if its removal fails now.
-                                if ($file ne $options->{root}) {
-                                    warning
-                                      "problem unlinking $file: $message";
-                                }
-                            }
-                        }
-                    }
                 },
                 \@idmap
             );
@@ -5984,8 +5915,7 @@ sub main() {
             $? == 0 or error "remove_tree failed";
             # in unshare mode, the toplevel directory might've been created in
             # a directory that the unshared user cannot change and thus cannot
-            # delete. If the root directory still exists after remove_tree
+            # delete. We attempt its removal again outside as the normal user.
-            # above, we attempt its removal again outside as the normal user.
             if (-e $options->{root}) {
                 rmdir "$options->{root}"
                   or error "cannot rmdir $options->{root}: $!";
@@ -6003,7 +5933,7 @@ sub main() {
             # be removed.
             0 == system('rm', '--interactive=never', '--recursive',
                 '--preserve-root', '--one-file-system', $options->{root})
-              or error "rm failed: $!";
+              or error "rm failed: $?";
         } else {
             error "unknown mode: $options->{mode}";
         }