CHANGELOG.md

@@ -1,12 +1,3 @@
-0.8.1 (2021-10-07)
-------------------
-
- - enforce dpkg >= 1.20.0 and apt >= 2.3.7
- - allow working directory be not world readable
- - do not run xz and zstd with --threads=0 since this is a bad default for
-   machines with more than 100 cores
- - bit-by-bit identical chrootless mode
-
 0.8.0 (2021-09-21)
 ------------------
 

coverage.sh

@@ -127,7 +127,7 @@ if [ ! -e shared/hooks/eatmydata/customize.sh ] || [ hooks/eatmydata/customize.s
 	fi
 fi
 starttime=
-total=213
+total=212
 skipped=0
 runtests=0
 i=1
@@ -1055,39 +1055,6 @@ else
 	runtests=$((runtests+1))
 fi
 
-print_header "mode=unshare,variant=apt: CWD directory not accessible by unshared user"
-cat << END > shared/test.sh
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
-	echo "this test modifies the system and should only be run inside a container" >&2
-	exit 1
-fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
-mkdir /tmp/debian-chroot
-chmod 700 /tmp/debian-chroot
-chown user:user /tmp/debian-chroot
-if [ "$CMD" = "./mmdebstrap" ]; then
-	CMD=\$(realpath --canonicalize-existing ./mmdebstrap)
-elif [ "$CMD" = "perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap" ]; then
-	CMD="perl -MDevel::Cover=-silent,-nogcov \$(realpath --canonicalize-existing ./mmdebstrap)"
-else
-	CMD="$CMD"
-fi
-env --chdir=/tmp/debian-chroot runuser -u user -- \$CMD --mode=unshare --variant=apt $DEFAULT_DIST /tmp/debian-chroot.tar $mirror
-tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
-rm /tmp/debian-chroot.tar
-END
-if [ "$HAVE_QEMU" = "yes" ]; then
-	./run_qemu.sh
-	runtests=$((runtests+1))
-else
-	echo "HAVE_QEMU != yes -- Skipping test..." >&2
-	skipped=$((skipped+1))
-fi
-
 print_header "mode=unshare,variant=apt: create gzip compressed tarball"
 cat << END > shared/test.sh
 #!/bin/sh

mmdebstrap

@@ -23,13 +23,13 @@
 use strict;
 use warnings;
 
-our $VERSION = '0.8.1';
+our $VERSION = '0.8.0';
 
 use English;
 use Getopt::Long;
 use Pod::Usage;
 use File::Copy;
-use File::Path qw(make_path);
+use File::Path qw(make_path remove_tree);
 use File::Temp qw(tempfile tempdir);
 use File::Basename;
 use File::Find;
@@ -1550,6 +1550,41 @@ sub setup {
 sub run_setup() {
     my $options = shift;
 
+    my $dpkgversion = version->new(0);
+    {
+        my $pid = open my $fh, '-|' // error "failed to fork(): $!";
+        if ($pid == 0) {
+            # redirect stderr to /dev/null to hide error messages from dpkg
+            # versions before 1.20.0
+            open(STDERR, '>', '/dev/null')
+              or error "cannot open /dev/null for writing: $!";
+            exec 'dpkg', '--robot', '--version';
+        }
+        chomp(
+            my $content = do { local $/; <$fh> }
+        );
+        close $fh;
+        # the --robot option was introduced in 1.20.0 but until 1.20.2 the
+        # output contained a string after the version, separated by a
+        # whitespace -- since then, it's only the version
+        if ($? == 0 and $content =~ /^([0-9.]+)( .*)?$/) {
+            # dpkg is new enough for the --robot option
+            $dpkgversion = version->new($1);
+        }
+    }
+    my $aptversion = version->new(0);
+    {
+        my $pid = open my $fh, '-|', 'apt-get',
+          '--version' // error "failed to fork(): $!";
+        chomp(my $firstline = <$fh>);
+        close $fh;
+        if (    $? == 0
+            and $firstline =~ /^apt ([0-9]+\.[0-9]+\.[0-9]+) \([a-z0-9-]+\)$/)
+        {
+            $aptversion = version->new($1);
+        }
+    }
+
     {
         my @directories = (
             '/etc/apt/apt.conf.d',        '/etc/apt/sources.list.d',
@@ -1560,7 +1595,8 @@ sub run_setup() {
         push @directories, '/var/lib/dpkg';
         # since we do not know the dpkg version inside the chroot at this
         # point, we can only omit it in chrootless mode
-        if ($options->{mode} ne 'chrootless'
+        if (   $options->{mode} ne 'chrootless'
+            or $dpkgversion < "1.20.0"
             or scalar @{ $options->{dpkgopts} } > 0) {
             push @directories, '/etc/dpkg/dpkg.cfg.d/';
         }
@@ -1573,7 +1609,8 @@ sub run_setup() {
             push @directories, '/var/log/apt';
             # since we do not know the dpkg version inside the chroot at this
             # point, we can only omit it in chrootless mode
-            if ($options->{mode} ne 'chrootless') {
+            if (   $options->{mode} ne 'chrootless'
+                or $dpkgversion < "1.20.0") {
                 push @directories, '/var/lib/dpkg/triggers',
                   '/var/lib/dpkg/info', '/var/lib/dpkg/alternatives',
                   '/var/lib/dpkg/updates';
@@ -1651,6 +1688,24 @@ sub run_setup() {
         # options.
         print $conf "pkgCacheGen::ForceEssential \",\";\n";
     }
+    if ($options->{dryrun}
+        and ($aptversion < "2.1.16")) {
+        # Without this option, apt will fail with:
+        # E: Could not configure 'libc6:amd64'.
+        # E: Could not perform immediate configuration on 'libgcc1:amd64'.
+        #
+        # See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=953260
+        #
+        # For some while there were other immediate configuration problems
+        # which were fixed by changing the dependencies in src:glibc:
+        # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=972552
+        #
+        # Immediate configuration only was fixed in apt later which then also
+        # fixed these bugs:
+        # https://bugs.debian.org/973325
+        # https://bugs.debian.org/973305
+        print $conf "APT::Immediate-Configure false;\n";
+    }
     close $conf;
 
     # We put certain configuration items in their own configuration file
@@ -1673,6 +1728,31 @@ sub run_setup() {
         close $fh;
     }
 
+    # /var/lib/dpkg/available is required to exist or otherwise package
+    # removals will fail
+    # since we do not know the dpkg version inside the chroot at this point, we
+    # can only omit it in chrootless mode
+    # since we want to produce equivalent output independent on the mode, we
+    # only check for the dpkg version
+    if ($dpkgversion < "1.20.0") {
+        open my $fh, '>', "$options->{root}/var/lib/dpkg/available"
+          or error "failed to open(): $!";
+        close $fh;
+    }
+
+    # /var/lib/dpkg/cmethopt is used by dselect
+    # see #930788
+    # since we do not know the dpkg version inside the chroot at this point, we
+    # can only omit it in chrootless mode
+    # since we want to produce equivalent output independent on the mode, we
+    # only check for the dpkg version
+    if ($dpkgversion < "1.20.0") {
+        open my $fh, '>', "$options->{root}/var/lib/dpkg/cmethopt"
+          or error "failed to open(): $!";
+        print $fh "apt apt\n";
+        close $fh;
+    }
+
     # we create /var/lib/dpkg/arch inside the chroot either if there is more
     # than the native architecture in the chroot or if chrootless mode is
     # used to create a chroot of a different architecture than the native
@@ -1961,10 +2041,27 @@ sub run_setup() {
 sub run_update() {
     my $options = shift;
 
+    my $aptversion = version->new(0);
+    {
+        my $pid = open my $fh, '-|', 'apt-get',
+          '--version' // error "failed to fork(): $!";
+        chomp(my $firstline = <$fh>);
+        close $fh;
+        if (    $? == 0
+            and $firstline =~ /^apt ([0-9]+\.[0-9]+\.[0-9]+) \([a-z0-9-]+\)$/)
+        {
+            $aptversion = version->new($1);
+        }
+    }
     my $aptopts = {
-        ARGV  => ['apt-get', 'update', '--error-on=any'],
+        ARGV  => ['apt-get', 'update'],
         CHDIR => $options->{root},
     };
+    if ($aptversion < "2.1.16") {
+        $aptopts->{FIND_APT_WARNINGS} = 1;
+    } else {
+        push @{ $aptopts->{ARGV} }, '--error-on=any';
+    }
 
     info "running apt-get update...";
     run_apt_progress($aptopts);
@@ -3007,14 +3104,15 @@ sub run_cleanup() {
         # apt since 1.6 creates the auxfiles directory. If apt inside the
         # chroot is older than that, then it will not know how to clean it.
         if (-e "$options->{root}/var/lib/apt/lists/auxfiles") {
-            0 == system(
-                'rm',
-                '--interactive=never',
-                '--recursive',
-                '--preserve-root',
-                '--one-file-system',
-                "$options->{root}/var/lib/apt/lists/auxfiles"
-            ) or error "rm failed: $?";
+            remove_tree("$options->{root}/var/lib/apt/lists/auxfiles",
+                { error => \my $err });
+            if (@$err) {
+                for my $diag (@$err) {
+                    my ($file, $message) = %$diag;
+                    if ($file eq '') { warning "general error: $message"; }
+                    else { warning "problem unlinking $file: $message"; }
+                }
+            }
         }
     }
 
@@ -3084,11 +3182,15 @@ sub run_cleanup() {
                 next if $entry eq ".";
                 next if $entry eq "..";
                 warning "deleting files in /tmp: $entry";
-                0 == system(
-                    'rm',                '--interactive=never',
-                    '--recursive',       '--preserve-root',
-                    '--one-file-system', "$options->{root}/tmp/$entry"
-                ) or error "rm failed: $?";
+                remove_tree("$options->{root}/tmp/$entry",
+                    { error => \my $err });
+                if (@$err) {
+                    for my $diag (@$err) {
+                        my ($file, $message) = %$diag;
+                        if ($file eq '') { warning "general error: $message"; }
+                        else { warning "problem unlinking $file: $message"; }
+                    }
+                }
             }
             closedir($dh);
         }
@@ -4464,11 +4566,8 @@ sub main() {
         $ENV{PATH} = "/usr/sbin:/usr/bin:/sbin:/bin";
     }
 
-    foreach my $tool (
-        'dpkg',       'dpkg-deb', 'apt-get', 'apt-cache',
-        'apt-config', 'tar',      'rm',      'find',
-        'env'
-    ) {
+    foreach my $tool ('dpkg', 'dpkg-deb', 'apt-get', 'apt-cache', 'apt-config',
+        'tar') {
         my $found = 0;
         foreach my $path (split /:/, $ENV{PATH}) {
             if (-f "$path/$tool" && -x _ ) {
@@ -4481,49 +4580,6 @@ sub main() {
         }
     }
 
-    {
-        my $dpkgversion = version->new(0);
-        my $pid         = open my $fh, '-|' // error "failed to fork(): $!";
-        if ($pid == 0) {
-            # redirect stderr to /dev/null to hide error messages from dpkg
-            # versions before 1.20.0
-            open(STDERR, '>', '/dev/null')
-              or error "cannot open /dev/null for writing: $!";
-            exec 'dpkg', '--robot', '--version';
-        }
-        chomp(
-            my $content = do { local $/; <$fh> }
-        );
-        close $fh;
-        # the --robot option was introduced in 1.20.0 but until 1.20.2 the
-        # output contained a string after the version, separated by a
-        # whitespace -- since then, it's only the version
-        if ($? == 0 and $content =~ /^([0-9.]+)( .*)?$/) {
-            # dpkg is new enough for the --robot option
-            $dpkgversion = version->new($1);
-        }
-        if ($dpkgversion < "1.20.0") {
-            error "need dpkg >= 1.20.0 but have $dpkgversion";
-        }
-    }
-
-    {
-        my $aptversion = version->new(0);
-        my $pid        = open my $fh, '-|', 'apt-get',
-          '--version' // error "failed to fork(): $!";
-        chomp(
-            my $content = do { local $/; <$fh> }
-        );
-        close $fh;
-        if (    $? == 0
-            and $content =~ /^apt ([0-9]+\.[0-9]+\.[0-9]+) \([a-z0-9-]+\)$/m) {
-            $aptversion = version->new($1);
-        }
-        if ($aptversion < "2.3.7") {
-            error "need apt >= 2.3.7 but have $aptversion";
-        }
-    }
-
     my $check_fakechroot_running = sub {
         # test if we are inside fakechroot already
         # We fork a child process because setting FAKECHROOT_DETECT seems to
@@ -5896,18 +5952,31 @@ sub main() {
             # no risk of removing anything important.
             $pid = get_unshare_cmd(
                 sub {
-                    # change CWD to chroot directory because find tries to
-                    # chdir to the current directory which might not be
-                    # accessible by the unshared user:
-                    # find: Failed to restore initial working directory
-                    0 == system('env', "--chdir=$options->{root}", 'find',
-                        $options->{root}, '-mount',
-                        '-mindepth', '1', '-delete')
-                      or error "rm failed: $?";
-                    # ignore failure in case the unshared user doesn't have the
-                    # required permissions -- we attempt again later if
-                    # necessary
-                    rmdir "$options->{root}";
+                    # File::Path will produce the error "cannot stat initial
+                    # working directory" if the working directory cannot be
+                    # accessed by the unprivileged unshared user. Thus, we
+                    # first navigate to the parent of the root directory.
+                    chdir "$options->{root}/.."
+                      or error "unable to chdir() to parent directory of"
+                      . " $options->{root}: $!";
+                    remove_tree($options->{root}, { error => \my $err });
+                    if (@$err) {
+                        for my $diag (@$err) {
+                            my ($file, $message) = %$diag;
+                            if ($file eq '') {
+                                warning "general error: $message";
+                            } else {
+                                # the unshared process might not have
+                                # sufficient permissions to remove the root
+                                # directory. We attempt its removal later, so
+                                # don't warn if its removal fails now.
+                                if ($file ne $options->{root}) {
+                                    warning
+                                      "problem unlinking $file: $message";
+                                }
+                            }
+                        }
+                    }
                 },
                 \@idmap
             );
@@ -5915,7 +5984,8 @@ sub main() {
             $? == 0 or error "remove_tree failed";
             # in unshare mode, the toplevel directory might've been created in
             # a directory that the unshared user cannot change and thus cannot
-            # delete. We attempt its removal again outside as the normal user.
+            # delete. If the root directory still exists after remove_tree
+            # above, we attempt its removal again outside as the normal user.
             if (-e $options->{root}) {
                 rmdir "$options->{root}"
                   or error "cannot rmdir $options->{root}: $!";
@@ -5933,7 +6003,7 @@ sub main() {
             # be removed.
             0 == system('rm', '--interactive=never', '--recursive',
                 '--preserve-root', '--one-file-system', $options->{root})
-              or error "rm failed: $?";
+              or error "rm failed: $!";
         } else {
             error "unknown mode: $options->{mode}";
         }