add jessie-or-older extract hook

coverage.py

@@ -207,13 +207,27 @@ def main():
     # parse coverage.txt
     config_order, config_dict = parse_config("coverage.txt")
 
-    if set(os.listdir("tests")) - set(config_order):
+    indirbutnotcovered = set(
+        [d for d in os.listdir("tests") if not d.startswith(".")]
+    ) - set(config_order)
+    if indirbutnotcovered:
         print(
             "test(s) missing from coverage.txt: %s"
-            % (", ".join(sorted(set(os.listdir("tests")) - set(config_order)))),
+            % (", ".join(sorted(indirbutnotcovered))),
             file=sys.stderr,
         )
         exit(1)
+    coveredbutnotindir = set(config_order) - set(
+        [d for d in os.listdir("tests") if not d.startswith(".")]
+    )
+    if coveredbutnotindir:
+        print(
+            "test(s) missing from ./tests: %s"
+            % (", ".join(sorted(coveredbutnotindir))),
+            file=sys.stderr,
+        )
+
+        exit(1)
 
     # produce the list of tests using the cartesian product of all allowed
     # dists, modes, variants and formats of a given test

coverage.sh

@@ -74,7 +74,6 @@ SOURCE_DATE_EPOCH=$(date --date="$(grep-dctrl -s Date -n '' "$mirrordir/dists/$D
 export LC_ALL=C.UTF-8
 
 : "${HAVE_UNSHARE:=yes}"
-: "${HAVE_PROOT:=yes}"
 : "${HAVE_BINFMT:=yes}"
 
 # by default, use the mmdebstrap executable in the current directory together

coverage.txt

@@ -347,6 +347,14 @@ Test: variant-custom-timeout
 
 Test: include-deb-file
 
+Test: unshare-include-deb
+Modes: unshare
+Needs-QEMU: true
+
 Test: pivot_root
 Modes: root unshare
 Needs-QEMU: true
+
+Test: jessie-or-older
+Needs-QEMU: true
+Variants: essential apt minbase

hooks/jessie-or-older/extract00.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
+
+TARGET="$1"
+
+for f in available diversions cmethopt; do
+	if [ ! -e "$TARGET/var/lib/dpkg/$f" ]; then
+		touch "$TARGET/var/lib/dpkg/$f"
+	fi
+done
+
+if [ -z "${MMDEBSTRAP_ESSENTIAL+x}" ]; then
+	MMDEBSTRAP_ESSENTIAL=
+	for f in "$TARGET/var/cache/apt/archives/"*.deb; do
+		[ -f "$f" ] || continue
+		f="${f#"$TARGET"}"
+		MMDEBSTRAP_ESSENTIAL="$MMDEBSTRAP_ESSENTIAL $f"
+	done
+fi
+
+fname_base_passwd=
+fname_base_files=
+fname_dpkg=
+for pkg in $MMDEBSTRAP_ESSENTIAL; do
+	pkgname=$(dpkg-deb --show --showformat='${Package}' "$TARGET/$pkg")
+	# shellcheck disable=SC2034
+	case $pkgname in
+		base-passwd) fname_base_passwd=$pkg;;
+		base-files)  fname_base_files=$pkg;;
+		dpkg)        fname_dpkg=$pkg;;
+	esac
+done
+
+for var in base_passwd base_files dpkg; do
+	eval 'val=$fname_'"$var"
+	[ -z "$val" ] && continue
+	chroot "$TARGET" dpkg --install --force-depends "$val"
+done
+
+# shellcheck disable=SC2086
+chroot "$TARGET" dpkg --unpack --force-depends $MMDEBSTRAP_ESSENTIAL
+
+chroot "$TARGET" dpkg --configure --pending

make_mirror.sh

@@ -391,7 +391,6 @@ components=main
 : "${DEFAULT_DIST:=unstable}"
 : "${HAVE_QEMU:=yes}"
 : "${RUN_MA_SAME_TESTS:=yes}"
-: "${HAVE_PROOT:=yes}"
 # by default, use the mmdebstrap executable in the current directory
 : "${CMD:=./mmdebstrap}"
 
@@ -505,9 +504,6 @@ if [ "$HAVE_QEMU" = "yes" ]; then
 	if [ "$DEFAULT_DIST" != "oldstable" ]; then
 		pkgs="$pkgs,squashfs-tools-ng,genext2fs"
 	fi
-	if [ "$HAVE_PROOT" = "yes" ]; then
-		pkgs="$pkgs,proot"
-	fi
 	if [ ! -e ./mmdebstrap ]; then
 		pkgs="$pkgs,mmdebstrap"
 	fi

mmdebstrap

@@ -950,7 +950,11 @@ sub run_dpkg_progress {
             }
             $num += 1;
         }
+        if ($total == 0) {
+            return 0, $status;
+        } else {
             return $num / $total * 100, $status;
+        }
     };
     run_progress $get_exec, $line_handler, $line_has_error;
     return;
@@ -1539,6 +1543,7 @@ sub setup_mounts {
 sub run_hooks {
     my $name           = shift;
     my $options        = shift;
+    my $essential_pkgs = shift;
 
     if (scalar @{ $options->{"${name}_hook"} } == 0) {
         return;
@@ -1597,6 +1602,12 @@ sub run_hooks {
         push @env_opts,
           ("MMDEBSTRAP_INCLUDE=" . (join ",", @escaped_includes));
     }
+    # Give the extract hook access to the essential packages that are about to
+    # be installed
+    if ($name eq "extract" and scalar @{$essential_pkgs} > 0) {
+        push @env_opts,
+          ("MMDEBSTRAP_ESSENTIAL=" . (join " ", @{$essential_pkgs}));
+    }
 
     # Unset the close-on-exec flag, so that the file descriptor does not
     # get closed when we exec
@@ -1814,14 +1825,16 @@ sub setup {
     }
 
     eval {
-        run_hooks('extract', $options);
-
-        if ($options->{variant} ne 'extract') {
         my $chrootcmd = [];
+        if ($options->{variant} ne 'extract') {
             if ($options->{mode} ne 'chrootless') {
                 $chrootcmd = run_prepare($options);
             }
+        }
 
+        run_hooks('extract', $options, $essential_pkgs);
+
+        if ($options->{variant} ne 'extract') {
             run_essential($options, $essential_pkgs, $chrootcmd, $cached_debs);
 
             run_hooks('essential', $options);
@@ -4359,9 +4372,15 @@ sub main() {
                 if (!defined $pkg) {
                     error "cannot resolve absolute path of $pkg: $!";
                 }
+                if ($pkg !~ /^\//) {
+                    error "absolute path of $pkg doesn't start with a slash";
+                }
                 if (!-f $pkg) {
                     error "$pkg is not an existing file";
                 }
+                if (!-r $pkg) {
+                    error "$pkg is not readable";
+                }
                 return $pkg;
             };
             if ($opt_value =~ /^[?~!(]/) {
@@ -5566,6 +5585,36 @@ sub main() {
         $? == 0 or error "chown failed";
     }
 
+    # check if .deb files given by --include are readable by the unshared user
+    if ($options->{mode} eq 'unshare'
+        and scalar(grep { /^\// } @{ $options->{include} }) > 0) {
+        my $pid = get_unshare_cmd(
+            sub {
+                my $ret = 0;
+                foreach my $f (grep { /^\// } @{ $options->{include} }) {
+                    # open the file for real because -r will report the file as
+                    # readable even though open will fail (in contrast to the
+                    # coreutils test utility, perl doesn't use faccessat)
+                    my $res = open(my $fh, '<', $f);
+                    if (!$res) {
+                        warning "unshared user cannot access $f for reading";
+                        $ret = 1;
+                    } else {
+                        close $fh;
+                    }
+                }
+                exit $ret;
+            },
+            \@idmap
+        );
+        waitpid $pid, 0;
+        if ($? != 0) {
+            warning "no read access for some packages for the unshared user";
+            warning "maybe try running mmdebstrap with "
+              . "--hook-dir=/usr/share/mmdebstrap/hooks/file-mirror-automount";
+        }
+    }
+
     # figure out whether we have mknod
     $options->{havemknod} = 0;
     if ($options->{mode} eq 'unshare') {
@@ -5880,7 +5929,13 @@ sub main() {
     my $numblocks = 0;
     close $nblkwriter;
     if (!$options->{dryrun} && $format eq 'ext2') {
-        chomp($numblocks = <$nblkreader>);
+        $numblocks = <$nblkreader>;
+        if (!defined $numblocks) {
+            # This can happen if the setup process died early and thus closes
+            # the pipe from the other and. The EOF is turned into undef.
+            error "failed to read required number of blocks";
+        }
+        chomp $numblocks;
     }
     close $nblkreader;
 
@@ -6806,7 +6861,7 @@ retained.
 This section gives an overview of the different steps to create a chroot. At
 its core, what B<mmdebstrap> does can be put into a 14 line shell script:
 
-    mkdir -p "$2/etc/apt" "$2/var/cache"
+    mkdir -p "$2/etc/apt" "$2/var/cache" "$2/var/lib"
     cat << END > "$2/apt.conf"
     Apt::Architecture "$(dpkg --print-architecture)";
     Apt::Architectures "$(dpkg --print-architecture)";
@@ -6902,10 +6957,6 @@ respectively.
 
 Extract the downloaded packages into the rootfs.
 
-=item B<extract-hook>
-
-Run B<--extract-hook> options and all F<extract*> scripts in B<--hook-dir>.
-
 =item B<prepare>
 
 In B<fakechroot> mode, environment variables C<LD_LIBRARY_PATH> will be set up
@@ -6914,6 +6965,10 @@ in. For foreign B<fakechroot> environments, C<LD_LIBRARY_PATH> and
 C<QEMU_LD_PREFIX> are set up accordingly. This step is not carried out in
 B<extract> mode and neither for the B<chrootless> variant.
 
+=item B<extract-hook>
+
+Run B<--extract-hook> options and all F<extract*> scripts in B<--hook-dir>.
+
 =item B<essential>
 
 Uses C<dpkg --install> to properly install all packages that have been

tests/file-mirror

@@ -5,6 +5,9 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test requires the cache directory to be mounted on /mnt and should only be run inside a container" >&2
 	exit 1
 fi
-{{ CMD }} --mode={{ MODE }} --variant=apt --setup-hook='mkdir -p "$1"/mnt/cache/debian; mount -o ro,bind /mnt/cache/debian "$1"/mnt/cache/debian' --customize-hook='umount "$1"/mnt/cache/debian; rmdir "$1"/mnt/cache/debian "$1"/mnt/cache' {{ DIST }} /tmp/debian-chroot.tar "deb file:///mnt/cache/debian {{ DIST }} main"
+{{ CMD }} --mode={{ MODE }} --variant=apt \
+	--setup-hook='mkdir -p "$1"/mnt/cache/debian; mount -o ro,bind /mnt/cache/debian "$1"/mnt/cache/debian' \
+	--customize-hook='umount "$1"/mnt/cache/debian; rmdir "$1"/mnt/cache/debian "$1"/mnt/cache' \
+	{{ DIST }} /tmp/debian-chroot.tar "deb file:///mnt/cache/debian {{ DIST }} main"
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar

tests/i386-which-can-be-executed-without-qemu

@@ -27,7 +27,8 @@ apt-get remove --yes qemu-user-static binfmt-support qemu-user
 	| grep -v '^\./usr/bin/x86_64$' \
 	| grep -v '^\./usr/lib32/$' \
 	| grep -v '^\./lib32$' \
-	| grep -v '^\./lib64/$' \
+	| grep -v '^\./lib64$' \
+	| grep -v '^\./usr/lib64/$' \
 	| grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
 	| grep -v '^\./usr/lib/gcc/x86_64-linux-gnu/$' \
 	| grep -v '^\./usr/lib/gcc/x86_64-linux-gnu/[0-9]\+/$' \

tests/jessie-or-older

@@ -0,0 +1,39 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+adduser --gecos user --disabled-password user
+sysctl -w kernel.unprivileged_userns_clone=1
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+
+filter() {
+	./tarfilter \
+		--path-exclude=/usr/bin/uncompress \
+		--path-exclude=/var/cache/debconf/config.dat-old \
+		--path-exclude=/var/cache/debconf/templates.dat-old \
+		--path-exclude=/var/lib/dpkg/available \
+		--path-exclude=/var/lib/dpkg/cmethopt \
+		--path-exclude=/var/lib/dpkg/status-old \
+		--path-exclude=/var/lib/shells.state
+}
+
+# base for comparison without jessie-or-older hook
+{{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-root-normal.tar
+
+# root
+{{ CMD }} --mode=root --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-root.tar
+cmp /tmp/debian-chroot-root-normal.tar /tmp/debian-chroot-root.tar
+rm /tmp/debian-chroot-root.tar
+# unshare
+runuser -u user -- {{ CMD }} --mode=unshare --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-unshare.tar
+cmp /tmp/debian-chroot-root-normal.tar /tmp/debian-chroot-unshare.tar
+rm /tmp/debian-chroot-unshare.tar
+# fakechroot
+runuser -u user -- {{ CMD }} --mode=fakechroot --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-fakechroot.tar
+cmp /tmp/debian-chroot-root-normal.tar /tmp/debian-chroot-fakechroot.tar
+rm /tmp/debian-chroot-fakechroot.tar
+
+rm /tmp/debian-chroot-root-normal.tar

tests/unshare-include-deb

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+set -eu
+export LC_ALL=C.UTF-8
+
+[ "{{ MODE }}" = unshare ]
+
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	if [ ! -e /mmdebstrap-testenv ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
+		exit 1
+	fi
+	adduser --gecos user --disabled-password user
+fi
+[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+
+# instead of obtaining a .deb from our cache, we create a new package because
+# otherwise apt might decide to download the package with the same name and
+# version from the cache instead of using the local .deb
+mkdir -p /tmp/dummypkg/DEBIAN
+cat << END > "/tmp/dummypkg/DEBIAN/control"
+Package: dummypkg
+Priority: optional
+Section: oldlibs
+Maintainer: Johannes Schauer Marin Rodrigues <josch@debian.org>
+Architecture: all
+Multi-Arch: foreign
+Source: dummypkg
+Version: 1
+Description: dummypkg
+END
+dpkg-deb --build "/tmp/dummypkg" "/tmp/dummypkg.deb"
+
+# make the .deb only redable by user which will exclude the unshared user
+chmod 600 /tmp/dummypkg.deb
+chown user /tmp/dummypkg.deb
+
+ret=0
+$prefix {{ CMD }} --variant=apt --mode={{ MODE }} --include="/tmp/dummypkg.deb" \
+	{{ DIST }} /dev/null {{ MIRROR }} || ret=$?
+
+if [ "$ret" -eq 0 ]; then
+	echo "expected failure but got exit $ret" >&2
+	exit 1
+fi