release 1.2.2

CHANGELOG.md

@@ -1,3 +1,12 @@
+1.2.2 (2022-10-27)
+------------------
+
+ - allow /etc/apt/trusted.gpg.d/ not to exist
+ - always create /var/lib/dpkg/arch to make foreign architecture chrootless
+   tarballs bit-by-bit identical
+ - write an empty /etc/machine-id instead of writing 'uninitialized'
+ - only print progress bars on interactive terminals that are wide enough
+
 1.2.1 (2022-09-08)
 ------------------
 

coverage.py

@@ -121,13 +121,13 @@ def parse_config(confname):
 
 def format_failed(num, total, name, dist, mode, variant, fmt, config_dict):
     ret = f"({num}/{total}) {name}"
-    if len(config_dict[name].get("Dists", [])) > 0:
+    if len(config_dict[name].get("Dists", [])) > 1:
         ret += f" --dist={dist}"
-    if len(config_dict[name].get("Modes", [])) > 0:
+    if len(config_dict[name].get("Modes", [])) > 1:
         ret += f" --mode={mode}"
-    if len(config_dict[name].get("Variants", [])) > 0:
+    if len(config_dict[name].get("Variants", [])) > 1:
         ret += f" --variant={variant}"
-    if len(config_dict[name].get("Formats", [])) > 0:
+    if len(config_dict[name].get("Formats", [])) > 1:
         ret += f" --format={fmt}"
     return ret
 

coverage.txt

@@ -288,12 +288,27 @@ Test: install-doc-debian
 Modes: chrootless
 Variants: custom
 
-Test: chrootless-essential
-Variants: custom
+Test: chrootless
+Variants: essential
 Modes: chrootless
 Skip-If:
  dist in ["oldstable", "stable"]
 
+Test: chrootless-fakeroot
+Variants: essential
+Modes: chrootless
+Skip-If:
+ dist in ["oldstable", "stable"]
+
+Test: chrootless-foreign
+Variants: essential
+Modes: chrootless
+Skip-If:
+ dist in ["oldstable", "stable"]
+ hostarch != "amd64"
+ not run_ma_same_tests
+Needs-QEMU: true
+
 Test: install-doc-debian-and-output-tarball
 Variants: custom
 Modes: chrootless

make_mirror.sh

@@ -270,7 +270,7 @@ END
 	# starting from Debian 12 (Bullseye)
 	case "$dist" in
 		oldstable|stable) : ;;
-		*) pkgs="$pkgs usr-is-merged" ;;
+		*) pkgs="$pkgs usr-is-merged usrmerge" ;;
 	esac
 
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs
@@ -546,7 +546,7 @@ timeout 0
 
 label linux
 kernel /vmlinuz
-append initrd=/initrd.img root=/dev/vda1 rw console=ttyS0,115200
+append initrd=/initrd.img root=/dev/vda1 rw console=tty0 console=ttyS0,115200n8
 serial 0 115200
 END
 	cat << END > "$tmpdir/mmdebstrap.service"

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '1.2.1';
+our $VERSION = '1.2.2';
 
 use English;
 use Getopt::Long;
@@ -34,7 +34,8 @@ use File::Temp qw(tempfile tempdir);
 use File::Basename;
 use File::Find;
 use Cwd qw(abs_path getcwd);
-require "syscall.ph";    ## no critic (Modules::RequireBarewordIncludes)
+require "syscall.ph";      ## no critic (Modules::RequireBarewordIncludes)
+require "sys/ioctl.ph";    ## no critic (Modules::RequireBarewordIncludes)
 use Fcntl qw(S_IFCHR S_IFBLK FD_CLOEXEC F_GETFD F_SETFD);
 use List::Util qw(any none);
 use POSIX
@@ -694,30 +695,87 @@ sub havemknod {
     return $havemknod;
 }
 
+# inspired by /usr/share/perl/5.34/pod/perlfaq8.pod
+sub terminal_width {
+    if (!stderr_is_tty()) {
+        return -1;
+    }
+    if (!defined &TIOCGWINSZ) {
+        return -1;
+    }
+    if (!-e "/dev/tty") {
+        return -1;
+    }
+    my $tty_fh;
+    if (!open($tty_fh, "+<", "/dev/tty")) {
+        return -1;
+    }
+    my $winsize = '';
+    if (!ioctl($tty_fh, &TIOCGWINSZ, $winsize)) {
+        return -1;
+    }
+    my (undef, $col, undef, undef) = unpack('S4', $winsize);
+    return $col;
+}
+
+# Prints the current status, the percentage and a progress bar on STDERR if
+# it is an interactive tty and if verbosity is set to 1.
+#
+#    * first 12 chars: status
+#    * following 7 chars: percentage
+#    * progress bar until 79 chars are filled
 sub print_progress {
     if ($verbosity_level != 1) {
         return;
     }
-    my $perc = shift;
     if (!stderr_is_tty()) {
         return;
     }
+    my $perc         = shift;
+    my $status       = shift;
+    my $len_status   = 12;
+    my $len_perc     = 7;
+    my $len_prog_min = 10;
+    my $len_prog_max = 60;
+    my $twidth       = terminal_width();
+
+    if ($twidth <= $len_status) {
+        return;
+    }
+    # \e[2K clears everything on the current line (i.e. the progress bar)
+    print STDERR "\e[2K";
     if ($perc eq "done") {
-        # \e[2K clears everything on the current line (i.e. the progress bar)
-        print STDERR "\e[2Kdone\n";
+        print STDERR "done\n";
+        return;
+    }
+    if (defined $status) {
+        printf STDERR "%*s", -$len_status, "$status:";
+    } else {
+        print STDERR (" " x $len_status);
+    }
+    if ($twidth <= $len_status + $len_perc) {
+        print STDERR "\r";
         return;
     }
     if ($perc >= 100) {
         $perc = 100;
     }
-    my $width = 50;
-    my $num_x = int($perc * $width / 100);
-    my $bar   = '=' x $num_x;
-    if ($num_x != $width) {
-        $bar .= '>';
-        $bar .= ' ' x ($width - $num_x - 1);
+    printf STDERR "%*.2f", $len_perc, $perc;
+    if ($twidth <= $len_status + $len_perc + $len_prog_min) {
+        print STDERR "\r";
+        return;
     }
-    printf STDERR "%6.2f [%s]\r", $perc, $bar;
+    my $len_prog = $twidth - $len_perc - $len_status;
+    if ($len_prog > $len_prog_max) {
+        $len_prog = $len_prog_max;
+    }
+    my $num_x = int($perc * ($len_prog - 3) / 100);
+    my $bar   = '=' x $num_x;
+    if ($num_x != ($len_prog - 3)) {
+        $bar .= '>';
+        $bar .= ' ' x ($len_prog - $num_x - 4);
+    }
+    print STDERR " [$bar]\r";
     return;
 }
 
@@ -787,6 +845,15 @@ sub run_progress {
         POSIX::sigprocmask(SIG_UNBLOCK, $sigset)
           or error "Can't unblock signals: $!";
 
+        if ($verbosity_level != 1 || !stderr_is_tty()) {
+            # no need to print any progress
+            # we still need to consume everything from $rfh or otherwise apt
+            # will block forever if there is too much output
+            local $/;
+            <$rfh>;
+            close $rfh;
+            exit 0;
+        }
         my $progress = 0.0;
         my $status   = undef;
         print_progress($progress);
@@ -801,14 +868,7 @@ sub run_progress {
             if (defined $newstatus) {
                 $status = $newstatus;
             }
-            if (    defined $status
-                and $verbosity_level == 1
-                and stderr_is_tty()) {
-                # \e[2K clears everything on the current line (i.e. the
-                # progress bar)
-                print STDERR "\e[2K$status: ";
-            }
-            print_progress($newprogress);
+            print_progress($newprogress, $status);
             $progress = $newprogress;
         }
         print_progress("done");
@@ -2424,7 +2484,7 @@ sub run_extract() {
         }
         waitpid($pid3, 0);
         $? == 0 or error "tar --extract failed: $?";
-        print_progress($counter / $total * 100);
+        print_progress($counter / $total * 100, "extracting");
     }
     print_progress "done";
 
@@ -7171,8 +7231,8 @@ I<--merged-usr> and I<--no-merged-usr>
 
 B<mmdebstrap> will create a merged-/usr chroot or not depending on whether
 packages setting up merged-/usr (i.e. the B<usrmerge> package) are installed or
-not. In Debian, the essential package B<init-system-helpers> is will depend
-on the B<usrmerge> package, starting with Debian 12 (Bookworm).
+not. In Debian, the essential package B<init-system-helpers> depends on the
+B<usrmerge> package, starting with Debian 12 (Bookworm).
 
 Before Debian 12 (Bookworm), to force B<mmdebstrap> to create a chroot with
 merged-/usr using symlinks, either explicitly install the B<usrmerge> package:

tests/as-debootstrap-unshare-wrapper

@@ -47,6 +47,20 @@ rm /tmp/debian-mm/var/cache/apt/archives/lock
 rm /tmp/debian-mm/var/lib/apt/lists/lock
 rm /tmp/debian-mm/var/lib/dpkg/arch
 
+# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
+# also needed for users that are created by systemd-sysusers before systemd 252
+# https://github.com/systemd/systemd/pull/24534
+for f in shadow shadow-; do
+	if ! cmp /tmp/debian-debootstrap/etc/$f /tmp/debian-mm/etc/$f >&2; then
+		echo patching /etc/$f >&2
+		awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-mm/etc/$f > /tmp/debian-mm/etc/$f.bak
+		cat /tmp/debian-mm/etc/$f.bak > /tmp/debian-mm/etc/$f
+		rm /tmp/debian-mm/etc/$f.bak
+	else
+		echo no difference for /etc/$f >&2
+	fi
+done
+
 # check if the file content differs
 diff --no-dereference --recursive /tmp/debian-debootstrap /tmp/debian-mm >&2
 

tests/check-against-debootstrap-dist

@@ -10,13 +10,8 @@ echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
 # order in comparison to the systemd users
 # https://bugs.debian.org/969631
 # we cannot use useradd because passwd is not Essential:yes
-#
-# with cron 3.0pl1-142 and the introduction of cron-daemon-common, installation
-# order of cron and systemd started to differ between debootstrap and
-# mmdebstrap, resulting in different gid values
 {{ CMD }} --variant={{ VARIANT }} --mode={{ MODE }} \
-	--essential-hook='if [ {{ VARIANT }} = - ]; then echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd; fi' \
-	--essential-hook='if [ {{ VARIANT }} = - ] && [ {{ DIST }} = unstable -o {{ DIST }} = testing ]; then printf "systemd-journal:x:999:\nsystemd-network:x:998:\ncrontab:x:101:" >> "$1"/etc/group; fi' \
+	--essential-hook='case {{ DIST }} in oldstable|stable) if [ {{ VARIANT }} = - ]; then echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd; fi;; esac' \
 	$(case {{ DIST }} in oldstable|stable) : ;; *) echo --hook-dir=./hooks/merged-usr ;; esac) \
 	{{ DIST }} /tmp/debian-{{ DIST }}-mm.tar {{ MIRROR }}
 
@@ -29,34 +24,34 @@ tar --xattrs --xattrs-include='*' -C /tmp/debian-{{ DIST }}-debootstrap -xf "cac
 
 # diff cannot compare device nodes, so we use tar to do that for us and then
 # delete the directory
-tar -C /tmp/debian-{{ DIST }}-debootstrap -cf dev1.tar ./dev
-tar -C /tmp/debian-{{ DIST }}-mm -cf dev2.tar ./dev
+tar -C /tmp/debian-{{ DIST }}-debootstrap -cf /tmp/dev1.tar ./dev
+tar -C /tmp/debian-{{ DIST }}-mm -cf /tmp/dev2.tar ./dev
 ret=0
-cmp dev1.tar dev2.tar >&2 || ret=$?
+cmp /tmp/dev1.tar /tmp/dev2.tar >&2 || ret=$?
 if [ "$ret" -ne 0 ]; then
 	if type diffoscope >/dev/null; then
-		diffoscope dev1.tar dev2.tar
+		diffoscope /tmp/dev1.tar /tmp/dev2.tar
 		exit 1
 	else
 		echo "no diffoscope installed" >&2
 	fi
 	if type base64 >/dev/null; then
-		base64 dev1.tar
-		base64 dev2.tar
+		base64 /tmp/dev1.tar
+		base64 /tmp/dev2.tar
 		exit 1
 	else
 		echo "no base64 installed" >&2
 	fi
 	if type xxd >/dev/null; then
-		xxd dev1.tar
-		xxd dev2.tar
+		xxd /tmp/dev1.tar
+		xxd /tmp/dev2.tar
 		exit 1
 	else
 		echo "no xxd installed" >&2
 	fi
 	exit 1
 fi
-rm dev1.tar dev2.tar
+rm /tmp/dev1.tar /tmp/dev2.tar
 rm -r /tmp/debian-{{ DIST }}-debootstrap/dev /tmp/debian-{{ DIST }}-mm/dev
 
 # remove downloaded deb packages
@@ -81,14 +76,14 @@ rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/cmethopt
 rm /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/arch
 # since we installed packages directly from the .deb files, Priorities differ
 # thus we first check for equality and then remove the files
-chroot /tmp/debian-{{ DIST }}-debootstrap dpkg --list > dpkg1
-chroot /tmp/debian-{{ DIST }}-mm dpkg --list > dpkg2
-diff -u dpkg1 dpkg2
-rm dpkg1 dpkg2
-grep -v '^Priority: ' /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status > status1
-grep -v '^Priority: ' /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status > status2
-diff -u status1 status2
-rm status1 status2
+chroot /tmp/debian-{{ DIST }}-debootstrap dpkg --list > /tmp/dpkg1
+chroot /tmp/debian-{{ DIST }}-mm dpkg --list > /tmp/dpkg2
+diff -u /tmp/dpkg1 /tmp/dpkg2 >&2
+rm /tmp/dpkg1 /tmp/dpkg2
+grep -v '^Priority: ' /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status > /tmp/status1
+grep -v '^Priority: ' /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status > /tmp/status2
+diff -u /tmp/status1 /tmp/status2 >&2
+rm /tmp/status1 /tmp/status2
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status
 # debootstrap exposes the hosts's kernel version
 if [ -e /tmp/debian-{{ DIST }}-debootstrap/etc/apt/apt.conf.d/01autoremove-kernels ]; then
@@ -152,21 +147,13 @@ done
 # Because of unreproducible uids (#969631) we created the _apt user ourselves
 # and because passwd is not Essential:yes we didn't use useradd. But newer
 # versions of adduser and shadow will create a different /etc/shadow
+case {{ DIST }} in oldstable|stable)
 for f in shadow shadow-; do
 	if grep -q '^_apt:!:' /tmp/debian-{{ DIST }}-debootstrap/etc/$f; then
 		sed -i 's/^_apt:\*:\([^:]\+\):0:99999:7:::$/_apt:!:\1::::::/' /tmp/debian-{{ DIST }}-mm/etc/$f
 	fi
-done
-# same as above but for cron and systemd groups
-for f in gshadow gshadow-; do
-	for group in systemd-journal systemd-network crontab; do
-		for password in "!" "!\\*"; do
-			if grep -q '^'"$group"':'"$password"':' /tmp/debian-{{ DIST }}-debootstrap/etc/$f; then
-				sed -i 's/^'"$group"':x::/'"$group"':'"$password"'::/' /tmp/debian-{{ DIST }}-mm/etc/$f
-			fi
-		done
-	done
-done
+done;;
+esac
 
 for log in faillog lastlog; do
 	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/var/log/$log /tmp/debian-{{ DIST }}-mm/var/log/$log >&2;then
@@ -178,35 +165,37 @@ for log in faillog lastlog; do
 	fi
 done
 
-# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
-if ! cmp /tmp/debian-{{ DIST }}-debootstrap/etc/shadow /tmp/debian-{{ DIST }}-mm/etc/shadow >&2; then
-	echo patching /etc/shadow on {{ DIST }} {{ VARIANT }} >&2
-	awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-{{ DIST }}-mm/etc/shadow > /tmp/debian-{{ DIST }}-mm/etc/shadow.bak
-	cat /tmp/debian-{{ DIST }}-mm/etc/shadow.bak > /tmp/debian-{{ DIST }}-mm/etc/shadow
-	rm /tmp/debian-{{ DIST }}-mm/etc/shadow.bak
-else
-	echo no difference for /etc/shadow on {{ DIST }} {{ VARIANT }} >&2
-fi
-if ! cmp /tmp/debian-{{ DIST }}-debootstrap/etc/shadow- /tmp/debian-{{ DIST }}-mm/etc/shadow- >&2; then
-	echo patching /etc/shadow- on {{ DIST }} {{ VARIANT }} >&2
-	awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-{{ DIST }}-mm/etc/shadow- > /tmp/debian-{{ DIST }}-mm/etc/shadow-.bak
-	cat /tmp/debian-{{ DIST }}-mm/etc/shadow-.bak > /tmp/debian-{{ DIST }}-mm/etc/shadow-
-	rm /tmp/debian-{{ DIST }}-mm/etc/shadow-.bak
-else
-	echo no difference for /etc/shadow- on {{ DIST }} {{ VARIANT }} >&2
+# the order in which systemd and cron get installed differ and thus the order
+# of lines in /etc/group and /etc/gshadow differs
+if [ "{{ VARIANT }}" = "-" ]; then
+	case {{ DIST }} in testing|unstable)
+		for f in group group- gshadow gshadow-; do
+			! cmp /tmp/debian-{{ DIST }}-mm/etc/$f /tmp/debian-{{ DIST }}-debootstrap/etc/$f 2>/dev/null
+			for d in mm debootstrap; do
+				sort /tmp/debian-{{ DIST }}-$d/etc/$f > /tmp/debian-{{ DIST }}-$d/etc/$f.bak
+				mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
+			done
+		done
+		;;
+	esac
 fi
 
-# Because of unreproducible uids (#969631) we created the _apt user ourselves
-# and because passwd is not Essential:yes we didn't use useradd. But passwd
-# since 1:4.11.1+dfsg1-1 will create empty mail files, so we create it too.
-# https://bugs.debian.org/1004710
-if [ {{ VARIANT }} = - ]; then
-	if [ -e /tmp/debian-{{ DIST }}-debootstrap/var/mail/_apt ]; then
-		touch /tmp/debian-{{ DIST }}-mm/var/mail/_apt
-		chmod 660 /tmp/debian-{{ DIST }}-mm/var/mail/_apt
-		chown 100:8 /tmp/debian-{{ DIST }}-mm/var/mail/_apt
+# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
+# also needed for users that are created by systemd-sysusers before systemd 252
+# https://github.com/systemd/systemd/pull/24534
+for f in shadow shadow-; do
+	if [ ! -e /tmp/debian-{{ DIST }}-mm/etc/$f ]; then
+		continue
 	fi
-fi
+	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/etc/$f /tmp/debian-{{ DIST }}-mm/etc/$f >&2; then
+		echo patching /etc/$f on {{ DIST }} {{ VARIANT }} >&2
+		awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-{{ DIST }}-mm/etc/$f > /tmp/debian-{{ DIST }}-mm/etc/$f.bak
+		cat /tmp/debian-{{ DIST }}-mm/etc/$f.bak > /tmp/debian-{{ DIST }}-mm/etc/$f
+		rm /tmp/debian-{{ DIST }}-mm/etc/$f.bak
+	else
+		echo no difference for /etc/$f on {{ DIST }} {{ VARIANT }} >&2
+	fi
+done
 
 # check if the file content differs
 diff --unified --no-dereference --recursive /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm >&2
@@ -217,13 +206,13 @@ find /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm -type d -print
 # debootstrap never ran apt -- fixing permissions
 for d in ./var/lib/apt/lists/partial ./var/cache/apt/archives/partial; do
 	chroot /tmp/debian-{{ DIST }}-debootstrap chmod 0700 $d
-	chroot /tmp/debian-{{ DIST }}-debootstrap chown _apt:root $d
+	chroot /tmp/debian-{{ DIST }}-debootstrap chown $(id -u _apt):root $d
 done
 tar -C /tmp/debian-{{ DIST }}-debootstrap --numeric-owner --sort=name --clamp-mtime --mtime=$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds) -cf /tmp/root1.tar .
 tar -C /tmp/debian-{{ DIST }}-mm --numeric-owner --sort=name --clamp-mtime --mtime=$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds) -cf /tmp/root2.tar .
 tar --full-time --verbose -tf /tmp/root1.tar > /tmp/root1.tar.list
 tar --full-time --verbose -tf /tmp/root2.tar > /tmp/root2.tar.list
-diff -u /tmp/root1.tar.list /tmp/root2.tar.list
+diff -u /tmp/root1.tar.list /tmp/root2.tar.list >&2
 rm /tmp/root1.tar /tmp/root2.tar /tmp/root1.tar.list /tmp/root2.tar.list
 
 # check if file properties (permissions, ownership, symlink names, modification time) differ

tests/check-for-bit-by-bit-identical-format-output

@@ -31,4 +31,7 @@ case {{ VARIANT }} in essential|apt|minbase|buildd)
 	rm /tmp/debian-chroot-fakechroot.{{ FORMAT }}
 	;;
 esac
+# we cannot test chrootless mode here, because mmdebstrap relies on the
+# usrmerge package to set up merged-/usr and that doesn't work in chrootless
+# mode
 rm /tmp/debian-chroot-root.{{ FORMAT }}

tests/chrootless

@@ -0,0 +1,16 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
+# we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
+# DPKG_ROOT
+for INCLUDE in '' 'systemd-sysv'; do
+	for MODE in root chrootless; do
+		{{ CMD }} --mode=$MODE --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
+			${INCLUDE:+--include="$INCLUDE"} \
+			{{ DIST }} "/tmp/$MODE.tar" {{ MIRROR }}
+	done
+	cmp /tmp/root.tar /tmp/chrootless.tar
+	rm /tmp/chrootless.tar /tmp/root.tar
+done

tests/chrootless-essential

@@ -1,13 +0,0 @@
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	if [ ! -e /mmdebstrap-testenv ]; then
-		echo "this test modifies the system and should only be run inside a container" >&2
-		exit 1
-	fi
-	adduser --gecos user --disabled-password user
-fi
-prefix=
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
-$prefix fakechroot fakeroot {{ CMD }} --mode=chrootless --variant=essential {{ DIST }} /dev/null {{ MIRROR }}

tests/chrootless-fakeroot

@@ -0,0 +1,29 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	if [ ! -e /mmdebstrap-testenv ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
+		exit 1
+	fi
+	adduser --gecos user --disabled-password user
+fi
+prefix=
+[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+# we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
+# DPKG_ROOT
+# permissions drwxr-sr-x and extended attributes of ./var/log/journal/ cannot
+# be preserved under fakeroot
+for INCLUDE in '' 'systemd-sysv'; do
+	{{ CMD }} --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
+		--customize-hook='if [ -d "$1"/var/log/journal ]; then rmdir "$1"/var/log/journal; mkdir --mode=2755 "$1"/var/log/journal; chroot "$1" chown root:systemd-journal /var/log/journal; fi' \
+		${INCLUDE:+--include="$INCLUDE"} \
+		{{ DIST }} /tmp/root.tar {{ MIRROR }}
+	$prefix fakeroot {{ CMD }} --mode=chrootless --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
+		${INCLUDE:+--include="$INCLUDE"} \
+		{{ DIST }} /tmp/chrootless.tar {{ MIRROR }}
+	cmp /tmp/root.tar /tmp/chrootless.tar
+	rm /tmp/chrootless.tar /tmp/root.tar
+done

tests/chrootless-foreign

@@ -0,0 +1,46 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+
+[ "$(id -u)" -eq 0 ]
+[ -e /proc/sys/fs/binfmt_misc/qemu-aarch64 ]
+
+# we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
+# DPKG_ROOT
+for INCLUDE in '' 'systemd-sysv'; do
+	echo 1 > /proc/sys/fs/binfmt_misc/qemu-aarch64
+	arch-test arm64
+	{{ CMD }} --mode=root --architecture=arm64 --variant={{ VARIANT }} \
+		--hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
+		{{ DIST }} "/tmp/root.tar" {{ MIRROR }}
+	echo 0 > /proc/sys/fs/binfmt_misc/qemu-aarch64
+	! arch-test arm64
+	{{ CMD }} --mode=chrootless --architecture=arm64 --variant={{ VARIANT }} \
+		--hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
+		{{ DIST }} "/tmp/chrootless.tar" {{ MIRROR }}
+	# when creating a foreign architecture chroot, the tarballs are not
+	# bit-by-bit identical but contain a few remaining differences:
+	#
+	#  * /etc/ld.so.cache -- hard problem, must be solved in glibc upstream
+	#  * /var/lib/dpkg/triggers -- #990712
+	#  * /var/cache/debconf/*.dat-old -- needs investigation
+	for tar in root chrootless; do
+		<"/tmp/$tar.tar" \
+		./tarfilter \
+			--path-exclude=/var/cache/debconf/config.dat-old \
+			--path-exclude=/var/cache/debconf/templates.dat-old \
+			--path-exclude=/etc/ld.so.cache \
+			--path-exclude=/var/lib/dpkg/triggers/File \
+			--path-exclude=/var/lib/dpkg/triggers/ldconfig \
+			> "/tmp/$tar.tar.tmp"
+		mv "/tmp/$tar.tar.tmp" "/tmp/$tar.tar"
+	done
+	cmp /tmp/root.tar /tmp/chrootless.tar
+	rm /tmp/chrootless.tar /tmp/root.tar
+done

tests/compare-output-with-pre-seeded-var-cache-apt-archives

@@ -10,6 +10,9 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test requires the cache directory to be mounted on /mnt and should only be run inside a container" >&2
 	exit 1
 fi
+tmpdir=$(mktemp -d)
+trap "rm -f \"$tmpdir\"/*.deb /tmp/orig.tar /tmp/test1.tar /tmp/test2.tar; rmdir \"$tmpdir\"" EXIT INT TERM
+
 include="--include=doc-debian"
 if [ "{{ VARIANT }}" = "custom" ]; then
 	include="$include,base-files,base-passwd,coreutils,dash,diffutils,dpkg,libc-bin,sed"
@@ -18,27 +21,24 @@ fi
 	--setup-hook='mkdir -p "$1"/var/cache/apt/archives/partial' \
 	--setup-hook='touch "$1"/var/cache/apt/archives/lock' \
 	--setup-hook='chmod 0640 "$1"/var/cache/apt/archives/lock' \
-	{{ DIST }} - {{ MIRROR }} > orig.tar
+	{{ DIST }} - {{ MIRROR }} > /tmp/orig.tar
 # somehow, when trying to create a tarball from the 9p mount, tar throws the
 # following error: tar: ./doc-debian_6.4_all.deb: File shrank by 132942 bytes; padding with zeros
 # to reproduce, try: tar --directory /mnt/cache/debian/pool/main/d/doc-debian/ --create --file - . | tar --directory /tmp/ --extract --file -
 # this will be different:
 # md5sum /mnt/cache/debian/pool/main/d/doc-debian/*.deb /tmp/*.deb
 # another reason to copy the files into a new directory is, that we can use shell globs
-tmpdir=$(mktemp -d)
 cp /mnt/cache/debian/pool/main/b/busybox/busybox_*"_{{ HOSTARCH }}.deb" /mnt/cache/debian/pool/main/a/apt/apt_*"_{{ HOSTARCH }}.deb" "$tmpdir"
 {{ CMD }} $include --mode={{ MODE }} --variant={{ VARIANT }} \
 	--setup-hook='mkdir -p "$1"/var/cache/apt/archives/partial' \
 	--setup-hook='sync-in "'"$tmpdir"'" /var/cache/apt/archives/partial' \
-	{{ DIST }} - {{ MIRROR }} > test1.tar
-cmp orig.tar test1.tar
+	{{ DIST }} - {{ MIRROR }} > /tmp/test1.tar
+cmp /tmp/orig.tar /tmp/test1.tar
 {{ CMD }} $include --mode={{ MODE }} --variant={{ VARIANT }} \
 	--customize-hook='touch "$1"/var/cache/apt/archives/partial' \
 	--setup-hook='mkdir -p "$1"/var/cache/apt/archives/' \
 	--setup-hook='sync-in "'"$tmpdir"'" /var/cache/apt/archives/' \
 	--setup-hook='chmod 0755 "$1"/var/cache/apt/archives/' \
 	--customize-hook='find "'"$tmpdir"'" -type f -exec md5sum "{}" \; | sed "s|"'"$tmpdir"'"|$1/var/cache/apt/archives|" | md5sum --check' \
-	{{ DIST }} - {{ MIRROR }} > test2.tar
-cmp orig.tar test2.tar
-rm "$tmpdir"/*.deb orig.tar test1.tar test2.tar
-rmdir "$tmpdir"
+	{{ DIST }} - {{ MIRROR }} > /tmp/test2.tar
+cmp /tmp/orig.tar /tmp/test2.tar

tests/create-arm64-tarball

@@ -22,8 +22,22 @@ $prefix {{ CMD }} --mode={{ MODE }} --variant=apt --architectures=arm64 {{ DIST
 # we ignore differences between architectures by ignoring some files
 # and renaming others
 { tar -tf /tmp/debian-chroot.tar \
-	| grep -v '^\./lib/ld-linux-aarch64\.so\.1$' \
-	| grep -v '^\./lib/aarch64-linux-gnu/ld-linux-aarch64\.so\.1$' \
+	| grep -v '^\./usr/lib/ld-linux-aarch64\.so\.1$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/ld-linux-aarch64\.so\.1$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/int-ll64\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/types\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/unistd\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/sigcontext\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/sve_context\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/types\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-extra\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-id\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-prregset\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/gnu/stubs-lp64\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/linux/types\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/sys/procfs\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/sys/user\.ph$' \
 	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.arm64\.gz$' \
 	| sed 's/aarch64-linux-gnu/x86_64-linux-gnu/' \
 	| sed 's/arm64/amd64/';
@@ -31,13 +45,24 @@ $prefix {{ CMD }} --mode={{ MODE }} --variant=apt --architectures=arm64 {{ DIST
 { cat tar1.txt \
 	| grep -v '^\./usr/bin/i386$' \
 	| grep -v '^\./usr/bin/x86_64$' \
-	| grep -v '^\./lib64/$' \
-	| grep -v '^\./lib64/ld-linux-x86-64\.so\.2$' \
-	| grep -v '^\./lib/x86_64-linux-gnu/ld-linux-x86-64\.so\.2$' \
-	| grep -v '^\./lib/x86_64-linux-gnu/libmvec-2\.[0-9]\+\.so$' \
-	| grep -v '^\./lib/x86_64-linux-gnu/libmvec\.so\.1$' \
+	| grep -v '^\./lib32$' \
+	| grep -v '^\./lib64$' \
+	| grep -v '^\./libx32$' \
+	| grep -v '^\./usr/lib32/$' \
+	| grep -v '^\./usr/libx32/$' \
+	| grep -v '^\./usr/lib64/$' \
+	| grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/ld-linux-x86-64\.so\.2$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/libmvec\.so\.1$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_32\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_64\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_x32\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_32\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_64\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_x32\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/gnu/stubs-64\.ph$' \
 	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.amd64\.gz$' \
 	| grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
 	| grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$';
-} | sort | diff -u - tar2.txt
+} | sort | diff -u - tar2.txt >&2
 rm /tmp/debian-chroot.tar

tests/create-tarball-with-tmp-mounted-nodev

@@ -5,7 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-mount -t tmpfs -o nodev,nosuid,size=300M tmpfs /tmp
+mount -t tmpfs -o nodev,nosuid,size=400M tmpfs /tmp
 # use --customize-hook to exercise the mounting/unmounting code of block devices in root mode
 {{ CMD }} --mode=root --variant=apt --customize-hook='mount | grep /dev/full' --customize-hook='test "$(echo foo | tee /dev/full 2>&1 1>/dev/null)" = "tee: /dev/full: No space left on device"' {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/i386-which-can-be-executed-without-qemu

@@ -12,27 +12,29 @@ apt-get remove --yes qemu-user-static binfmt-support qemu-user
 # and renaming others
 { tar -tf /tmp/debian-chroot.tar \
 	| grep -v '^\./usr/bin/i386$' \
-	| grep -v '^\./lib/ld-linux\.so\.2$' \
-	| grep -v '^\./lib/i386-linux-gnu/ld-linux\.so\.2$' \
+	| grep -v '^\./usr/lib/ld-linux\.so\.2$' \
+	| grep -v '^\./usr/lib/i386-linux-gnu/ld-linux\.so\.2$' \
 	| grep -v '^\./usr/lib/gcc/i686-linux-gnu/$' \
 	| grep -v '^\./usr/lib/gcc/i686-linux-gnu/[0-9]\+/$' \
 	| grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
 	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.i386\.gz$' \
 	| sed 's/i386-linux-gnu/x86_64-linux-gnu/' \
-	| sed 's/i386/amd64/';
+	| sed 's/i386/amd64/' \
+	| sed 's/\/stubs-32.ph$/\/stubs-64.ph/';
 } | sort > tar2.txt
 { cat tar1.txt \
 	| grep -v '^\./usr/bin/i386$' \
 	| grep -v '^\./usr/bin/x86_64$' \
+	| grep -v '^\./usr/lib32/$' \
+	| grep -v '^\./lib32$' \
 	| grep -v '^\./lib64/$' \
-	| grep -v '^\./lib64/ld-linux-x86-64\.so\.2$' \
+	| grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
 	| grep -v '^\./usr/lib/gcc/x86_64-linux-gnu/$' \
 	| grep -v '^\./usr/lib/gcc/x86_64-linux-gnu/[0-9]\+/$' \
-	| grep -v '^\./lib/x86_64-linux-gnu/ld-linux-x86-64\.so\.2$' \
-	| grep -v '^\./lib/x86_64-linux-gnu/libmvec-2\.[0-9]\+\.so$' \
-	| grep -v '^\./lib/x86_64-linux-gnu/libmvec\.so\.1$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/ld-linux-x86-64\.so\.2$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/libmvec\.so\.1$' \
 	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.amd64\.gz$' \
 	| grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
 	| grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$';
-} | sort | diff -u - tar2.txt
+} | sort | diff -u - tar2.txt >&2
 rm /tmp/debian-chroot.tar

tests/tarfilter-idshift

@@ -18,11 +18,12 @@ runuser -u user -- {{ CMD }} --mode=unshare --variant=apt --include=iputils-ping
 	| ./tarfilter --idshift 0 > /tmp/debian-chroot.tar
 # make sure that xattrs are set in the original tarball
 mkdir /tmp/debian-chroot
-tar --xattrs --xattrs-include='*' --directory /tmp/debian-chroot -xf /tmp/debian-chroot.tar ./bin/ping
-echo "/tmp/debian-chroot/bin/ping cap_net_raw=ep" > /tmp/expected
-getcap /tmp/debian-chroot/bin/ping | diff -u /tmp/expected -
-rm /tmp/debian-chroot/bin/ping
-rmdir /tmp/debian-chroot/bin
+tar --xattrs --xattrs-include='*' --directory /tmp/debian-chroot -xf /tmp/debian-chroot.tar ./usr/bin/ping
+echo "/tmp/debian-chroot/usr/bin/ping cap_net_raw=ep" > /tmp/expected
+getcap /tmp/debian-chroot/usr/bin/ping | diff -u /tmp/expected - >&2
+rm /tmp/debian-chroot/usr/bin/ping
+rmdir /tmp/debian-chroot/usr/bin
+rmdir /tmp/debian-chroot/usr
 rmdir /tmp/debian-chroot
 # shift the uid/gid forward by 100000 and backward by 100000
 ./tarfilter --idshift 100000 < /tmp/debian-chroot.tar > /tmp/debian-chroot-shifted.tar
@@ -32,8 +33,7 @@ rmdir /tmp/debian-chroot
 cmp /tmp/debian-chroot.tar /tmp/debian-chroot-shiftedback.tar
 # manually adjust uid/gid and compare "tar -t" output
 tar --numeric-owner -tvf /tmp/debian-chroot.tar \
-	| sed 's# 100/0 # 100100/100000 #' \
-	| sed 's# 100/8 # 100100/100008 #' \
+	| sed 's# 42/0 # 100042/100000 #' \
 	| sed 's# 0/0 # 100000/100000 #' \
 	| sed 's# 0/5 # 100000/100005 #' \
 	| sed 's# 0/8 # 100000/100008 #' \
@@ -44,16 +44,16 @@ tar --numeric-owner -tvf /tmp/debian-chroot.tar \
 	> /tmp/debian-chroot.txt
 tar --numeric-owner -tvf /tmp/debian-chroot-shifted.tar \
 	| sed 's/ \+/ /g' \
-	| diff -u /tmp/debian-chroot.txt -
+	| diff -u /tmp/debian-chroot.txt - >&2
 mkdir /tmp/debian-chroot
 tar --xattrs --xattrs-include='*' --directory /tmp/debian-chroot -xf /tmp/debian-chroot-shifted.tar
 echo "100000 100000" > /tmp/expected
-stat --format="%u %g" /tmp/debian-chroot/bin/ping | diff -u /tmp/expected -
-echo "/tmp/debian-chroot/bin/ping cap_net_raw=ep" > /tmp/expected
-getcap /tmp/debian-chroot/bin/ping | diff -u /tmp/expected -
+stat --format="%u %g" /tmp/debian-chroot/usr/bin/ping | diff -u /tmp/expected - >&2
+echo "/tmp/debian-chroot/usr/bin/ping cap_net_raw=ep" > /tmp/expected
+getcap /tmp/debian-chroot/usr/bin/ping | diff -u /tmp/expected - >&2
 echo "0 0" > /tmp/expected
-runuser -u user -- {{ CMD }} --unshare-helper /usr/sbin/chroot /tmp/debian-chroot stat --format="%u %g" /bin/ping \
-	| diff -u /tmp/expected -
-echo "/bin/ping cap_net_raw=ep" > /tmp/expected
-runuser -u user -- {{ CMD }} --unshare-helper /usr/sbin/chroot /tmp/debian-chroot getcap /bin/ping \
-	| diff -u /tmp/expected -
+runuser -u user -- {{ CMD }} --unshare-helper /usr/sbin/chroot /tmp/debian-chroot stat --format="%u %g" /usr/bin/ping \
+	| diff -u /tmp/expected - >&2
+echo "/usr/bin/ping cap_net_raw=ep" > /tmp/expected
+runuser -u user -- {{ CMD }} --unshare-helper /usr/sbin/chroot /tmp/debian-chroot getcap /usr/bin/ping \
+	| diff -u /tmp/expected - >&2