CHANGELOG.md

@@ -1,12 +1,3 @@
-1.2.2 (2022-10-27)
-------------------
-
- - allow /etc/apt/trusted.gpg.d/ not to exist
- - always create /var/lib/dpkg/arch to make foreign architecture chrootless
-   tarballs bit-by-bit identical
- - write an empty /etc/machine-id instead of writing 'uninitialized'
- - only print progress bars on interactive terminals that are wide enough
-
 1.2.1 (2022-09-08)
 ------------------
 

coverage.py

@@ -121,13 +121,13 @@ def parse_config(confname):
 
 def format_failed(num, total, name, dist, mode, variant, fmt, config_dict):
     ret = f"({num}/{total}) {name}"
-    if len(config_dict[name].get("Dists", [])) > 1:
+    if len(config_dict[name].get("Dists", [])) > 0:
         ret += f" --dist={dist}"
-    if len(config_dict[name].get("Modes", [])) > 1:
+    if len(config_dict[name].get("Modes", [])) > 0:
         ret += f" --mode={mode}"
-    if len(config_dict[name].get("Variants", [])) > 1:
+    if len(config_dict[name].get("Variants", [])) > 0:
         ret += f" --variant={variant}"
-    if len(config_dict[name].get("Formats", [])) > 1:
+    if len(config_dict[name].get("Formats", [])) > 0:
         ret += f" --format={fmt}"
     return ret
 

coverage.txt

@@ -288,27 +288,12 @@ Test: install-doc-debian
 Modes: chrootless
 Variants: custom
 
-Test: chrootless
+Test: chrootless-essential
-Variants: essential
+Variants: custom
 Modes: chrootless
 Skip-If:
  dist in ["oldstable", "stable"]
 
-Test: chrootless-fakeroot
-Variants: essential
-Modes: chrootless
-Skip-If:
- dist in ["oldstable", "stable"]
-
-Test: chrootless-foreign
-Variants: essential
-Modes: chrootless
-Skip-If:
- dist in ["oldstable", "stable"]
- hostarch != "amd64"
- not run_ma_same_tests
-Needs-QEMU: true
-
 Test: install-doc-debian-and-output-tarball
 Variants: custom
 Modes: chrootless

make_mirror.sh

@@ -270,7 +270,7 @@ END
 	# starting from Debian 12 (Bullseye)
 	case "$dist" in
 		oldstable|stable) : ;;
-		*) pkgs="$pkgs usr-is-merged usrmerge" ;;
+		*) pkgs="$pkgs usr-is-merged" ;;
 	esac
 
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs
@@ -546,7 +546,7 @@ timeout 0
 
 label linux
 kernel /vmlinuz
-append initrd=/initrd.img root=/dev/vda1 rw console=tty0 console=ttyS0,115200n8
+append initrd=/initrd.img root=/dev/vda1 rw console=ttyS0,115200
 serial 0 115200
 END
 	cat << END > "$tmpdir/mmdebstrap.service"

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '1.2.2';
+our $VERSION = '1.2.1';
 
 use English;
 use Getopt::Long;
@@ -35,7 +35,6 @@ use File::Basename;
 use File::Find;
 use Cwd qw(abs_path getcwd);
 require "syscall.ph";    ## no critic (Modules::RequireBarewordIncludes)
-require "sys/ioctl.ph";    ## no critic (Modules::RequireBarewordIncludes)
 use Fcntl qw(S_IFCHR S_IFBLK FD_CLOEXEC F_GETFD F_SETFD);
 use List::Util qw(any none);
 use POSIX
@@ -695,87 +694,30 @@ sub havemknod {
     return $havemknod;
 }
 
-# inspired by /usr/share/perl/5.34/pod/perlfaq8.pod
-sub terminal_width {
-    if (!stderr_is_tty()) {
-        return -1;
-    }
-    if (!defined &TIOCGWINSZ) {
-        return -1;
-    }
-    if (!-e "/dev/tty") {
-        return -1;
-    }
-    my $tty_fh;
-    if (!open($tty_fh, "+<", "/dev/tty")) {
-        return -1;
-    }
-    my $winsize = '';
-    if (!ioctl($tty_fh, &TIOCGWINSZ, $winsize)) {
-        return -1;
-    }
-    my (undef, $col, undef, undef) = unpack('S4', $winsize);
-    return $col;
-}
-
-# Prints the current status, the percentage and a progress bar on STDERR if
-# it is an interactive tty and if verbosity is set to 1.
-#
-#    * first 12 chars: status
-#    * following 7 chars: percentage
-#    * progress bar until 79 chars are filled
 sub print_progress {
     if ($verbosity_level != 1) {
         return;
     }
+    my $perc = shift;
     if (!stderr_is_tty()) {
         return;
     }
-    my $perc         = shift;
-    my $status       = shift;
-    my $len_status   = 12;
-    my $len_perc     = 7;
-    my $len_prog_min = 10;
-    my $len_prog_max = 60;
-    my $twidth       = terminal_width();
-
-    if ($twidth <= $len_status) {
-        return;
-    }
-    # \e[2K clears everything on the current line (i.e. the progress bar)
-    print STDERR "\e[2K";
     if ($perc eq "done") {
-        print STDERR "done\n";
+        # \e[2K clears everything on the current line (i.e. the progress bar)
-        return;
+        print STDERR "\e[2Kdone\n";
-    }
-    if (defined $status) {
-        printf STDERR "%*s", -$len_status, "$status:";
-    } else {
-        print STDERR (" " x $len_status);
-    }
-    if ($twidth <= $len_status + $len_perc) {
-        print STDERR "\r";
         return;
     }
     if ($perc >= 100) {
         $perc = 100;
     }
-    printf STDERR "%*.2f", $len_perc, $perc;
+    my $width = 50;
-    if ($twidth <= $len_status + $len_perc + $len_prog_min) {
+    my $num_x = int($perc * $width / 100);
-        print STDERR "\r";
-        return;
-    }
-    my $len_prog = $twidth - $len_perc - $len_status;
-    if ($len_prog > $len_prog_max) {
-        $len_prog = $len_prog_max;
-    }
-    my $num_x = int($perc * ($len_prog - 3) / 100);
     my $bar   = '=' x $num_x;
-    if ($num_x != ($len_prog - 3)) {
+    if ($num_x != $width) {
         $bar .= '>';
-        $bar .= ' ' x ($len_prog - $num_x - 4);
+        $bar .= ' ' x ($width - $num_x - 1);
     }
-    print STDERR " [$bar]\r";
+    printf STDERR "%6.2f [%s]\r", $perc, $bar;
     return;
 }
 
@@ -845,15 +787,6 @@ sub run_progress {
         POSIX::sigprocmask(SIG_UNBLOCK, $sigset)
           or error "Can't unblock signals: $!";
 
-        if ($verbosity_level != 1 || !stderr_is_tty()) {
-            # no need to print any progress
-            # we still need to consume everything from $rfh or otherwise apt
-            # will block forever if there is too much output
-            local $/;
-            <$rfh>;
-            close $rfh;
-            exit 0;
-        }
         my $progress = 0.0;
         my $status   = undef;
         print_progress($progress);
@@ -868,7 +801,14 @@ sub run_progress {
             if (defined $newstatus) {
                 $status = $newstatus;
             }
-            print_progress($newprogress, $status);
+            if (    defined $status
+                and $verbosity_level == 1
+                and stderr_is_tty()) {
+                # \e[2K clears everything on the current line (i.e. the
+                # progress bar)
+                print STDERR "\e[2K$status: ";
+            }
+            print_progress($newprogress);
             $progress = $newprogress;
         }
         print_progress("done");
@@ -2484,7 +2424,7 @@ sub run_extract() {
         }
         waitpid($pid3, 0);
         $? == 0 or error "tar --extract failed: $?";
-        print_progress($counter / $total * 100, "extracting");
+        print_progress($counter / $total * 100);
     }
     print_progress "done";
 
@@ -7231,8 +7171,8 @@ I<--merged-usr> and I<--no-merged-usr>
 
 B<mmdebstrap> will create a merged-/usr chroot or not depending on whether
 packages setting up merged-/usr (i.e. the B<usrmerge> package) are installed or
-not. In Debian, the essential package B<init-system-helpers> depends on the
+not. In Debian, the essential package B<init-system-helpers> is will depend
-B<usrmerge> package, starting with Debian 12 (Bookworm).
+on the B<usrmerge> package, starting with Debian 12 (Bookworm).
 
 Before Debian 12 (Bookworm), to force B<mmdebstrap> to create a chroot with
 merged-/usr using symlinks, either explicitly install the B<usrmerge> package:

tests/as-debootstrap-unshare-wrapper

@@ -47,20 +47,6 @@ rm /tmp/debian-mm/var/cache/apt/archives/lock
 rm /tmp/debian-mm/var/lib/apt/lists/lock
 rm /tmp/debian-mm/var/lib/dpkg/arch
 
-# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
-# also needed for users that are created by systemd-sysusers before systemd 252
-# https://github.com/systemd/systemd/pull/24534
-for f in shadow shadow-; do
-	if ! cmp /tmp/debian-debootstrap/etc/$f /tmp/debian-mm/etc/$f >&2; then
-		echo patching /etc/$f >&2
-		awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-mm/etc/$f > /tmp/debian-mm/etc/$f.bak
-		cat /tmp/debian-mm/etc/$f.bak > /tmp/debian-mm/etc/$f
-		rm /tmp/debian-mm/etc/$f.bak
-	else
-		echo no difference for /etc/$f >&2
-	fi
-done
-
 # check if the file content differs
 diff --no-dereference --recursive /tmp/debian-debootstrap /tmp/debian-mm >&2
 

tests/check-against-debootstrap-dist

@@ -10,8 +10,13 @@ echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
 # order in comparison to the systemd users
 # https://bugs.debian.org/969631
 # we cannot use useradd because passwd is not Essential:yes
+#
+# with cron 3.0pl1-142 and the introduction of cron-daemon-common, installation
+# order of cron and systemd started to differ between debootstrap and
+# mmdebstrap, resulting in different gid values
 {{ CMD }} --variant={{ VARIANT }} --mode={{ MODE }} \
-	--essential-hook='case {{ DIST }} in oldstable|stable) if [ {{ VARIANT }} = - ]; then echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd; fi;; esac' \
+	--essential-hook='if [ {{ VARIANT }} = - ]; then echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd; fi' \
+	--essential-hook='if [ {{ VARIANT }} = - ] && [ {{ DIST }} = unstable -o {{ DIST }} = testing ]; then printf "systemd-journal:x:999:\nsystemd-network:x:998:\ncrontab:x:101:" >> "$1"/etc/group; fi' \
 	$(case {{ DIST }} in oldstable|stable) : ;; *) echo --hook-dir=./hooks/merged-usr ;; esac) \
 	{{ DIST }} /tmp/debian-{{ DIST }}-mm.tar {{ MIRROR }}
 
@@ -24,34 +29,34 @@ tar --xattrs --xattrs-include='*' -C /tmp/debian-{{ DIST }}-debootstrap -xf "cac
 
 # diff cannot compare device nodes, so we use tar to do that for us and then
 # delete the directory
-tar -C /tmp/debian-{{ DIST }}-debootstrap -cf /tmp/dev1.tar ./dev
+tar -C /tmp/debian-{{ DIST }}-debootstrap -cf dev1.tar ./dev
-tar -C /tmp/debian-{{ DIST }}-mm -cf /tmp/dev2.tar ./dev
+tar -C /tmp/debian-{{ DIST }}-mm -cf dev2.tar ./dev
 ret=0
-cmp /tmp/dev1.tar /tmp/dev2.tar >&2 || ret=$?
+cmp dev1.tar dev2.tar >&2 || ret=$?
 if [ "$ret" -ne 0 ]; then
 	if type diffoscope >/dev/null; then
-		diffoscope /tmp/dev1.tar /tmp/dev2.tar
+		diffoscope dev1.tar dev2.tar
 		exit 1
 	else
 		echo "no diffoscope installed" >&2
 	fi
 	if type base64 >/dev/null; then
-		base64 /tmp/dev1.tar
+		base64 dev1.tar
-		base64 /tmp/dev2.tar
+		base64 dev2.tar
 		exit 1
 	else
 		echo "no base64 installed" >&2
 	fi
 	if type xxd >/dev/null; then
-		xxd /tmp/dev1.tar
+		xxd dev1.tar
-		xxd /tmp/dev2.tar
+		xxd dev2.tar
 		exit 1
 	else
 		echo "no xxd installed" >&2
 	fi
 	exit 1
 fi
-rm /tmp/dev1.tar /tmp/dev2.tar
+rm dev1.tar dev2.tar
 rm -r /tmp/debian-{{ DIST }}-debootstrap/dev /tmp/debian-{{ DIST }}-mm/dev
 
 # remove downloaded deb packages
@@ -76,14 +81,14 @@ rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/cmethopt
 rm /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/arch
 # since we installed packages directly from the .deb files, Priorities differ
 # thus we first check for equality and then remove the files
-chroot /tmp/debian-{{ DIST }}-debootstrap dpkg --list > /tmp/dpkg1
+chroot /tmp/debian-{{ DIST }}-debootstrap dpkg --list > dpkg1
-chroot /tmp/debian-{{ DIST }}-mm dpkg --list > /tmp/dpkg2
+chroot /tmp/debian-{{ DIST }}-mm dpkg --list > dpkg2
-diff -u /tmp/dpkg1 /tmp/dpkg2 >&2
+diff -u dpkg1 dpkg2
-rm /tmp/dpkg1 /tmp/dpkg2
+rm dpkg1 dpkg2
-grep -v '^Priority: ' /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status > /tmp/status1
+grep -v '^Priority: ' /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status > status1
-grep -v '^Priority: ' /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status > /tmp/status2
+grep -v '^Priority: ' /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status > status2
-diff -u /tmp/status1 /tmp/status2 >&2
+diff -u status1 status2
-rm /tmp/status1 /tmp/status2
+rm status1 status2
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status
 # debootstrap exposes the hosts's kernel version
 if [ -e /tmp/debian-{{ DIST }}-debootstrap/etc/apt/apt.conf.d/01autoremove-kernels ]; then
@@ -147,13 +152,21 @@ done
 # Because of unreproducible uids (#969631) we created the _apt user ourselves
 # and because passwd is not Essential:yes we didn't use useradd. But newer
 # versions of adduser and shadow will create a different /etc/shadow
-case {{ DIST }} in oldstable|stable)
 for f in shadow shadow-; do
 	if grep -q '^_apt:!:' /tmp/debian-{{ DIST }}-debootstrap/etc/$f; then
 		sed -i 's/^_apt:\*:\([^:]\+\):0:99999:7:::$/_apt:!:\1::::::/' /tmp/debian-{{ DIST }}-mm/etc/$f
 	fi
-done;;
+done
-esac
+# same as above but for cron and systemd groups
+for f in gshadow gshadow-; do
+	for group in systemd-journal systemd-network crontab; do
+		for password in "!" "!\\*"; do
+			if grep -q '^'"$group"':'"$password"':' /tmp/debian-{{ DIST }}-debootstrap/etc/$f; then
+				sed -i 's/^'"$group"':x::/'"$group"':'"$password"'::/' /tmp/debian-{{ DIST }}-mm/etc/$f
+			fi
+		done
+	done
+done
 
 for log in faillog lastlog; do
 	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/var/log/$log /tmp/debian-{{ DIST }}-mm/var/log/$log >&2;then
@@ -165,37 +178,35 @@ for log in faillog lastlog; do
 	fi
 done
 
-# the order in which systemd and cron get installed differ and thus the order
+# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
-# of lines in /etc/group and /etc/gshadow differs
+if ! cmp /tmp/debian-{{ DIST }}-debootstrap/etc/shadow /tmp/debian-{{ DIST }}-mm/etc/shadow >&2; then
-if [ "{{ VARIANT }}" = "-" ]; then
+	echo patching /etc/shadow on {{ DIST }} {{ VARIANT }} >&2
-	case {{ DIST }} in testing|unstable)
+	awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-{{ DIST }}-mm/etc/shadow > /tmp/debian-{{ DIST }}-mm/etc/shadow.bak
-		for f in group group- gshadow gshadow-; do
+	cat /tmp/debian-{{ DIST }}-mm/etc/shadow.bak > /tmp/debian-{{ DIST }}-mm/etc/shadow
-			! cmp /tmp/debian-{{ DIST }}-mm/etc/$f /tmp/debian-{{ DIST }}-debootstrap/etc/$f 2>/dev/null
+	rm /tmp/debian-{{ DIST }}-mm/etc/shadow.bak
-			for d in mm debootstrap; do
+else
-				sort /tmp/debian-{{ DIST }}-$d/etc/$f > /tmp/debian-{{ DIST }}-$d/etc/$f.bak
+	echo no difference for /etc/shadow on {{ DIST }} {{ VARIANT }} >&2
-				mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
+fi
-			done
+if ! cmp /tmp/debian-{{ DIST }}-debootstrap/etc/shadow- /tmp/debian-{{ DIST }}-mm/etc/shadow- >&2; then
-		done
+	echo patching /etc/shadow- on {{ DIST }} {{ VARIANT }} >&2
-		;;
+	awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-{{ DIST }}-mm/etc/shadow- > /tmp/debian-{{ DIST }}-mm/etc/shadow-.bak
-	esac
+	cat /tmp/debian-{{ DIST }}-mm/etc/shadow-.bak > /tmp/debian-{{ DIST }}-mm/etc/shadow-
+	rm /tmp/debian-{{ DIST }}-mm/etc/shadow-.bak
+else
+	echo no difference for /etc/shadow- on {{ DIST }} {{ VARIANT }} >&2
 fi
 
-# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
+# Because of unreproducible uids (#969631) we created the _apt user ourselves
-# also needed for users that are created by systemd-sysusers before systemd 252
+# and because passwd is not Essential:yes we didn't use useradd. But passwd
-# https://github.com/systemd/systemd/pull/24534
+# since 1:4.11.1+dfsg1-1 will create empty mail files, so we create it too.
-for f in shadow shadow-; do
+# https://bugs.debian.org/1004710
-	if [ ! -e /tmp/debian-{{ DIST }}-mm/etc/$f ]; then
+if [ {{ VARIANT }} = - ]; then
-		continue
+	if [ -e /tmp/debian-{{ DIST }}-debootstrap/var/mail/_apt ]; then
+		touch /tmp/debian-{{ DIST }}-mm/var/mail/_apt
+		chmod 660 /tmp/debian-{{ DIST }}-mm/var/mail/_apt
+		chown 100:8 /tmp/debian-{{ DIST }}-mm/var/mail/_apt
 	fi
-	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/etc/$f /tmp/debian-{{ DIST }}-mm/etc/$f >&2; then
-		echo patching /etc/$f on {{ DIST }} {{ VARIANT }} >&2
-		awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-{{ DIST }}-mm/etc/$f > /tmp/debian-{{ DIST }}-mm/etc/$f.bak
-		cat /tmp/debian-{{ DIST }}-mm/etc/$f.bak > /tmp/debian-{{ DIST }}-mm/etc/$f
-		rm /tmp/debian-{{ DIST }}-mm/etc/$f.bak
-	else
-		echo no difference for /etc/$f on {{ DIST }} {{ VARIANT }} >&2
 fi
-done
 
 # check if the file content differs
 diff --unified --no-dereference --recursive /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm >&2
@@ -206,13 +217,13 @@ find /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm -type d -print
 # debootstrap never ran apt -- fixing permissions
 for d in ./var/lib/apt/lists/partial ./var/cache/apt/archives/partial; do
 	chroot /tmp/debian-{{ DIST }}-debootstrap chmod 0700 $d
-	chroot /tmp/debian-{{ DIST }}-debootstrap chown $(id -u _apt):root $d
+	chroot /tmp/debian-{{ DIST }}-debootstrap chown _apt:root $d
 done
 tar -C /tmp/debian-{{ DIST }}-debootstrap --numeric-owner --sort=name --clamp-mtime --mtime=$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds) -cf /tmp/root1.tar .
 tar -C /tmp/debian-{{ DIST }}-mm --numeric-owner --sort=name --clamp-mtime --mtime=$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds) -cf /tmp/root2.tar .
 tar --full-time --verbose -tf /tmp/root1.tar > /tmp/root1.tar.list
 tar --full-time --verbose -tf /tmp/root2.tar > /tmp/root2.tar.list
-diff -u /tmp/root1.tar.list /tmp/root2.tar.list >&2
+diff -u /tmp/root1.tar.list /tmp/root2.tar.list
 rm /tmp/root1.tar /tmp/root2.tar /tmp/root1.tar.list /tmp/root2.tar.list
 
 # check if file properties (permissions, ownership, symlink names, modification time) differ

tests/check-for-bit-by-bit-identical-format-output

@@ -31,7 +31,4 @@ case {{ VARIANT }} in essential|apt|minbase|buildd)
 	rm /tmp/debian-chroot-fakechroot.{{ FORMAT }}
 	;;
 esac
-# we cannot test chrootless mode here, because mmdebstrap relies on the
-# usrmerge package to set up merged-/usr and that doesn't work in chrootless
-# mode
 rm /tmp/debian-chroot-root.{{ FORMAT }}

tests/chrootless

@@ -1,16 +0,0 @@
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
-# we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
-# DPKG_ROOT
-for INCLUDE in '' 'systemd-sysv'; do
-	for MODE in root chrootless; do
-		{{ CMD }} --mode=$MODE --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
-			${INCLUDE:+--include="$INCLUDE"} \
-			{{ DIST }} "/tmp/$MODE.tar" {{ MIRROR }}
-	done
-	cmp /tmp/root.tar /tmp/chrootless.tar
-	rm /tmp/chrootless.tar /tmp/root.tar
-done

tests/chrootless-essential

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	if [ ! -e /mmdebstrap-testenv ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
+		exit 1
+	fi
+	adduser --gecos user --disabled-password user
+fi
+prefix=
+[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+$prefix fakechroot fakeroot {{ CMD }} --mode=chrootless --variant=essential {{ DIST }} /dev/null {{ MIRROR }}

tests/chrootless-fakeroot

@@ -1,29 +0,0 @@
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	if [ ! -e /mmdebstrap-testenv ]; then
-		echo "this test modifies the system and should only be run inside a container" >&2
-		exit 1
-	fi
-	adduser --gecos user --disabled-password user
-fi
-prefix=
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
-# we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
-# DPKG_ROOT
-# permissions drwxr-sr-x and extended attributes of ./var/log/journal/ cannot
-# be preserved under fakeroot
-for INCLUDE in '' 'systemd-sysv'; do
-	{{ CMD }} --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
-		--customize-hook='if [ -d "$1"/var/log/journal ]; then rmdir "$1"/var/log/journal; mkdir --mode=2755 "$1"/var/log/journal; chroot "$1" chown root:systemd-journal /var/log/journal; fi' \
-		${INCLUDE:+--include="$INCLUDE"} \
-		{{ DIST }} /tmp/root.tar {{ MIRROR }}
-	$prefix fakeroot {{ CMD }} --mode=chrootless --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
-		${INCLUDE:+--include="$INCLUDE"} \
-		{{ DIST }} /tmp/chrootless.tar {{ MIRROR }}
-	cmp /tmp/root.tar /tmp/chrootless.tar
-	rm /tmp/chrootless.tar /tmp/root.tar
-done

tests/chrootless-foreign

@@ -1,46 +0,0 @@
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
-if [ ! -e /mmdebstrap-testenv ]; then
-	echo "this test modifies the system and should only be run inside a container" >&2
-	exit 1
-fi
-
-[ "$(id -u)" -eq 0 ]
-[ -e /proc/sys/fs/binfmt_misc/qemu-aarch64 ]
-
-# we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
-# DPKG_ROOT
-for INCLUDE in '' 'systemd-sysv'; do
-	echo 1 > /proc/sys/fs/binfmt_misc/qemu-aarch64
-	arch-test arm64
-	{{ CMD }} --mode=root --architecture=arm64 --variant={{ VARIANT }} \
-		--hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
-		{{ DIST }} "/tmp/root.tar" {{ MIRROR }}
-	echo 0 > /proc/sys/fs/binfmt_misc/qemu-aarch64
-	! arch-test arm64
-	{{ CMD }} --mode=chrootless --architecture=arm64 --variant={{ VARIANT }} \
-		--hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
-		{{ DIST }} "/tmp/chrootless.tar" {{ MIRROR }}
-	# when creating a foreign architecture chroot, the tarballs are not
-	# bit-by-bit identical but contain a few remaining differences:
-	#
-	#  * /etc/ld.so.cache -- hard problem, must be solved in glibc upstream
-	#  * /var/lib/dpkg/triggers -- #990712
-	#  * /var/cache/debconf/*.dat-old -- needs investigation
-	for tar in root chrootless; do
-		<"/tmp/$tar.tar" \
-		./tarfilter \
-			--path-exclude=/var/cache/debconf/config.dat-old \
-			--path-exclude=/var/cache/debconf/templates.dat-old \
-			--path-exclude=/etc/ld.so.cache \
-			--path-exclude=/var/lib/dpkg/triggers/File \
-			--path-exclude=/var/lib/dpkg/triggers/ldconfig \
-			> "/tmp/$tar.tar.tmp"
-		mv "/tmp/$tar.tar.tmp" "/tmp/$tar.tar"
-	done
-	cmp /tmp/root.tar /tmp/chrootless.tar
-	rm /tmp/chrootless.tar /tmp/root.tar
-done

tests/compare-output-with-pre-seeded-var-cache-apt-archives

@@ -10,9 +10,6 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test requires the cache directory to be mounted on /mnt and should only be run inside a container" >&2
 	exit 1
 fi
-tmpdir=$(mktemp -d)
-trap "rm -f \"$tmpdir\"/*.deb /tmp/orig.tar /tmp/test1.tar /tmp/test2.tar; rmdir \"$tmpdir\"" EXIT INT TERM
-
 include="--include=doc-debian"
 if [ "{{ VARIANT }}" = "custom" ]; then
 	include="$include,base-files,base-passwd,coreutils,dash,diffutils,dpkg,libc-bin,sed"
@@ -21,24 +18,27 @@ fi
 	--setup-hook='mkdir -p "$1"/var/cache/apt/archives/partial' \
 	--setup-hook='touch "$1"/var/cache/apt/archives/lock' \
 	--setup-hook='chmod 0640 "$1"/var/cache/apt/archives/lock' \
-	{{ DIST }} - {{ MIRROR }} > /tmp/orig.tar
+	{{ DIST }} - {{ MIRROR }} > orig.tar
 # somehow, when trying to create a tarball from the 9p mount, tar throws the
 # following error: tar: ./doc-debian_6.4_all.deb: File shrank by 132942 bytes; padding with zeros
 # to reproduce, try: tar --directory /mnt/cache/debian/pool/main/d/doc-debian/ --create --file - . | tar --directory /tmp/ --extract --file -
 # this will be different:
 # md5sum /mnt/cache/debian/pool/main/d/doc-debian/*.deb /tmp/*.deb
 # another reason to copy the files into a new directory is, that we can use shell globs
+tmpdir=$(mktemp -d)
 cp /mnt/cache/debian/pool/main/b/busybox/busybox_*"_{{ HOSTARCH }}.deb" /mnt/cache/debian/pool/main/a/apt/apt_*"_{{ HOSTARCH }}.deb" "$tmpdir"
 {{ CMD }} $include --mode={{ MODE }} --variant={{ VARIANT }} \
 	--setup-hook='mkdir -p "$1"/var/cache/apt/archives/partial' \
 	--setup-hook='sync-in "'"$tmpdir"'" /var/cache/apt/archives/partial' \
-	{{ DIST }} - {{ MIRROR }} > /tmp/test1.tar
+	{{ DIST }} - {{ MIRROR }} > test1.tar
-cmp /tmp/orig.tar /tmp/test1.tar
+cmp orig.tar test1.tar
 {{ CMD }} $include --mode={{ MODE }} --variant={{ VARIANT }} \
 	--customize-hook='touch "$1"/var/cache/apt/archives/partial' \
 	--setup-hook='mkdir -p "$1"/var/cache/apt/archives/' \
 	--setup-hook='sync-in "'"$tmpdir"'" /var/cache/apt/archives/' \
 	--setup-hook='chmod 0755 "$1"/var/cache/apt/archives/' \
 	--customize-hook='find "'"$tmpdir"'" -type f -exec md5sum "{}" \; | sed "s|"'"$tmpdir"'"|$1/var/cache/apt/archives|" | md5sum --check' \
-	{{ DIST }} - {{ MIRROR }} > /tmp/test2.tar
+	{{ DIST }} - {{ MIRROR }} > test2.tar
-cmp /tmp/orig.tar /tmp/test2.tar
+cmp orig.tar test2.tar
+rm "$tmpdir"/*.deb orig.tar test1.tar test2.tar
+rmdir "$tmpdir"

tests/create-arm64-tarball

@@ -22,22 +22,8 @@ $prefix {{ CMD }} --mode={{ MODE }} --variant=apt --architectures=arm64 {{ DIST
 # we ignore differences between architectures by ignoring some files
 # and renaming others
 { tar -tf /tmp/debian-chroot.tar \
-	| grep -v '^\./usr/lib/ld-linux-aarch64\.so\.1$' \
+	| grep -v '^\./lib/ld-linux-aarch64\.so\.1$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/ld-linux-aarch64\.so\.1$' \
+	| grep -v '^\./lib/aarch64-linux-gnu/ld-linux-aarch64\.so\.1$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/int-ll64\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/types\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/unistd\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/sigcontext\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/sve_context\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/types\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-extra\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-id\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-prregset\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/gnu/stubs-lp64\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/linux/types\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/sys/procfs\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/sys/user\.ph$' \
 	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.arm64\.gz$' \
 	| sed 's/aarch64-linux-gnu/x86_64-linux-gnu/' \
 	| sed 's/arm64/amd64/';
@@ -45,24 +31,13 @@ $prefix {{ CMD }} --mode={{ MODE }} --variant=apt --architectures=arm64 {{ DIST
 { cat tar1.txt \
 	| grep -v '^\./usr/bin/i386$' \
 	| grep -v '^\./usr/bin/x86_64$' \
-	| grep -v '^\./lib32$' \
+	| grep -v '^\./lib64/$' \
-	| grep -v '^\./lib64$' \
+	| grep -v '^\./lib64/ld-linux-x86-64\.so\.2$' \
-	| grep -v '^\./libx32$' \
+	| grep -v '^\./lib/x86_64-linux-gnu/ld-linux-x86-64\.so\.2$' \
-	| grep -v '^\./usr/lib32/$' \
+	| grep -v '^\./lib/x86_64-linux-gnu/libmvec-2\.[0-9]\+\.so$' \
-	| grep -v '^\./usr/libx32/$' \
+	| grep -v '^\./lib/x86_64-linux-gnu/libmvec\.so\.1$' \
-	| grep -v '^\./usr/lib64/$' \
-	| grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/ld-linux-x86-64\.so\.2$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/libmvec\.so\.1$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_32\.ph$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_64\.ph$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_x32\.ph$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_32\.ph$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_64\.ph$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_x32\.ph$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/gnu/stubs-64\.ph$' \
 	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.amd64\.gz$' \
 	| grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
 	| grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$';
-} | sort | diff -u - tar2.txt >&2
+} | sort | diff -u - tar2.txt
 rm /tmp/debian-chroot.tar

tests/create-tarball-with-tmp-mounted-nodev

@@ -5,7 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-mount -t tmpfs -o nodev,nosuid,size=400M tmpfs /tmp
+mount -t tmpfs -o nodev,nosuid,size=300M tmpfs /tmp
 # use --customize-hook to exercise the mounting/unmounting code of block devices in root mode
 {{ CMD }} --mode=root --variant=apt --customize-hook='mount | grep /dev/full' --customize-hook='test "$(echo foo | tee /dev/full 2>&1 1>/dev/null)" = "tee: /dev/full: No space left on device"' {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/i386-which-can-be-executed-without-qemu

@@ -12,29 +12,27 @@ apt-get remove --yes qemu-user-static binfmt-support qemu-user
 # and renaming others
 { tar -tf /tmp/debian-chroot.tar \
 	| grep -v '^\./usr/bin/i386$' \
-	| grep -v '^\./usr/lib/ld-linux\.so\.2$' \
+	| grep -v '^\./lib/ld-linux\.so\.2$' \
-	| grep -v '^\./usr/lib/i386-linux-gnu/ld-linux\.so\.2$' \
+	| grep -v '^\./lib/i386-linux-gnu/ld-linux\.so\.2$' \
 	| grep -v '^\./usr/lib/gcc/i686-linux-gnu/$' \
 	| grep -v '^\./usr/lib/gcc/i686-linux-gnu/[0-9]\+/$' \
 	| grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
 	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.i386\.gz$' \
 	| sed 's/i386-linux-gnu/x86_64-linux-gnu/' \
-	| sed 's/i386/amd64/' \
+	| sed 's/i386/amd64/';
-	| sed 's/\/stubs-32.ph$/\/stubs-64.ph/';
 } | sort > tar2.txt
 { cat tar1.txt \
 	| grep -v '^\./usr/bin/i386$' \
 	| grep -v '^\./usr/bin/x86_64$' \
-	| grep -v '^\./usr/lib32/$' \
-	| grep -v '^\./lib32$' \
 	| grep -v '^\./lib64/$' \
-	| grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
+	| grep -v '^\./lib64/ld-linux-x86-64\.so\.2$' \
 	| grep -v '^\./usr/lib/gcc/x86_64-linux-gnu/$' \
 	| grep -v '^\./usr/lib/gcc/x86_64-linux-gnu/[0-9]\+/$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/ld-linux-x86-64\.so\.2$' \
+	| grep -v '^\./lib/x86_64-linux-gnu/ld-linux-x86-64\.so\.2$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/libmvec\.so\.1$' \
+	| grep -v '^\./lib/x86_64-linux-gnu/libmvec-2\.[0-9]\+\.so$' \
+	| grep -v '^\./lib/x86_64-linux-gnu/libmvec\.so\.1$' \
 	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.amd64\.gz$' \
 	| grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
 	| grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$';
-} | sort | diff -u - tar2.txt >&2
+} | sort | diff -u - tar2.txt
 rm /tmp/debian-chroot.tar

tests/tarfilter-idshift

@@ -18,12 +18,11 @@ runuser -u user -- {{ CMD }} --mode=unshare --variant=apt --include=iputils-ping
 	| ./tarfilter --idshift 0 > /tmp/debian-chroot.tar
 # make sure that xattrs are set in the original tarball
 mkdir /tmp/debian-chroot
-tar --xattrs --xattrs-include='*' --directory /tmp/debian-chroot -xf /tmp/debian-chroot.tar ./usr/bin/ping
+tar --xattrs --xattrs-include='*' --directory /tmp/debian-chroot -xf /tmp/debian-chroot.tar ./bin/ping
-echo "/tmp/debian-chroot/usr/bin/ping cap_net_raw=ep" > /tmp/expected
+echo "/tmp/debian-chroot/bin/ping cap_net_raw=ep" > /tmp/expected
-getcap /tmp/debian-chroot/usr/bin/ping | diff -u /tmp/expected - >&2
+getcap /tmp/debian-chroot/bin/ping | diff -u /tmp/expected -
-rm /tmp/debian-chroot/usr/bin/ping
+rm /tmp/debian-chroot/bin/ping
-rmdir /tmp/debian-chroot/usr/bin
+rmdir /tmp/debian-chroot/bin
-rmdir /tmp/debian-chroot/usr
 rmdir /tmp/debian-chroot
 # shift the uid/gid forward by 100000 and backward by 100000
 ./tarfilter --idshift 100000 < /tmp/debian-chroot.tar > /tmp/debian-chroot-shifted.tar
@@ -33,7 +32,8 @@ rmdir /tmp/debian-chroot
 cmp /tmp/debian-chroot.tar /tmp/debian-chroot-shiftedback.tar
 # manually adjust uid/gid and compare "tar -t" output
 tar --numeric-owner -tvf /tmp/debian-chroot.tar \
-	| sed 's# 42/0 # 100042/100000 #' \
+	| sed 's# 100/0 # 100100/100000 #' \
+	| sed 's# 100/8 # 100100/100008 #' \
 	| sed 's# 0/0 # 100000/100000 #' \
 	| sed 's# 0/5 # 100000/100005 #' \
 	| sed 's# 0/8 # 100000/100008 #' \
@@ -44,16 +44,16 @@ tar --numeric-owner -tvf /tmp/debian-chroot.tar \
 	> /tmp/debian-chroot.txt
 tar --numeric-owner -tvf /tmp/debian-chroot-shifted.tar \
 	| sed 's/ \+/ /g' \
-	| diff -u /tmp/debian-chroot.txt - >&2
+	| diff -u /tmp/debian-chroot.txt -
 mkdir /tmp/debian-chroot
 tar --xattrs --xattrs-include='*' --directory /tmp/debian-chroot -xf /tmp/debian-chroot-shifted.tar
 echo "100000 100000" > /tmp/expected
-stat --format="%u %g" /tmp/debian-chroot/usr/bin/ping | diff -u /tmp/expected - >&2
+stat --format="%u %g" /tmp/debian-chroot/bin/ping | diff -u /tmp/expected -
-echo "/tmp/debian-chroot/usr/bin/ping cap_net_raw=ep" > /tmp/expected
+echo "/tmp/debian-chroot/bin/ping cap_net_raw=ep" > /tmp/expected
-getcap /tmp/debian-chroot/usr/bin/ping | diff -u /tmp/expected - >&2
+getcap /tmp/debian-chroot/bin/ping | diff -u /tmp/expected -
 echo "0 0" > /tmp/expected
-runuser -u user -- {{ CMD }} --unshare-helper /usr/sbin/chroot /tmp/debian-chroot stat --format="%u %g" /usr/bin/ping \
+runuser -u user -- {{ CMD }} --unshare-helper /usr/sbin/chroot /tmp/debian-chroot stat --format="%u %g" /bin/ping \
-	| diff -u /tmp/expected - >&2
+	| diff -u /tmp/expected -
-echo "/usr/bin/ping cap_net_raw=ep" > /tmp/expected
+echo "/bin/ping cap_net_raw=ep" > /tmp/expected
-runuser -u user -- {{ CMD }} --unshare-helper /usr/sbin/chroot /tmp/debian-chroot getcap /usr/bin/ping \
+runuser -u user -- {{ CMD }} --unshare-helper /usr/sbin/chroot /tmp/debian-chroot getcap /bin/ping \
-	| diff -u /tmp/expected - >&2
+	| diff -u /tmp/expected -