CHANGELOG.md

@@ -1,52 +1,3 @@
-1.5.4 (2024-10-28)
-------------------
-
- - do not generate apt sources.list entry if SUITE is empty
-
-1.5.3 (2024-09-13)
-------------------
-
- - tidy up any zombie processes
- - chrootless hurd-i386
- - add --skip=cleanup/reproducible/machine-id
- - m-a-b-q: replace test_installed by dpkg-checkbuilddeps
-
-1.5.2 (2024-06-26)
-------------------
-
- - mmdebstrap-autopkgtest-build-qemu produces bit-by-bit reproducible output
-
-1.5.1 (2024-06-03)
-------------------
-
- - in root and unshare mode, run 'mount --make-rprivate /' before bind-mounting
- - switch apt variant from using 'apt-get dist-upgrade' to apt patterns
-
-1.5.0 (2024-05-14)
-------------------
-
- - add --format=ext4
-
-1.4.3 (2024-02-01)
-------------------
-
- - take hard links into account when computing disk usage
-
-1.4.2 (2024-01-29)
-------------------
-
- - allow for start-stop-daemon to be in either /sbin or /usr/sbin
- - mmdebstrap-autopkgtest-build-qemu: fix octal mode computation and hostname
-
-1.4.1 (2024-01-09)
-------------------
-
- - set DPkg::Chroot-Directory in APT_CONFIG to simplify calling apt in hooks
- - disallow running chrootless as root without fakeroot unless
-   --skip=check/chrootless is used
- - only print short --help output if wrong args are passed
- - read files passed as --aptopt and --dpkgopt outside the unshared namespace
-
 1.4.0 (2023-10-24)
 ------------------
 

README.md

@@ -158,19 +158,15 @@ Contributors
 
  - Johannes Schauer Marin Rodrigues (main author)
  - Helmut Grohne
- - Jochen Sprickerhof
  - Gioele Barabucci
  - Benjamin Drung
+ - Jochen Sprickerhof
  - Josh Triplett
  - Konstantin Demin
- - Chris Hofstaedtler
- - Colin Watson
  - David Kalnischkies
  - Emilio Pozuelo Monfort
- - Francesco Poli
  - Jakub Wilk
  - Joe Groocock
- - Max-Julian Pogner
  - Nicolas Vigier
  - Raul Tambre
  - Steve Dodd

coverage.py

@@ -34,7 +34,7 @@ all_variants = [
     "standard",
 ]
 default_format = "auto"
-all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "ext4", "null"]
+all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "null"]
 
 mirror = os.getenv("mirror", "http://127.0.0.1/debian")
 hostarch = subprocess.check_output(["dpkg", "--print-architecture"]).decode().strip()
@@ -362,21 +362,6 @@ def main():
             check=False,
             stdout=subprocess.PIPE,
         ).stdout.decode()
-        shfmt = subprocess.run(
-            [
-                "shfmt",
-                "--posix",
-                "--binary-next-line",
-                "--case-indent",
-                "--indent",
-                "2",
-                "--simplify",
-                "-d",
-                "shared/test.sh",
-            ],
-            check=False,
-            stdout=subprocess.PIPE,
-        ).stdout.decode()
         argv = None
         match test:
             case "qemu":
@@ -426,11 +411,9 @@ def main():
         acc_time_per_test[name].append(walltime)
         print(separator, file=sys.stderr)
         print(f"duration: {walltime}", file=sys.stderr)
-        if proc.returncode != 0 or shellcheck != "" or shfmt != "":
+        if proc.returncode != 0 or shellcheck != "":
             if shellcheck != "":
                 print(shellcheck)
-            if shfmt != "":
-                print(shfmt)
             failed.append(formated_test_name)
             print("result: FAILURE", file=sys.stderr)
         else:

coverage.sh

@@ -2,22 +2,11 @@
 
 set -eu
 
-# by default, use the mmdebstrap executable in the current directory together
+if [ -e ./mmdebstrap ]; then
-# with perl Devel::Cover but allow to overwrite this
-: "${CMD:=perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap}"
-
-case "$CMD" in
-  "mmdebstrap "* | mmdebstrap | *" mmdebstrap" | *" mmdebstrap "*)
-    MMSCRIPT="$(command -v mmdebstrap 2>/dev/null)"
-    ;;
-  *) MMSCRIPT=./mmdebstrap ;;
-esac
-
-if [ -e "$MMSCRIPT" ]; then
 	TMPFILE=$(mktemp)
-  perltidy <"$MMSCRIPT" >"$TMPFILE"
+	perltidy < ./mmdebstrap > "$TMPFILE"
 	ret=0
-  diff -u "$MMSCRIPT" "$TMPFILE" || ret=$?
+	diff -u ./mmdebstrap "$TMPFILE" || ret=$?
 	if [ "$ret" -ne 0 ]; then
 		echo "perltidy failed" >&2
 		rm "$TMPFILE"
@@ -25,14 +14,12 @@ if [ -e "$MMSCRIPT" ]; then
 	fi
 	rm "$TMPFILE"
 
-  if [ "$(sed -e '/^__END__$/,$d' "$MMSCRIPT" | wc --max-line-length)" -gt 79 ]; then
+	if [ "$(sed -e '/^__END__$/,$d' ./mmdebstrap | wc --max-line-length)" -gt 79 ]; then
 		echo "exceeded maximum line length of 79 characters" >&2
 		exit 1
 	fi
 
-  perlcritic --severity 4 --verbose 8 "$MMSCRIPT"
+	perlcritic --severity 4 --verbose 8 ./mmdebstrap
-
-  pod2man "$MMSCRIPT" >/dev/null
 fi
 
 for f in tarfilter coverage.py caching_proxy.py; do
@@ -40,9 +27,7 @@ for f in tarfilter coverage.py caching_proxy.py; do
 	black --check "./$f"
 done
 
-shellcheck --exclude=SC2016 coverage.sh make_mirror.sh run_null.sh run_qemu.sh gpgvnoexpkeysig mmdebstrap-autopkgtest-build-qemu hooks/*/*.sh
+shellcheck --exclude=SC2016 coverage.sh make_mirror.sh run_null.sh run_qemu.sh gpgvnoexpkeysig hooks/*/*.sh
-
-shfmt --binary-next-line --case-indent --indent 2 --simplify -d coverage.sh make_mirror.sh run_null.sh run_qemu.sh mmdebstrap-autopkgtest-build-qemu gpgvnoexpkeysig
 
 mirrordir="./shared/cache/debian"
 
@@ -78,6 +63,9 @@ export LC_ALL=C.UTF-8
 
 : "${HAVE_BINFMT:=yes}"
 
+# by default, use the mmdebstrap executable in the current directory together
+# with perl Devel::Cover but allow to overwrite this
+: "${CMD:=perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap}"
 mirror="http://127.0.0.1/debian"
 
 export HAVE_QEMU HAVE_BINFMT RUN_MA_SAME_TESTS DEFAULT_DIST SOURCE_DATE_EPOCH CMD mirror
@@ -108,8 +96,8 @@ END
 fi
 
 # check if the wiki has to be updated with pod2markdown output
-if [ "${DEBEMAIL-}" = "josch@debian.org" ]; then
+if [ "${DEBEMAIL:-}" = "josch@debian.org" ]; then
-  bash -exc "diff -u <(curl --silent https://gitlab.mister-muffin.de/josch/mmdebstrap/wiki/raw/Home | dos2unix; echo) <(pod2markdown < mmdebstrap)" || :
+	bash -exc "diff -u <(curl --silent https://gitlab.mister-muffin.de/josch/mmdebstrap/wiki/raw/Home | dos2unix) <(pod2markdown < mmdebstrap)" || :
 fi
 
 rm -f shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/tarfilter shared/proxysolver

coverage.txt

@@ -59,21 +59,21 @@ Needs-QEMU: true
 Test: mmdebstrap
 Needs-Root: true
 Modes: root
-Formats: tar squashfs ext2 ext4
+Formats: tar squashfs ext2
 Variants: essential apt minbase buildd - standard
 Skip-If:
  variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
  mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
- variant == "standard" and dist in ["oldstable", "stable"] and hostarch in ["armel", "armhf", "mipsel"] # #1031276
+ variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
 Test: check-for-bit-by-bit-identical-format-output
 Modes: unshare fakechroot
-Formats: tar squashfs ext2 ext4
+Formats: tar squashfs ext2
 Variants: essential apt minbase buildd - standard
 Skip-If:
  variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
  mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
- variant == "standard" and dist in ["oldstable", "stable"] and hostarch in ["armel", "armhf", "mipsel"] # #1031276
+ variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
 Test: tarfilter-idshift
 Needs-QEMU: true
@@ -194,18 +194,18 @@ Skip-If:
  hostarch != "amd64"
  not run_ma_same_tests
 
-Test: include-foreign-libmagic-mgc
+Test: include-libmagic-mgc-arm64
 Needs-Root: true
 Needs-APT-Config: true
 Skip-If:
- hostarch not in ["amd64", "arm64"]
+ hostarch != "amd64"
  not run_ma_same_tests
 
-Test: include-foreign-libmagic-mgc-with-multiple-arch-options
+Test: include-libmagic-mgc-arm64-with-multiple-arch-options
 Needs-Root: true
 Needs-APT-Config: true
 Skip-If:
- hostarch not in ["amd64", "arm64"]
+ hostarch != "amd64"
  not run_ma_same_tests
 
 Test: aptopt
@@ -273,13 +273,17 @@ Test: debootstrap-no-op-options
 Needs-Root: true
 
 Test: verbose
-Variants: standard
+Variants: - standard
 Skip-If:
+ variant == "-" and hostarch not in ["armel", "armhf", "mipsel"] # #1031276
+ variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
  variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
 
 Test: debug
-Variants: standard
+Variants: - standard
 Skip-If:
+ variant == "-" and hostarch not in ["armel", "armhf", "mipsel"] # #1031276
+ variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
  variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
 
 Test: quiet
@@ -361,22 +365,21 @@ Variants: custom
 Modes: chrootless
 Needs-APT-Config: true
 
-Test: install-libmagic-mgc-on-foreign
+Test: install-libmagic-mgc-on-arm64
 Variants: custom
 Modes: chrootless
 Skip-If:
- hostarch not in ["amd64", "arm64"]
+ hostarch != "amd64"
  not have_binfmt
 
 Test: install-busybox-based-sub-essential-system
 Needs-Root: true
 
-Test: create-foreign-tarball
+Test: create-arm64-tarball
 Modes: root unshare fakechroot
 Skip-If:
- hostarch not in ["amd64", "arm64"]
+ hostarch != "amd64"
  mode == "fakechroot" and not run_ma_same_tests
- mode == "fakechroot" and hostarch == "arm64" # usrmerge postinst under fakechroot wants to copy /lib/ld-linux-x86-64.so.2 (which does not exist) instead of /lib64/ld-linux-x86-64.so.2
  not have_binfmt
 
 Test: no-sbin-in-path
@@ -430,8 +433,3 @@ Modes: root unshare
 
 Test: skip-tar-in-mknod
 Modes: unshare
-
-Test: zombie-reaping
-Modes: unshare
-
-Test: empty-suite

hooks/copy-host-apt-sources-and-preferences/setup00.sh

@@ -1,13 +1,4 @@
 #!/bin/sh
-#
-# This script makes sure that the apt sources.list and preferences from outside
-# the chroot also exist inside the chroot by *appending* them to any existing
-# files. If you do not want to keep the original content, add another setup
-# hook before this one which cleans up the files you don't want to keep.
-#
-# If instead of copying sources.list verbatim you want to mangle its contents,
-# consider using python-apt for that. An example can be found in the Debian
-# packaging of mmdebstrap in ./debian/tests/sourcesfilter
 
 set -eu
 

hooks/file-mirror-automount/setup00.sh

@@ -15,7 +15,7 @@ env APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get indextargets --no-release-info -
 	| while read -r path; do
 		mkdir -p "$rootdir/run/mmdebstrap"
 		if [ ! -d "/$path" ]; then
-			echo "W: /$path is not an existing directory" >&2
+			echo "/$path is not an existing directory" >&2
 			continue
 		fi
 		case $MMDEBSTRAP_MODE in

hooks/maybe-merged-usr/essential00.sh

@@ -15,10 +15,6 @@ case "$ver" in
 		echo "usr-is-merged package from src:usrmerge installed -- not running merged-usr essential hook" >&2
 		exit 0
 		;;
-	'not-installed  ')
-		echo "usr-is-merged was not installed in a previous hook -- not running merged-usr essential hook" >&2
-		exit 0
-		;;
 	*)
 		echo "unexpected situation for package usr-is-merged: $ver" >&2
 		exit 1

hooks/maybe-merged-usr/extract00.sh

@@ -4,22 +4,12 @@ set -eu
 
 env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get update --error-on=any
 
-if env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
-	# if apt-cache exited successfully, then usr-is-merged exists either as
-	# a real or virtual package
-	if env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged 2>/dev/null | grep -q "Package: usr-is-merged"; then
-		echo "usr-is-merged found -- running merged-usr extract hook" >&2
-	else
-		# The usr-is-merged must be virtual, so assume that nothing
-		# has to be done. This is the case with Debian Trixie or later
-		# or with Ubuntu Lunar or later
-		echo "usr-is-merged found but not real -- not running merged-usr extract hook" >&2
-		exit 0
-	fi
-else
 # if the usr-is-merged package cannot be installed with apt, do nothing
-	echo "no package providing usr-is-merged found -- not running merged-usr extract hook" >&2
+if ! env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
+	echo "no package called usr-is-merged found -- not running merged-usr extract hook" >&2
 	exit 0
+else
+	echo "package usr-is-merged found -- running merged-usr extract hook" >&2
 fi
 
 # resolve the script path using several methods in order:

hooks/maybe-merged-usr/setup00.sh

@@ -4,22 +4,12 @@ set -eu
 
 env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get update --error-on=any
 
-if env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
-	# if apt-cache exited successfully, then usr-is-merged exists either as
-	# a real or virtual package
-	if env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged 2>/dev/null | grep -q "Package: usr-is-merged"; then
-		echo "usr-is-merged found -- running merged-usr setup hook" >&2
-	else
-		# The usr-is-merged must be virtual, so assume that nothing
-		# has to be done. This is the case with Debian Trixie or later
-		# or with Ubuntu Lunar or later
-		echo "usr-is-merged found but not real -- not running merged-usr setup hook" >&2
-		exit 0
-	fi
-else
 # if the usr-is-merged package cannot be installed with apt, do nothing
-	echo "no package providing usr-is-merged found -- not running merged-usr setup hook" >&2
+if ! env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
+	echo "no package called usr-is-merged found -- not running merged-usr setup hook" >&2
 	exit 0
+else
+	echo "package usr-is-merged found -- running merged-usr setup hook" >&2
 fi
 
 # resolve the script path using several methods in order:

make_mirror.sh

@@ -33,7 +33,7 @@ deletecache() {
 		done
 		# deleting artifacts from test "mmdebstrap"
 		for variant in essential apt minbase buildd - standard; do
-      for format in tar ext2 ext4 squashfs; do
+			for format in tar ext2 squashfs; do
 				if [ -e "$dir/mmdebstrap-$dist-$variant.$format" ]; then
 					# attempt to delete for all dists because DEFAULT_DIST might've been different the last time
 					rm "$dir/mmdebstrap-$dist-$variant.$format"
@@ -236,11 +236,7 @@ END
 	esac
 
 	# shellcheck disable=SC2086
-  APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs \
+	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs
-    || APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install \
-      -oDebug::pkgProblemResolver=true -oDebug::pkgDepCache::Marker=1 \
-      -oDebug::pkgDepCache::AutoInstall=1 \
-      $pkgs
 
 	rm "$rootdir/var/cache/apt/archives/lock"
 	rmdir "$rootdir/var/cache/apt/archives/partial"
@@ -322,14 +318,8 @@ if [ "$FORCE_UPDATE" != "yes" ] && [ -e "$oldmirrordir/dists/$DEFAULT_DIST/InRel
 	http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/InRelease")
 	case "$http_code" in
 		200) ;; # need update
-    304)
+		304) echo up-to-date; exit 0;;
-      echo up-to-date
+		*) echo "unexpected status: $http_code"; exit 1;;
-      exit 0
-      ;;
-    *)
-      echo "unexpected status: $http_code"
-      exit 1
-      ;;
 	esac
 fi
 
@@ -459,7 +449,7 @@ if [ "$HAVE_QEMU" = "yes" ]; then
 	tmpdir="$(mktemp -d)"
 	trap 'kill "$PROXYPID" || :;cleanuptmpdir; cleanup_newcachedir' EXIT INT TERM
 
-  pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-binfmt,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,squashfs-tools-ng,genext2fs,linux-image-generic,passwd,e2fsprogs,uuid-runtime
+	pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,squashfs-tools-ng,genext2fs,linux-image-generic
 	if [ ! -e ./mmdebstrap ]; then
 		pkgs="$pkgs,mmdebstrap"
 	fi
@@ -503,8 +493,6 @@ mount -t 9p -o trans=virtio,access=any,msize=128k mmdebstrap /mnt
 # need to restart mini-httpd because we mounted different content into www-root
 systemctl restart mini-httpd
 
-ip link set enp0s1 down || :
-
 handler () {
 	while IFS= read -r line || [ -n "$line" ]; do
 		printf "%s %s: %s\n" "$(date -u -d "0 $(date +%s.%3N) seconds - $2 seconds" +"%T.%3N")" "$1" "$line"
@@ -542,8 +530,7 @@ END
 	fi
 	# set PATH to pick up the correct mmdebstrap variant
 	env PATH="$(dirname "$(realpath --canonicalize-existing "$CMD")"):$PATH" \
-    debvm-create --skip=usrmerge,systemdnetwork \
+		debvm-create --skip=usrmerge --size="$DISK_SIZE" --release="$DEFAULT_DIST" \
-    --size="$DISK_SIZE" --release="$DEFAULT_DIST" \
 		--output="$newcachedir/debian-$DEFAULT_DIST.ext4" -- \
 		--architectures="$arches" --include="$pkgs" \
 		--setup-hook='echo "Acquire::http::Proxy \"http://127.0.0.1:8080/\";" > "$1/etc/apt/apt.conf.d/00proxy"' \
@@ -554,9 +541,8 @@ END
 		--customize-hook='touch "$1/mmdebstrap-testenv"' \
 		--customize-hook='copy-in "'"$tmpdir"'/mmdebstrap.service" /etc/systemd/system/' \
 		--customize-hook='copy-in "'"$tmpdir"'/worker.sh" /' \
-    --customize-hook='echo 127.0.0.1 localhost > "$1/etc/hosts"' \
+		--customize-hook='printf 127.0.0.1 localhost > "$1/etc/hosts"' \
 		--customize-hook='printf "START=1\nDAEMON_OPTS=\"-h 127.0.0.1 -p 80 -u nobody -dd /mnt/cache -i /var/run/mini-httpd.pid -T UTF-8\"\n" > "$1/etc/default/mini-httpd"' \
-    --customize-hook='touch "$1/etc/systemd/system/tmp.mount"' \
 		"$mirror"
 
 	kill $PROXYPID

mmdebstrap-autopkgtest-build-qemu

@@ -29,11 +29,8 @@ B<mmdebstrap-autopkgtest-build-qemu> [I<OPTIONS>] B<--boot>=B<efi> I<RELEASE> I<
 B<mmdebstrap-autopkgtest-build-qemu> is a mostly compatible drop-in replacement
 for L<autopkgtest-build-qemu(1)> with two main differences: Firstly, it uses
 L<mmdebstrap(1)> instead of L<vmdb2(1)> and thus is able to create QEMU disk
-images without requiring superuser privileges and with bit-by-bit reproducible
+images without requiring superuser privileges.  Secondly, it uses
-output. Secondly, it uses L<systemd-boot(7)> and thus only supports booting via
+L<systemd-boot(7)> and thus only supports booting via EFI.
-EFI. For architectures for which L<autopkgtest-virt-qemu(1)> does not default
-to EFI booting you must pass B<--boot=efi> when invoking the autopkgtest virt
-backend.
 
 =head1 POSITIONAL PARAMETERS
 
@@ -99,23 +96,9 @@ Passes an additional B<--keyring> parameter to B<mmdebstrap>.
 
 =head1 EXAMPLES
 
-Make sure, that F</path/to/debian-unstable.img> is a path that the unshared
+    $ mmdebstrap-autopkgtest-build-qemu --boot=efi stable /path/to/debian-stable-i386.img i386
-user has access to. This can be done by ensuring world-execute permissions on
-all path components or by creating the image in a world-readable directory like
-/tmp before copying it into its final location.
 
-    $ mmdebstrap-autopkgtest-build-qemu --boot=efi --arch=amd64 unstable /path/to/debian-unstable.img
+    $ mmdebstrap-autopkgtest-build-qemu --boot=efi unstable /path/to/debian-unstable.img
-    [...]
-    $ autopkgtest mypackage -- qemu --boot=efi --dpkg-architecture=amd64 /path/to/debian-unstable.img
-
-Make sure to add B<--boot=efi> to both the B<mmdebstrap-autopkgtest-build-qemu>
-as well as the B<autopkgtest-virt-qemu> invocation.
-
-Create bit-by-bit reproducible images from a given snapshot.d.o timestamp.
-
-    SOURCE_DATE_EPOCH=1612543740 mmdebstrap-autopkgtest-build-qemu --boot=efi \
-        --mirror=http://snapshot.debian.org/archive/debian/20210205T164900Z/ \
-        unstable /path/to/debian-unstable.img
 
 =head1 SEE ALSO
 
@@ -234,66 +217,43 @@ while test "$#" -gt 0; do
 done
 
 test -z "$RELEASE" -o -z "$IMAGE" && usage_error "missing positional arguments"
-test "$BOOT" = efi \
+test "$BOOT" = efi ||
-  || die "this tool does not support boot modes other than efi"
+	die "this tool does not support boot modes other than efi"
 
 case "$ARCHITECTURE" in
 	amd64)
 		EFIIMG=bootx64.efi
-    QEMUARCH=x86_64
-    VMFPKG=ovmf
-    LINUXIMAGE=linux-image-amd64
 	;;
 	arm64)
 		EFIIMG=bootaa64.efi
-    QEMUARCH=aarch64
-    VMFPKG=qemu-efi-aarch64
-    LINUXIMAGE=linux-image-arm64
 	;;
 	armhf)
 		EFIIMG=bootarm.efi
-    QEMUARCH=arm
-    VMFPKG=qemu-efi-arm
-    LINUXIMAGE=linux-image-armmp
 	;;
 	i386)
 		EFIIMG=bootia32.efi
-    QEMUARCH=i386
-    VMFPKG=ovmf-ia32
-    LINUXIMAGE=linux-image-686-pae
 	;;
 	riscv64)
 		EFIIMG=bootriscv64.efi
-    QEMUARCH=riscv64
-    VMFPKG=
-    LINUXIMAGE=linux-image-riscv64
 	;;
 	*)
-    die "unsupported architecture: $ARCHITECTURE"
+		die "unsupported architecture"
 	;;
 esac
 
 if test "$(dpkg-query -f '${db:Status-Status}' -W binutils-multiarch)" = installed; then
 	GNU_PREFIX=
-  BINUTILS=
 else
 	GNU_ARCHITECTURE="$(dpkg-architecture "-a$ARCHITECTURE" -qDEB_HOST_GNU_TYPE)"
 	GNU_PREFIX="$GNU_ARCHITECTURE-"
 	GNU_SUFFIX="-$(echo "$GNU_ARCHITECTURE" | tr _ -)"
-  BINUTILS=", binutils$GNU_SUFFIX | binutils-multiarch"
+	test "$(dpkg-query -f '${db:Status-Status}' -W "binutils$GNU_SUFFIX")" = installed ||
+		die "please install binutils$GNU_SUFFIX or binutils-multiarch"
 fi
-
+for pkg in autopkgtest dosfstools e2fsprogs fdisk mount mtools passwd "systemd-boot-efi:$ARCHITECTURE" uidmap; do
-arches=" $(dpkg --print-architecture) $(dpkg --print-foreign-architectures | tr '\n' ' ') "
+	test "$(dpkg-query -f '${db:Status-Status}' -W "$pkg")" = installed ||
-case $arches in
+		die "please install $pkg"
-  *" $ARCHITECTURE "*) : ;; # nothing to do
+done
-  *) die "enable $ARCHITECTURE by running: sudo dpkg --add-architecture $ARCHITECTURE && sudo apt update" ;;
-esac
-
-test "$(dpkg-query -f '${db:Status-Status}' -W "dpkg-dev")" = installed \
-  || die "please install dpkg-dev"
-
-dpkg-checkbuilddeps -d "autopkgtest, dosfstools, e2fsprogs, fdisk, mount, mtools, passwd, uidmap, libarchive13, systemd-boot-efi:$ARCHITECTURE $BINUTILS" /dev/null \
-  || die "please install the required packages listed above"
 
 BOOTSTUB="/usr/lib/systemd/boot/efi/linux${EFIIMG#boot}.stub"
 
@@ -310,39 +270,26 @@ WORKDIR=$(mktemp -d)
 FAT_OFFSET_SECTORS=$((1024*2))
 FAT_SIZE_SECTORS=$((1024*254))
 
-# The image is raw and not in qcow2 format because:
+# Make the image writeable to the first subgid. mmdebstrap will map this gid to
-#  - faster run-time as the "qemu-image convert" step is not needed
+# the root group. unshare instead will map the current gid to 0 and the first
-#  - image can be used independent of qemu tooling
+# subgid to 1. Therefore mmdebstrap will be able to write to the image.
-#  - modifying the image just with "mount" instead of requiring qemu-nbd
+rm -f "$IMAGE"
-#  - sparse images make the file just as small as with qcow2
+: >"$IMAGE"
-#  - trim support is more difficult on qcow2
+unshare -U -r --map-groups=auto chown 0:1 "$IMAGE"
-#  - snapshots and overlays work just as well with raw images
+chmod 0660 "$IMAGE"
-#  - users who prefer qcow2 get to choose to run it themselves with their own
-#    custom options like compression
 
 set -- \
 	--mode=unshare \
-  --format=tar \
 	--variant=important \
 	--architecture="$ARCHITECTURE"
 
-case $MIRROR in http://snapshot.debian.org/archive/* | https://snapshot.debian.org/archive/*)
+test "$RELEASE" = jessie &&
-  set -- "$@" --aptopt='Acquire::Check-Valid-Until "false"'
-  ;;
-esac
-
-EXT_FEATURES=
-if test "$RELEASE" = jessie; then
-  set -- "$@" --keyring=/usr/share/keyrings/debian-archive-removed-keys.gpg
-  set -- "$@" --aptopt='Apt::Key::gpgvcommand "/usr/libexec/mmdebstrap/gpgvnoexpkeysig"'
 	set -- "$@" --hook-dir=/usr/share/mmdebstrap/hooks/jessie-or-older
-  EXT_FEATURES="^metadata_csum,^metadata_csum_seed,^orphan_file"
-fi
 
 set -- "$@" \
-  "--include=init,$LINUXIMAGE,python3" \
+	"--include=init,linux-image-$ARCHITECTURE,python3" \
-  '--customize-hook=echo host >"$1/etc/hostname"' \
+	'--customize-hook=echo autopkgtestvm >"$1/etc/hostname"' \
-  '--customize-hook=echo 127.0.0.1 localhost host >"$1/etc/hosts"' \
+	'--customize-hook=echo 127.0.0.1 localhost autopkgtestvm >"$1/etc/hosts"' \
 	'--customize-hook=passwd --root "$1" --delete root' \
 	'--customize-hook=useradd --root "$1" --home-dir /home/user --create-home user' \
 	'--customize-hook=passwd --root "$1" --delete user' \
@@ -355,34 +302,26 @@ if test -n "$SCRIPT"; then
 		'--customize-hook=rm -f "$1/userscript"'
 fi
 
+EXT4_OFFSET_BYTES=$(( (FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS) * 512))
+EXT4_OPTIONS="offset=$EXT4_OFFSET_BYTES,assume_storage_prezeroed=1"
 set -- "$@" \
 	"--customize-hook=download vmlinuz '$WORKDIR/kernel'" \
 	"--customize-hook=download initrd.img '$WORKDIR/initrd'" \
+	'--customize-hook=mount --bind "$1" "$1/mnt"' \
+	'--customize-hook=mount --bind "$1/mnt/mnt" "$1/mnt/dev"' \
+	'--customize-hook=/sbin/mkfs.ext4 -d "$1/mnt" -L autopkgtestvm -E '"'$EXT4_OPTIONS' '$IMAGE' '$SIZE'" \
+	'--customize-hook=umount --lazy "$1/mnt"' \
 	"$RELEASE" \
-  -
+	/dev/null
 
 test -n "$MIRROR" && set -- "$@" "$MIRROR"
 test -n "$KEYRING" && set -- "$@" "--keyring=$KEYRING"
 
-echo "+ mmdebstrap $*" >&2
+echo "mmdebstrap $*"
-# https://github.com/koalaman/shellcheck/issues/2555
+mmdebstrap "$@" || die "mmdebstrap failed"
-# shellcheck disable=SC3040
+
-set -o pipefail
+unshare -U -r --map-groups=auto chown 0:0 "$IMAGE"
-mmdebstrap "$@" | {
+chmod "$(printf %o "$(( 0666 - 0$(umask) ))")" "$IMAGE"
-  set -- -t ext4 -L autopkgtestvm -d -
-  if test -n "$EXT_FEATURES"; then
-    set -- "$@" -O "$EXT_FEATURES"
-  fi
-  EXTOPTS="offset=$(((FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS) * 512))"
-  if test -n "${SOURCE_DATE_EPOCH-}"; then
-    uuid="$(uuidgen --sha1 --namespace="$(uuidgen --sha1 --namespace='@dns' --name mister-muffin.de)" --name "$SOURCE_DATE_EPOCH")"
-    set -- "$@" -U "$uuid"
-    EXTOPTS="$EXTOPTS,hash_seed=$uuid"
-  fi
-  set -- "$@" -E "$EXTOPTS" "$IMAGE" "$SIZE"
-  echo "+ mke2fs $*" >&2
-  /sbin/mke2fs "$@"
-}
 
 echo "root=LABEL=autopkgtestvm rw console=ttyS0" > "$WORKDIR/cmdline"
 
@@ -399,8 +338,8 @@ test "$RELEASE" = jessie -a "$((alignment))" -lt "$((1024 * 1024))" && {
 }
 lastoffset=0
 # shellcheck disable=SC2034  # unused variables serve documentation
-lastoffset="$("${GNU_PREFIX}objdump" -h "$BOOTSTUB" \
+lastoffset="$("${GNU_PREFIX}objdump" -h "$BOOTSTUB" |
-  | while read -r idx name size vma lma fileoff algn behind; do
+	while read -r idx name size vma lma fileoff algn behind; do
 		test -z "$behind" -a "${algn#"2**"}" != "$algn" || continue
 		offset=$(( 0x$vma + 0x$size ))
 		test "$offset" -gt "$lastoffset" || continue
@@ -448,13 +387,3 @@ start=$((FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS)), type=0FC63DAF-8483-4772-8E79-3
 EOF
 
 dd if="$WORKDIR/fat" of="$IMAGE" conv=notrunc,sparse bs=512 "seek=$FAT_OFFSET_SECTORS" status=none
-
-if test "$(dpkg --print-architecture)" != "$ARCHITECTURE" && test "$(dpkg-query -f '${db:Status-Status}' -W "qemu-system-$QEMUARCH")" != installed; then
-  echo "I: you might need to install a package providing qemu-system-$QEMUARCH to use this image with autopkgtest-virt-qemu" >&2
-fi
-if test -n "$VMFPKG" && test "$(dpkg-query -f '${db:Status-Status}' -W "$VMFPKG")" != installed; then
-  echo "I: you might need to install $VMFPKG to use this image with autopkgtest-virt-qemu" >&2
-fi
-
-echo "I: SUCCESS! Your new image can be found here: $IMAGE" >&2
-echo "I: Don't forget to pass --boot=efi when running autopkgtest-virt-qemu with this image" >&2

run_null.sh

@@ -26,22 +26,13 @@ done
 # - Read fd 3 and let the group exit with that value
 # - Redirect fd 4 to stdout
 ret=0
-{
+{ { { {
-  {
+        ret=0;
-    {
+        ( exec 3>&- 4>&-; env --chdir=./shared $SUDO sh -x ./test.sh 2>&1) || ret=$?;
-      {
+        echo $ret >&3;
-        ret=0
+      } | tee shared/output.txt >&4;
-        (
+    } 3>&1;
-          exec 3>&- 4>&-
+  } | { read -r xs; exit "$xs"; }
-          env --chdir=./shared $SUDO sh -x ./test.sh 2>&1
-        ) || ret=$?
-        echo $ret >&3
-      } | tee shared/output.txt >&4
-    } 3>&1
-  } | {
-    read -r xs
-    exit "$xs"
-  }
 } 4>&1 || ret=$?
 if [ "$ret" -ne 0 ]; then
 	echo "test.sh failed"

run_qemu.sh

@@ -4,13 +4,15 @@ set -eu
 
 : "${DEFAULT_DIST:=unstable}"
 : "${cachedir:=./shared/cache}"
-: "${MMDEBSTRAP_TESTS_DEBUG:=no}"
 tmpdir="$(mktemp -d)"
 
 cleanup() {
 	rv=$?
 	rm -f "$tmpdir/log"
 	[ -e "$tmpdir" ] && rmdir "$tmpdir"
+	if [ -n "${TAIL_PID:-}" ]; then
+		kill "$TAIL_PID"
+	fi
 	if [ -e shared/output.txt ]; then
 		res="$(cat shared/exitstatus.txt)"
 		if [ "$res" != "0" ]; then
@@ -28,44 +30,22 @@ if [ -e shared/output.txt ]; then
 	rm shared/output.txt
 fi
 touch shared/output.txt
-setpriv --pdeathsig TERM tail -f shared/output.txt &
+tail -f shared/output.txt &
+TAIL_PID=$!
 
-set -- timeout --foreground 40m \
-  debvm-run --image="$(realpath "$cachedir")/debian-$DEFAULT_DIST.ext4" \
-  --
-cpuname=$(lscpu | awk '/Model name:/ {print $3}' | tr '\n' '+')
-ncpu=$(lscpu | awk '/Core\(s\) per socket:/ {print $4}' | tr '\n' '+')
-if [ "$cpuname" = "Cortex-A53+Cortex-A73+" ] && [ "$ncpu" = "2+4+" ]; then
-  # crude detection of the big.LITTLE heterogeneous setup of cores on the
-  # amlogic a311d bananapi
-  #
-  # https://lists.nongnu.org/archive/html/qemu-devel/2020-10/msg08494.html
-  # https://gitlab.com/qemu-project/qemu/-/issues/239
-  # https://segments.zhan.science/posts/kvm_on_pinehone_pro/#trouble-with-heterogeneous-architecture
-  set -- taskset --cpu-list 2,3,4,5 "$@" -smp 4
-fi
-
-set -- "$@" -nic none -m 4G -snapshot
-
-if [ "$MMDEBSTRAP_TESTS_DEBUG" = "no" ]; then
 # to connect to serial use:
 #   minicom -D 'unix#/tmp/ttyS0'
+#
 # or this (quit with ctrl+q):
 #   socat stdin,raw,echo=0,escape=0x11 unix-connect:/tmp/ttyS0
-  set -- "$@" \
+ret=0
+timeout --foreground 40m debvm-run --image="$(realpath "$cachedir")/debian-$DEFAULT_DIST.ext4" -- \
+	-m 4G -snapshot \
 	-monitor unix:/tmp/monitor,server,nowait \
 	-serial unix:/tmp/ttyS0,server,nowait \
-    -serial unix:/tmp/ttyS1,server,nowait
+	-serial unix:/tmp/ttyS1,server,nowait \
-fi
+	-virtfs local,id=mmdebstrap,path="$(pwd)/shared",security_model=none,mount_tag=mmdebstrap \
-
+	>"$tmpdir/log" 2>&1 || ret=$?
-set -- "$@" -virtfs local,id=mmdebstrap,path="$(pwd)/shared",security_model=none,mount_tag=mmdebstrap
-
-ret=0
-if [ "$MMDEBSTRAP_TESTS_DEBUG" = "no" ]; then
-  "$@" >"$tmpdir/log" 2>&1 || ret=$?
-else
-  "$@" 2>&1 | tee "$tmpdir/log" || ret=$?
-fi
 if [ "$ret" -ne 0 ]; then
 	cat "$tmpdir/log"
 	exit $ret

tarfilter

@@ -218,11 +218,8 @@ Lastly, shift user id and group id of each entry by the value given by the
         skip = False
         if not hasattr(args, "pathfilter"):
             return False
-        # normalize path and make it absolute by stripping off all leading
-        # dots and slashes and then prepending a slash
-        name = "/" + member.name.lstrip("./")
         for t, r in args.pathfilter:
-            if r.match(name) is not None:
+            if r.match(member.name[1:]) is not None:
                 if t == "path_include":
                     skip = False
                 else:
@@ -233,7 +230,7 @@ Lastly, shift user id and group id of each entry by the value given by the
                     continue
                 prefix = prefix_prog.sub(r"\1", r.pattern)
                 prefix = prefix.rstrip("/")
-                if name.startswith(prefix):
+                if member.name[1:].startswith(prefix):
                     return False
         return skip
 

tests/apt-patterns

@@ -5,4 +5,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 {{ CMD }} --mode={{ MODE }} --variant=essential \
 	--include '?or(?exact-name(dummy-does-not-exist),?exact-name(apt))' \
 	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -

tests/apt-patterns-custom

@@ -6,7 +6,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 	--include '?narrow(?archive(^{{ DIST }}$),?essential)' \
 	--include apt \
 	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-{
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
-  tar -tf /tmp/debian-chroot.tar
-  echo ./var/lib/apt/extended_states
-} | sort | diff -u tar1.txt -

tests/arm64-without-qemu-support

@@ -5,7 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-apt-get remove --yes qemu-user-binfmt binfmt-support qemu-user
+apt-get remove --yes qemu-user-static binfmt-support qemu-user
 # the following is not necessary anymore since systemd-binfmt
 # successfully disables support upon removal of qemu-user with
 # the upload of src:systemd 251.2-4: https://bugs.debian.org/1012163

tests/as-debootstrap-unshare-wrapper

@@ -31,7 +31,7 @@ AUTOPROXY=
 eval "$(apt-config shell AUTOPROXY Acquire::http::Proxy-Auto-Detect)"
 if [ -n "$AUTOPROXY" ] && [ -x "$AUTOPROXY" ] && [ -e /tmp/.auto-apt-proxy-0 ]; then
 	TMP_APT_CONFIG=$(mktemp)
-  echo 'Dir "/dev/null";' >"$TMP_APT_CONFIG"
+	echo "Dir \"/dev/null\";" > "$TMP_APT_CONFIG"
 	chmod 644 "$TMP_APT_CONFIG"
 fi
 

tests/aspcud-apt-solver

@@ -6,9 +6,6 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
     --include "$(tr '\n' ',' < pkglist.txt)" \
     --aptopt='APT::Solver "aspcud"' \
     {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-{
+tar -tf /tmp/debian-chroot.tar | sort \
-  tar -tf /tmp/debian-chroot.tar
-  echo ./var/lib/apt/extended_states
-} | sort \
     | grep -v '^./etc/apt/apt.conf.d/99mmdebstrap$' \
     | diff -u tar1.txt -

tests/check-against-debootstrap-dist

@@ -13,7 +13,6 @@ echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
 {{ CMD }} --variant={{ VARIANT }} --mode={{ MODE }} \
 	--essential-hook='[ {{ DIST }} = oldstable ] && [ {{ VARIANT }} = - ] && echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd || :' \
 	"$(if [ {{ DIST }} = oldstable ]; then echo --merged-usr; else echo --hook-dir=./hooks/merged-usr; fi)" \
-  "$(case {{ DIST }} in oldstable) echo --include=e2fsprogs,mount,tzdata,gcc-9-base ;; stable) echo --include=e2fsprogs,mount,tzdata ;; *) echo --include=base-files ;; esac)" \
 	{{ DIST }} /tmp/debian-{{ DIST }}-mm.tar {{ MIRROR }}
 
 mkdir /tmp/debian-{{ DIST }}-mm
@@ -70,8 +69,6 @@ rm /tmp/debian-{{ DIST }}-debootstrap/var/cache/debconf/templates.dat-old \
 	/tmp/debian-{{ DIST }}-mm/var/cache/debconf/templates.dat-old
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status-old \
 	/tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status-old
-rm -f /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/diversions-old \
-  /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/diversions-old
 # remove dpkg files
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/available
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/cmethopt
@@ -109,7 +106,6 @@ if [ "{{ VARIANT }}" = "-" ]; then
 	rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/systemd/catalog/database
 	rm /tmp/debian-{{ DIST }}-mm/var/lib/systemd/catalog/database
 
-  case {{ DIST }} in oldstable | stable)
 	cap=$(chroot /tmp/debian-{{ DIST }}-debootstrap /sbin/getcap /bin/ping)
 	expected="/bin/ping cap_net_raw=ep"
 	if [ "$cap" != "$expected" ]; then
@@ -123,10 +119,7 @@ if [ "{{ VARIANT }}" = "-" ]; then
 		echo "but mmdebstrap produced: $cap" >&2
 		exit 1
 	fi
-    ;;
-  esac
 fi
-
 rm /tmp/debian-{{ DIST }}-mm/var/cache/apt/archives/lock
 rm /tmp/debian-{{ DIST }}-mm/var/lib/apt/extended_states
 rm /tmp/debian-{{ DIST }}-mm/var/lib/apt/lists/lock
@@ -161,75 +154,33 @@ if [ "{{ VARIANT }}" = "-" ] && [ "{{ DIST}}" = oldstable ]; then
 fi
 
 for log in faillog lastlog; do
-  f1="/tmp/debian-{{ DIST }}-debootstrap/var/log/$log"
+	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/var/log/$log /tmp/debian-{{ DIST }}-mm/var/log/$log >&2;then
-  f2="/tmp/debian-{{ DIST }}-mm/var/log/$log"
-  # skip cmp if file is absent in both chroots
-  if [ ! -e "$f1" ] && [ ! -e "$f2" ]; then
-    continue
-  fi
-  if ! cmp "$f1" "$f2" >&2; then
 		# if the files differ, make sure they are all zeroes
-    cmp -n "$(stat -c %s "$f1")" "$f1" /dev/zero >&2
+		cmp -n "$(stat -c %s "/tmp/debian-{{ DIST }}-debootstrap/var/log/$log")" "/tmp/debian-{{ DIST }}-debootstrap/var/log/$log" /dev/zero >&2
-    cmp -n "$(stat -c %s "$f2")" "$f2" /dev/zero >&2
+		cmp -n "$(stat -c %s "/tmp/debian-{{ DIST }}-mm/var/log/$log")" "/tmp/debian-{{ DIST }}-mm/var/log/$log" /dev/zero >&2
 		# then delete them
-    rm "$f1" "$f2"
+		rm /tmp/debian-{{ DIST }}-debootstrap/var/log/$log /tmp/debian-{{ DIST }}-mm/var/log/$log
 	fi
 done
 
-if [ "{{ VARIANT }}" = "-" ]; then
 # the order in which systemd and cron get installed differ and thus the order
 # of lines in /etc/group and /etc/gshadow differs
+if [ "{{ VARIANT }}" = "-" ]; then
 	for f in group group- gshadow gshadow-; do
 		for d in mm debootstrap; do
 			sort /tmp/debian-{{ DIST }}-$d/etc/$f > /tmp/debian-{{ DIST }}-$d/etc/$f.bak
 			mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
 		done
 	done
-  # the order in which systemd and passwd get installed differ and thus
-  # the order of lines in /etc/shadow and /etc/shadow- differs
-  for f in shadow shadow-; do
-    for d in mm debootstrap; do
-      sort /tmp/debian-{{ DIST }}-$d/etc/$f >/tmp/debian-{{ DIST }}-$d/etc/$f.bak
-      mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
-    done
-  done
-  # and since the order was different, ignore the *- files
-  for f in shadow- passwd-; do
-    for d in mm debootstrap; do
-      rm /tmp/debian-{{ DIST }}-$d/etc/$f
-    done
-  done
 fi
 
 # since debootstrap 1.0.133 there is no tzdata in the buildd variant and thus
 # debootstrap creates its own /etc/localtime
-if [ "{{ VARIANT }}" = "buildd" ] && [ "{{ DIST }}" != "stable" ] && [ "{{ DIST }}" != "oldstable" ]; then
+if [ "{{ VARIANT }}" = "buildd" ]; then
 	[ "$(readlink /tmp/debian-{{ DIST }}-debootstrap/etc/localtime)" = /usr/share/zoneinfo/UTC ]
 	rm /tmp/debian-{{ DIST }}-debootstrap/etc/localtime
 fi
 
-# starting with systemd 255 upstream dropped splitusr support and depending on
-# the installation order, symlink targets are prefixed with /usr or not
-# See #1060000 and #1054137
-case {{ DIST }} in testing | unstable)
-  for f in multi-user.target.wants/e2scrub_reap.service timers.target.wants/apt-daily-upgrade.timer timers.target.wants/apt-daily.timer timers.target.wants/e2scrub_all.timer; do
-    for d in mm debootstrap; do
-      [ -L "/tmp/debian-{{ DIST }}-$d/etc/systemd/system/$f" ] || continue
-      oldlink="$(readlink "/tmp/debian-{{ DIST }}-$d/etc/systemd/system/$f")"
-      case $oldlink in
-        /usr/*) : ;;
-        /*) oldlink="/usr$oldlink" ;;
-        *)
-          echo unexpected >&2
-          exit 1
-          ;;
-      esac
-      ln -sf "$oldlink" "/tmp/debian-{{ DIST }}-$d/etc/systemd/system/$f"
-    done
-  done
-  ;;
-esac
-
 # check if the file content differs
 diff --unified --no-dereference --recursive /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm >&2
 
@@ -238,9 +189,8 @@ diff --unified --no-dereference --recursive /tmp/debian-{{ DIST }}-debootstrap /
 find /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm -type d -print0 | xargs -0 touch --date="@{{ SOURCE_DATE_EPOCH }}"
 # debootstrap never ran apt -- fixing permissions
 for d in ./var/lib/apt/lists/partial ./var/cache/apt/archives/partial; do
-  unmergedPATH="$PATH$(if [ "{{ DIST }}" = oldstable ]; then echo :/bin:/sbin; fi)"
+	chroot /tmp/debian-{{ DIST }}-debootstrap chmod 0700 $d
-  PATH="$unmergedPATH" chroot /tmp/debian-{{ DIST }}-debootstrap chmod 0700 $d
+	chroot /tmp/debian-{{ DIST }}-debootstrap chown "$(id -u _apt):root" $d
-  PATH="$unmergedPATH" chroot /tmp/debian-{{ DIST }}-debootstrap chown "$(id -u _apt):root" $d
 done
 tar -C /tmp/debian-{{ DIST }}-debootstrap --numeric-owner --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/root1.tar .
 tar -C /tmp/debian-{{ DIST }}-mm --numeric-owner --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/root2.tar .

tests/chrootless

@@ -8,7 +8,7 @@ trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
 for INCLUDE in '' 'apt' 'apt,build-essential' 'systemd-sysv'; do
 	for MODE in root chrootless; do
 		{{ CMD }} --mode=$MODE --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
-      ${INCLUDE:+--include="$INCLUDE"} --skip=check/chrootless \
+			${INCLUDE:+--include="$INCLUDE"} \
 			{{ DIST }} "/tmp/$MODE.tar" {{ MIRROR }}
 	done
 	cmp /tmp/root.tar /tmp/chrootless.tar || diffoscope /tmp/root.tar /tmp/chrootless.tar

tests/chrootless-fakeroot

@@ -18,26 +18,18 @@ if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto
 	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
 
-MMTARFILTER=
-[ -x /usr/bin/mmtarfilter ] && MMTARFILTER=/usr/bin/mmtarfilter
-[ -x ./tarfilter ] && MMTARFILTER=./tarfilter
-
 # we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
 # DPKG_ROOT
 # permissions drwxr-sr-x and extended attributes of ./var/log/journal/ cannot
 # be preserved under fakeroot
-# this applies to 'z' lines in files in /usr/lib/tmpfiles.d/
 for INCLUDE in '' 'apt' 'apt,build-essential' 'systemd-sysv'; do
 	{{ CMD }} --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
+		--customize-hook='if [ -d "$1"/var/log/journal ]; then rmdir "$1"/var/log/journal; mkdir --mode=2755 "$1"/var/log/journal; chroot "$1" chown root:systemd-journal /var/log/journal; fi' \
 		${INCLUDE:+--include="$INCLUDE"} \
-    {{ DIST }} - {{ MIRROR }} \
+		{{ DIST }} /tmp/root.tar {{ MIRROR }}
-    | "$MMTARFILTER" --path-exclude="/var/log/journal" --path-exclude="/etc/credstore*" \
-      >/tmp/root.tar
 	$prefix fakeroot {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
 		${INCLUDE:+--include="$INCLUDE"} \
-    {{ DIST }} - {{ MIRROR }} \
+		{{ DIST }} /tmp/chrootless.tar {{ MIRROR }}
-    | "$MMTARFILTER" --path-exclude="/var/log/journal" --path-exclude="/etc/credstore*" \
-      >/tmp/chrootless.tar
 	cmp /tmp/root.tar /tmp/chrootless.tar || diffoscope /tmp/root.tar /tmp/chrootless.tar
 	rm /tmp/chrootless.tar /tmp/root.tar
 done

tests/chrootless-foreign

@@ -26,6 +26,7 @@ fi
 [ "$(id -u)" -eq 0 ]
 [ -e "/proc/sys/fs/binfmt_misc/qemu-$(deb2qemu "$arch")" ]
 
+
 # we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
 # DPKG_ROOT
 #
@@ -44,7 +45,7 @@ for INCLUDE in '' 'apt' 'systemd-sysv'; do
 	arch-test "$arch" && exit 1
 	{{ CMD }} --mode=chrootless --architecture="$arch" --variant={{ VARIANT }} \
 		--hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
-    --skip=check/chrootless {{ DIST }} "/tmp/chrootless.tar" {{ MIRROR }}
+		{{ DIST }} "/tmp/chrootless.tar" {{ MIRROR }}
 	# when creating a foreign architecture chroot, the tarballs are not
 	# bit-by-bit identical but contain a few remaining differences:
 	#
@@ -52,7 +53,8 @@ for INCLUDE in '' 'apt' 'systemd-sysv'; do
 	#  * /var/lib/dpkg/triggers -- #990712
 	#  * /var/cache/debconf/*.dat-old -- needs investigation
 	for tar in root chrootless; do
-    ./tarfilter <"/tmp/$tar.tar" \
+		<"/tmp/$tar.tar" \
+		./tarfilter \
 			--path-exclude=/var/cache/debconf/config.dat-old \
 			--path-exclude=/var/cache/debconf/templates.dat-old \
 			--path-exclude=/etc/ld.so.cache \

tests/create-arm64-tarball

@@ -0,0 +1,65 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+	fi
+	prefix="runuser -u ${SUDO_USER:-user} --"
+fi
+
+[ "{{ MODE }}" = "fakechroot" ] && prefix="$prefix fakechroot fakeroot"
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --architectures=arm64 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+# we ignore differences between architectures by ignoring some files
+# and renaming others
+{ tar -tf /tmp/debian-chroot.tar \
+	| grep -v '^\./usr/lib/ld-linux-aarch64\.so\.1$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/ld-linux-aarch64\.so\.1$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/int-ll64\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/types\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/unistd\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/sigcontext\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/sve_context\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/types\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-extra\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-id\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-prregset\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/gnu/stubs-lp64\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/linux/types\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/sys/procfs\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/sys/user\.ph$' \
+	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.arm64\.gz$' \
+	| sed 's/aarch64-linux-gnu/x86_64-linux-gnu/' \
+	| sed 's/arm64/amd64/';
+} | sort > tar2.txt
+{ < tar1.txt \
+	grep -v '^\./usr/bin/i386$' \
+	| grep -v '^\./usr/bin/x86_64$' \
+	| grep -v '^\./lib32$' \
+	| grep -v '^\./lib64$' \
+	| grep -v '^\./libx32$' \
+	| grep -v '^\./usr/lib32/$' \
+	| grep -v '^\./usr/libx32/$' \
+	| grep -v '^\./usr/lib64/$' \
+	| grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/ld-linux-x86-64\.so\.2$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/libmvec\.so\.1$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_32\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_64\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_x32\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_32\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_64\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_x32\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/gnu/stubs-64\.ph$' \
+	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.amd64\.gz$' \
+	| grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
+	| grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$';
+} | sort | diff -u - tar2.txt >&2
+rm /tmp/debian-chroot.tar

tests/create-foreign-tarball

@@ -1,78 +0,0 @@
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-
-prefix=
-if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
-  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
-    if [ ! -e /mmdebstrap-testenv ]; then
-      echo "this test modifies the system and should only be run inside a container" >&2
-      exit 1
-    fi
-    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
-  fi
-  prefix="runuser -u ${SUDO_USER:-user} --"
-fi
-
-case "$(dpkg --print-architecture)" in
-  arm64)
-    native_arch=arm64
-    native_gnu=aarch64-linux-gnu
-    foreign_arch=amd64
-    foreign_gnu=x86_64-linux-gnu
-    ;;
-  amd64)
-    native_arch=amd64
-    native_gnu=x86_64-linux-gnu
-    foreign_arch=arm64
-    foreign_gnu=aarch64-linux-gnu
-    ;;
-  *)
-    echo "unsupported native architecture" >&2
-    exit 1
-    ;;
-esac
-
-[ "{{ MODE }}" = "fakechroot" ] && prefix="$prefix fakechroot fakeroot"
-$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --architectures="$foreign_arch" \
-  {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-# we ignore differences between architectures by ignoring some files
-# and renaming others
-{
-  tar -tf /tmp/debian-chroot.tar \
-    | grep -v '^\./usr/bin/i386$' \
-    | grep -v '^\./usr/bin/x86_64$' \
-    | grep -v '^\./lib64$' \
-    | grep -v '^\./usr/lib64/$' \
-    | grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
-    | grep -v '^\./usr/lib/ld-linux-aarch64\.so\.1$' \
-    | grep -v "^\\./usr/lib/$foreign_gnu/ld-linux-aarch64\\.so\\.1$" \
-    | grep -v "^\\./usr/lib/$foreign_gnu/ld-linux-x86-64\\.so\\.2$" \
-    | grep -v "^\\./usr/lib/$foreign_gnu/perl/5\\.[0-9][.0-9]\\+/.*\\.ph$" \
-    | grep -v "^\\./usr/lib/$foreign_gnu/libmvec\\.so\\.1$" \
-    | grep -v "^\\./usr/share/doc/[^/]\\+/changelog\\(\\.Debian\\)\\?\\.$foreign_arch\\.gz$" \
-    | grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
-    | grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$' \
-    | sed "s/$foreign_gnu/$native_gnu/" \
-    | sed "s/$foreign_arch/$native_arch/"
-} | sort >/tmp/tar2.txt
-{
-  grep <tar1.txt -v '^\./usr/bin/i386$' \
-    | grep -v '^\./usr/bin/x86_64$' \
-    | grep -v '^\./lib32$' \
-    | grep -v '^\./lib64$' \
-    | grep -v '^\./libx32$' \
-    | grep -v '^\./usr/lib32/$' \
-    | grep -v '^\./usr/libx32/$' \
-    | grep -v '^\./usr/lib64/$' \
-    | grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
-    | grep -v '^\./usr/lib/ld-linux-aarch64\.so\.1$' \
-    | grep -v "^\\./usr/lib/$native_gnu/ld-linux-x86-64\\.so\\.2$" \
-    | grep -v "^\\./usr/lib/$native_gnu/ld-linux-aarch64\\.so\\.1$" \
-    | grep -v "^\\./usr/lib/$native_gnu/libmvec\\.so\\.1$" \
-    | grep -v "^\\./usr/lib/$native_gnu/perl/5\\.[0-9][.0-9]\\+/.*\\.ph$" \
-    | grep -v "^\\./usr/share/doc/[^/]\\+/changelog\\(\\.Debian\\)\\?\\.$native_arch\\.gz$" \
-    | grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
-    | grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$'
-} | sort | diff -u - /tmp/tar2.txt >&2
-rm /tmp/debian-chroot.tar /tmp/tar2.txt

tests/custom-tmpdir

@@ -18,7 +18,7 @@ prefix="runuser -u ${SUDO_USER:-user} --"
 quote () { printf %s\\n "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" ; }
 homedir=$($prefix sh -c 'cd && pwd')
 # apt:test/integration/test-apt-key
-TMPDIR_ADD='This is fü$$ing cràzy, $(apt -v)$!'
+TMPDIR_ADD="This is fü\$\$ing cràzy, \$(apt -v)\$!"
 $prefix mkdir "$homedir/$TMPDIR_ADD"
 # make sure the unshared user can traverse into the TMPDIR
 chmod 711 "$homedir"

tests/deb822-1-2

@@ -40,6 +40,6 @@ tar -C /tmp/debian-chroot --one-file-system -c . \
 		tar -t \
 			| grep -v "^./etc/apt/sources.list.d/0000deb822.sources$" \
 			| grep -v "^./etc/apt/sources.list.d/0001main.list$" \
-      | grep -v "^./etc/apt/sources.list.d/0002sources.list"
+			| grep -v "^./etc/apt/sources.list.d/0002sources.list";
-    printf "./etc/apt/sources.list\n"
+		printf "./etc/apt/sources.list\n";
 	} | sort | diff -u tar1.txt -

tests/deb822-2-2

@@ -39,6 +39,6 @@ tar -C /tmp/debian-chroot --one-file-system -c . \
 		tar -t \
 			| grep -v "^./etc/apt/sources.list.d/0000deb822.sources$" \
 			| grep -v "^./etc/apt/sources.list.d/0001main.sources$" \
-      | grep -v "^./etc/apt/sources.list.d/0002sources.list$"
+			| grep -v "^./etc/apt/sources.list.d/0002sources.list$";
-    printf "./etc/apt/sources.list\n"
+		printf "./etc/apt/sources.list\n";
 	} | sort | diff -u tar1.txt -

tests/debootstrap

@@ -5,12 +5,6 @@ export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
 tmpdir="$(mktemp -d)"
 chmod 755 "$tmpdir"
-ret=0
+debootstrap "$([ "{{ DIST }}" = oldstable ] && echo --no-merged-usr || echo --merged-usr)" --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
-debootstrap "$([ "{{ DIST }}" = oldstable ] && echo --no-merged-usr || echo --merged-usr)" --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }} || ret=$?
-if [ "$ret" -ne 0 ]; then
-  echo "E: debootstrap failed, dumping $tmpdir/debootstrap/debootstrap.log"
-  cat "$tmpdir/debootstrap/debootstrap.log"
-  exit 1
-fi
 tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system --xattrs -C "$tmpdir" -c . > "./cache/debian-{{ DIST }}-{{ VARIANT }}.tar"
 rm -r "$tmpdir"

tests/debug

@@ -7,7 +7,9 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 
 # we use variant standard in verbose mode to see the maximum number of packages
 # that was chosen in case of USE_HOST_APT_CONFIG=yes
-case {{ VARIANT }} in standard) : ;; *) exit 1 ;; esac
+# we use variant important on arches where variant standard is not bit-by-bit
+# reproducible due to #1031276
+case {{ VARIANT }} in standard|-) : ;; *) exit 1;; esac
 
 {{ CMD }} --variant={{ VARIANT }} --debug {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 

tests/dpkgopt

@@ -7,8 +7,4 @@ echo no-pager >/tmp/config
 printf 'path-exclude=/usr/share/doc/*\nno-pager\npath-include=/usr/share/doc/dpkg/copyright\n' | cmp /tmp/debian-chroot/etc/dpkg/dpkg.cfg.d/99mmdebstrap -
 rm /tmp/debian-chroot/etc/dpkg/dpkg.cfg.d/99mmdebstrap
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort > tar2.txt
-{
+{ grep -v '^./usr/share/doc/.' tar1.txt; echo ./usr/share/doc/dpkg/; echo ./usr/share/doc/dpkg/copyright; } | sort | diff -u - tar2.txt
-  grep -v '^./usr/share/doc/.' tar1.txt
-  echo ./usr/share/doc/dpkg/
-  echo ./usr/share/doc/dpkg/copyright
-} | sort | diff -u - tar2.txt

tests/eatmydata-via-hook-dir

@@ -12,24 +12,18 @@ EOF
 SCRIPT
 chmod +x /tmp/checkeatmydata.sh
 # first four bytes: magic
-elfheader='\177ELF'
+elfheader="\\177ELF"
 # fifth byte: bits
 case "$(dpkg-architecture -qDEB_HOST_ARCH_BITS)" in
 	32) elfheader="$elfheader\\001";;
 	64) elfheader="$elfheader\\002";;
-  *)
+	*) echo "bits not supported"; exit 1;;
-    echo "bits not supported"
-    exit 1
-    ;;
 esac
 # sixth byte: endian
 case "$(dpkg-architecture -qDEB_HOST_ARCH_ENDIAN)" in
 	little) elfheader="$elfheader\\001";;
 	big) elfheader="$elfheader\\002";;
-  *)
+	*) echo "endian not supported"; exit 1;;
-    echo "endian not supported"
-    exit 1
-    ;;
 esac
 # seventh and eigth byte: elf version (1) and abi (unset)
 elfheader="$elfheader\\001\\000"
@@ -43,6 +37,7 @@ elfheader="$elfheader\\001\\000"
  tar -C /tmp/debian-chroot --one-file-system -c . \
 	 | tar -t \
 	 | sort \
+	 | grep -v '^\./var/lib/dpkg/diversions\(-old\)\?$' \
 	 | diff -u tar1.txt -
 rm /tmp/checkeatmydata.sh
 rm -r /tmp/debian-chroot

tests/empty-sources.list

@@ -3,6 +3,6 @@ set -eu
 export LC_ALL=C.UTF-8
 trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 printf '' | {{ CMD }} --mode={{ MODE }} --variant=apt \
-  --setup-hook='test -e "$1"/etc/apt/sources.list || echo "deb {{ MIRROR }} {{ DIST }} main" > "$1"/etc/apt/sources.list' \
+	--setup-hook='echo "deb {{ MIRROR }} {{ DIST }} main" > "$1"/etc/apt/sources.list' \
 	{{ DIST }} /tmp/debian-chroot.tar -
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/empty-suite

@@ -1,13 +0,0 @@
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-
-trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
-
-{{ CMD }} --variant={{ VARIANT }} --verbose \
-  --setup-hook='echo deb {{ MIRROR }} {{ DIST }} main >> "$1"/etc/apt/sources.list' \
-  '' /tmp/debian-chroot.tar
-
-cmp ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar /tmp/debian-chroot.tar \
-  || diffoscope ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar /tmp/debian-chroot.tar

tests/essential-hook

@@ -17,4 +17,5 @@ tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort \
 	| grep -v '^./usr/share/lintian/overrides/tzdata' \
 	| grep -v '^./usr/share/zoneinfo' \
 	| grep -v '^./var/lib/dpkg/info/tzdata.' \
+	| grep -v '^./var/lib/apt/extended_states$' \
 	| diff -u tar1.txt -

tests/i386-which-can-be-executed-without-qemu

@@ -6,12 +6,11 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	exit 1
 fi
 # remove qemu just to be sure
-apt-get remove --yes qemu-user-binfmt binfmt-support qemu-user
+apt-get remove --yes qemu-user-static binfmt-support qemu-user
 {{ CMD }} --mode={{ MODE }} --variant=apt --architectures=i386 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 # we ignore differences between architectures by ignoring some files
 # and renaming others
-{
+{ tar -tf /tmp/debian-chroot.tar \
-  tar -tf /tmp/debian-chroot.tar \
 	| grep -v '^\./usr/bin/i386$' \
 	| grep -v '^\./usr/lib/ld-linux\.so\.2$' \
 	| grep -v '^\./usr/lib/i386-linux-gnu/ld-linux\.so\.2$' \
@@ -21,10 +20,10 @@ apt-get remove --yes qemu-user-binfmt binfmt-support qemu-user
 	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.i386\.gz$' \
 	| sed 's/i386-linux-gnu/x86_64-linux-gnu/' \
 	| sed 's/i386/amd64/' \
-    | sed 's/\/stubs-32.ph$/\/stubs-64.ph/'
+	| sed 's/\/stubs-32.ph$/\/stubs-64.ph/';
 } | sort > tar2.txt
-{
+{ < tar1.txt \
-  grep <tar1.txt -v '^\./usr/bin/i386$' \
+	grep -v '^\./usr/bin/i386$' \
 	| grep -v '^\./usr/bin/x86_64$' \
 	| grep -v '^\./usr/lib32/$' \
 	| grep -v '^\./lib32$' \
@@ -37,6 +36,6 @@ apt-get remove --yes qemu-user-binfmt binfmt-support qemu-user
 	| grep -v '^\./usr/lib/x86_64-linux-gnu/libmvec\.so\.1$' \
 	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.amd64\.gz$' \
 	| grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
-    | grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$'
+	| grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$';
 } | sort | diff -u - tar2.txt >&2
 rm /tmp/debian-chroot.tar

tests/include

@@ -6,6 +6,7 @@ trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
 rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
+rm /tmp/debian-chroot/var/lib/apt/extended_states
 rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.list
 rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.md5sums
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/include-foreign-libmagic-mgc-with-multiple-arch-options

@@ -1,50 +0,0 @@
-#!/bin/sh
-#
-# to test foreign architecture package installation we choose a package which
-#   - is not part of the native installation set
-#   - does not have any dependencies
-#   - installs only few files
-#   - doesn't change its name regularly (like gcc-*-base)
-
-case "$(dpkg --print-architecture)" in
-  arm64)
-    native_arch=arm64
-    foreign_arch=amd64
-    ;;
-  amd64)
-    native_arch=amd64
-    foreign_arch=arm64
-    ;;
-  *)
-    echo "unsupported native architecture" >&2
-    exit 1
-    ;;
-esac
-
-set -eu
-export LC_ALL=C.UTF-8
-{{ CMD }} --mode=root --variant=apt \
-  --architectures="$native_arch" \
-  --architectures="$foreign_arch" \
-  --include="libmagic-mgc:$foreign_arch" \
-  {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-{
-  echo "$native_arch"
-  echo "$foreign_arch"
-} | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
-rm /tmp/debian-chroot/usr/lib/file/magic.mgc
-rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian
-rm -f /tmp/debian-chroot/usr/share/doc/libmagic-mgc/"changelog.Debian.$foreign_arch.gz"
-rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz
-rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
-rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
-rm /tmp/debian-chroot/usr/share/file/magic.mgc
-rm /tmp/debian-chroot/usr/share/misc/magic.mgc
-rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
-rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
-rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/
-rmdir /tmp/debian-chroot/usr/share/file/magic/
-rmdir /tmp/debian-chroot/usr/share/file/
-rmdir /tmp/debian-chroot/usr/lib/file/
-tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
-rm -r /tmp/debian-chroot

tests/include-libmagic-mgc-arm64

@@ -6,41 +6,20 @@
 #   - installs only few files
 #   - doesn't change its name regularly (like gcc-*-base)
 
-case "$(dpkg --print-architecture)" in
-  arm64)
-    native_arch=arm64
-    foreign_arch=amd64
-    ;;
-  amd64)
-    native_arch=amd64
-    foreign_arch=arm64
-    ;;
-  *)
-    echo "unsupported native architecture" >&2
-    exit 1
-    ;;
-esac
-
 set -eu
 export LC_ALL=C.UTF-8
-{{ CMD }} --mode=root --variant=apt \
+{{ CMD }} --mode=root --variant=apt --architectures=amd64,arm64 --include=libmagic-mgc:arm64 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-  --architectures="$native_arch,$foreign_arch" \
+{ echo "amd64"; echo "arm64"; } | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
-  --include="libmagic-mgc:$foreign_arch" \
+rm /tmp/debian-chroot/var/lib/apt/extended_states
-  {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
-{
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
-  echo "$native_arch"
-  echo "$foreign_arch"
-} | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
 rm /tmp/debian-chroot/usr/lib/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian
-rm -f /tmp/debian-chroot/usr/share/doc/libmagic-mgc/"changelog.Debian.$foreign_arch.gz"
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
 rm /tmp/debian-chroot/usr/share/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/misc/magic.mgc
-rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
-rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
 rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/
 rmdir /tmp/debian-chroot/usr/share/file/magic/
 rmdir /tmp/debian-chroot/usr/share/file/

tests/include-libmagic-mgc-arm64-with-multiple-arch-options

@@ -0,0 +1,21 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+{{ CMD }} --mode=root --variant=apt --architectures=amd64 --architectures=arm64 --include=libmagic-mgc:arm64 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+{ echo "amd64"; echo "arm64"; } | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
+rm /tmp/debian-chroot/var/lib/apt/extended_states
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
+rm /tmp/debian-chroot/usr/lib/file/magic.mgc
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
+rm /tmp/debian-chroot/usr/share/file/magic.mgc
+rm /tmp/debian-chroot/usr/share/misc/magic.mgc
+rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/
+rmdir /tmp/debian-chroot/usr/share/file/magic/
+rmdir /tmp/debian-chroot/usr/share/file/
+rmdir /tmp/debian-chroot/usr/lib/file/
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+rm -r /tmp/debian-chroot

tests/install-busybox-based-sub-essential-system

@@ -22,17 +22,12 @@ chroot /tmp/debian-chroot dpkg-query -f '${binary:Package}\n' -W \
 rm /tmp/expected
 for cmd in echo cat sed grep; do
 	test -L /tmp/debian-chroot/bin/$cmd
-  test "$(readlink /tmp/debian-chroot/bin/$cmd)" = "/usr/bin/busybox"
+	test "$(readlink /tmp/debian-chroot/bin/$cmd)" = "/bin/busybox"
 done
 for cmd in sort tee; do
 	test -L /tmp/debian-chroot/usr/bin/$cmd
-  test "$(readlink /tmp/debian-chroot/usr/bin/$cmd)" = "/usr/bin/busybox"
+	test "$(readlink /tmp/debian-chroot/usr/bin/$cmd)" = "/bin/busybox"
 done
-
-# if /bin or /sbin are not symlinks, add /bin and /sbin to PATH
-if [ ! -L /tmp/debian-chroot/bin ] || [ ! -L /tmp/debian-chroot/sbin ]; then
-  export PATH="$PATH:/sbin:/bin"
-fi
 chroot /tmp/debian-chroot echo foobar \
 	| chroot /tmp/debian-chroot cat \
 	| chroot /tmp/debian-chroot sort \

tests/install-libmagic-mgc-on-arm64

@@ -17,27 +17,13 @@ if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto
 	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
 
-case "$(dpkg --print-architecture)" in
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --architectures=arm64 --include=libmagic-mgc {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-  arm64)
-    foreign_arch=amd64
-    ;;
-  amd64)
-    foreign_arch=arm64
-    ;;
-  *)
-    echo "unsupported native architecture" >&2
-    exit 1
-    ;;
-esac
-
-$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --architectures="$foreign_arch" --include=libmagic-mgc {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 # delete contents of libmagic-mgc
 rm /tmp/debian-chroot/usr/lib/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
-rm -f /tmp/debian-chroot/usr/share/doc/libmagic-mgc/"changelog.Debian.$foreign_arch.gz"
 rm /tmp/debian-chroot/usr/share/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/misc/magic.mgc
 # delete real files

tests/jessie-or-older

@@ -30,7 +30,6 @@ filter() {
 		--path-exclude=/var/cache/debconf/templates.dat-old \
 		--path-exclude=/var/lib/dpkg/available \
 		--path-exclude=/var/lib/dpkg/diversions \
-    --path-exclude=/var/lib/dpkg/diversions-old \
 		--path-exclude=/var/lib/dpkg/cmethopt \
 		--path-exclude=/var/lib/dpkg/status-old \
 		--path-exclude=/var/lib/shells.state

tests/logfile

@@ -16,7 +16,6 @@ I: running apt-get update...
 I: downloading packages with apt...
 I: extracting archives...
 I: installing essential packages...
-I: installing remaining packages inside the chroot...
 I: cleaning package lists and apt cache...
 LOG
 tail --lines=1 /tmp/log | grep '^I: success in .* seconds$'

tests/missing-dev-sys-proc-inside-the-chroot

@@ -17,8 +17,4 @@ if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto
 	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
 
-# do not install base-files, so that /proc, /sys and /dev are missing
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --include=dpkg,dash,diffutils,coreutils,libc-bin,sed {{ DIST }} /dev/null {{ MIRROR }}
-$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} \
-  --setup-hook='for dir in bin lib lib32 lib64 libo32 libx32 sbin; do ln -s "usr/$dir" "$1/$dir"; done' \
-  --include=dpkg,dash,diffutils,coreutils,libc-bin,sed \
-  {{ DIST }} /dev/null {{ MIRROR }}

tests/mmdebstrap

@@ -5,26 +5,15 @@ export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
 [ "$(id -u)" -eq 0 ]
 [ {{ MODE }} = "root" ]
-case {{ FORMAT }} in tar | squashfs | ext2 | ext4) : ;; *) exit 1 ;; esac
+case {{ FORMAT }} in tar|squashfs|ext2) : ;; *) exit 1;; esac
 
-{{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} /tmp/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} {{ MIRROR }}
+{{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} {{ MIRROR }}
-# creating an ext4 image on a 9p filesystem produces different results compared
-# to creating it on a tmpfs or ext4 fs because 9p does not support discards and
-# even when running with -E nodiscard, the number of written bytes will differ
-# https://lore.kernel.org/linux-ext4/171484520952.2626447.2160419274451668597@localhost/T/#t
-mv /tmp/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }}
 if [ "{{ FORMAT }}" = tar ]; then
-  printf 'ustar\0' | cmp --bytes=6 --ignore-initial=257:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar -
+	printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar -
 elif [ "{{ FORMAT }}" = squashfs ]; then
 	printf 'hsqs' | cmp --bytes=4 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.squashfs -
 elif [ "{{ FORMAT }}" = ext2 ]; then
 	printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext2 -
-  printf '\000\000\000\000\000\000\000\000\000\000\000\000' | cmp --bytes=12 --ignore-initial=1116:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext2 -
-elif [ "{{ FORMAT }}" = ext4 ]; then
-  printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext4 -
-  printf '\074\020\000\000\302\042\000\000\153\004\000\000' | cmp --bytes=12 --ignore-initial=1116:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext4 -
-
-  [ "$(/sbin/blkid --match-tag UUID --output value ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext4)" = "$(uuidgen --sha1 --namespace="$(uuidgen --sha1 --namespace='@dns' --name mister-muffin.de)" --name $SOURCE_DATE_EPOCH)" ]
 else
 	echo "unknown format: {{ FORMAT }}" >&2
 	exit 1

tests/multiple-include

@@ -11,6 +11,7 @@ rm /tmp/debian-chroot/etc/localtime
 rm /tmp/debian-chroot/etc/timezone
 rm -r /tmp/debian-chroot/usr/share/doc/tzdata
 rm -r /tmp/debian-chroot/usr/share/zoneinfo
+rm /tmp/debian-chroot/var/lib/apt/extended_states
 for p in doc-debian tzdata; do
 	for f in list md5sums config postinst postrm templates preinst prerm; do
 		[ -e "/tmp/debian-chroot/var/lib/dpkg/info/$p.$f" ] || continue

tests/not-having-to-install-apt-in-include-because-a-hook-did-it-before

@@ -6,4 +6,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 	--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get update' \
 	--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install apt' \
 	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -

tests/pivot_root

@@ -17,19 +17,19 @@ if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto
 fi
 
 $prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
-  --include=mount,perl \
+	--include=mount \
 	{{ DIST }} /tmp/chroot1.tar {{ MIRROR }}
 
 if [ {{ MODE }} = "unshare" ]; then
 	# calling pivot_root in root mode does not work for mysterious reasons:
 	# pivot_root: failed to change root from `.' to `mnt': Invalid argument
-  $prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount,perl \
+	$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount \
 		--customize-hook='mkdir -p "$1/mnt" "$1/oldroot"' \
 		--customize-hook='[ ! -e /usr/bin/mmdebstrap ] || cp -aT /usr/bin/mmdebstrap "$1/usr/bin/mmdebstrap"' \
 		--customize-hook='[ ! -e ./mmdebstrap ] || cp -aT ./mmdebstrap "$1/mnt/mmdebstrap"' \
 		--customize-hook='mount -o rbind "$1" /mnt && cd /mnt && /sbin/pivot_root . oldroot' \
 		--customize-hook='unshare -U echo nested unprivileged unshare' \
-    --customize-hook='env --chdir=/mnt {{ CMD }} --mode=unshare --variant=apt --include=mount,perl {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
+		--customize-hook='env --chdir=/mnt {{ CMD }} --mode=unshare --variant=apt --include=mount {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
 		--customize-hook='copy-out /tmp/chroot3.tar /tmp' \
 		--customize-hook='rm -f "/usr/bin/mmdebstrap" "/mnt/mmdebstrap"' \
 		--customize-hook='umount -l oldroot sys' \
@@ -41,11 +41,11 @@ if [ {{ MODE }} = "unshare" ]; then
 		rm /tmp/chroot2.tar /tmp/chroot3.tar
 fi
 
-$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount,perl \
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount \
 	--customize-hook='mkdir -p "$1/mnt"' \
 	--customize-hook='[ ! -e /usr/bin/mmdebstrap ] || cp -aT /usr/bin/mmdebstrap "$1/usr/bin/mmdebstrap"' \
 	--customize-hook='[ ! -e ./mmdebstrap ] || cp -aT ./mmdebstrap "$1/mnt/mmdebstrap"' \
-  --chrooted-customize-hook='env --chdir=/mnt {{ CMD }} --mode=unshare --variant=apt --include=mount,perl {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
+	--chrooted-customize-hook='env --chdir=/mnt {{ CMD }} --mode=unshare --variant=apt --include=mount {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
 	--customize-hook='copy-out /tmp/chroot3.tar /tmp' \
 	--customize-hook='rm -f "$1/usr/bin/mmdebstrap" "$1/mnt/mmdebstrap"' \
 	{{ DIST }} /tmp/chroot2.tar {{ MIRROR }}

tests/remove-start-stop-daemon-and-policy-rc-d-in-hook

@@ -2,7 +2,5 @@
 set -eu
 export LC_ALL=C.UTF-8
 trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
-{{ CMD }} --mode={{ MODE }} --variant=apt \
+{{ CMD }} --mode={{ MODE }} --variant=apt --customize-hook='rm "$1/usr/sbin/policy-rc.d"; rm "$1/sbin/start-stop-daemon"' {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-  --customize-hook='rm "$1/usr/sbin/policy-rc.d"; rm "$1/usr/sbin/start-stop-daemon"' \
-  {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/skip-output-dev

@@ -17,19 +17,19 @@ fi
 # test this for both unshare and root mode because the code paths creating
 # entries in /dev are different depending on whether mknod is available or not
 $prefix {{ CMD }} --mode={{ MODE }} --variant=apt --skip=output/dev {{ DIST }} - {{ MIRROR }} | {
-  tar -t
+	tar -t;
-  echo ./dev/console
+	echo ./dev/console;
-  echo ./dev/fd
+	echo ./dev/fd;
-  echo ./dev/full
+	echo ./dev/full;
-  echo ./dev/null
+	echo ./dev/null;
-  echo ./dev/ptmx
+	echo ./dev/ptmx;
-  echo ./dev/pts/
+	echo ./dev/pts/;
-  echo ./dev/random
+	echo ./dev/random;
-  echo ./dev/shm/
+	echo ./dev/shm/;
-  echo ./dev/stderr
+	echo ./dev/stderr;
-  echo ./dev/stdin
+	echo ./dev/stdin;
-  echo ./dev/stdout
+	echo ./dev/stdout;
-  echo ./dev/tty
+	echo ./dev/tty;
-  echo ./dev/urandom
+	echo ./dev/urandom;
-  echo ./dev/zero
+	echo ./dev/zero;
 } | sort | diff -u tar1.txt -

tests/skip-output-mknod

@@ -18,13 +18,13 @@ fi
 # entries in /dev are different depending on whether mknod is available or not
 $prefix {{ CMD }} --mode={{ MODE }} --variant=apt --skip=output/mknod \
 	{{ DIST }} - {{ MIRROR }} | {
-  tar -t
+	tar -t;
-  echo ./dev/console
+	echo ./dev/console;
-  echo ./dev/full
+	echo ./dev/full;
-  echo ./dev/null
+	echo ./dev/null;
-  echo ./dev/ptmx
+	echo ./dev/ptmx;
-  echo ./dev/random
+	echo ./dev/random;
-  echo ./dev/tty
+	echo ./dev/tty;
-  echo ./dev/urandom
+	echo ./dev/urandom;
-  echo ./dev/zero
+	echo ./dev/zero;
 } | sort | diff -u tar1.txt -

tests/special-hooks-using-helpers

@@ -8,21 +8,9 @@ mkdir /tmp/root/real
 run_testA() {
   echo content > /tmp/foo
   # shellcheck disable=SC2094
-  {
+  { { { {{ CMD }} --hook-helper /tmp/root root setup '' 1 upload /tmp/foo "$1" < /tmp/myfifo 3>&-; echo $? >&3; printf "\\000\\000adios";
-    {
+      } | {{ CMD }} --hook-listener 1 3>&- >/tmp/myfifo; echo $?; } 3>&1;
-      {
+  } | { read -r xs1; [ "$xs1" -eq 0 ]; read -r xs2; [ "$xs2" -eq 0 ]; }
-        {{ CMD }} --hook-helper /tmp/root root setup '' 1 upload /tmp/foo "$1" </tmp/myfifo 3>&-
-        echo $? >&3
-        printf '\000\000adios'
-      } | {{ CMD }} --hook-listener 1 3>&- >/tmp/myfifo
-      echo $?
-    } 3>&1
-  } | {
-    read -r xs1
-    [ "$xs1" -eq 0 ]
-    read -r xs2
-    [ "$xs2" -eq 0 ]
-  }
   echo content | diff -u - /tmp/root/real/foo
   rm /tmp/foo
   rm /tmp/root/real/foo

tests/tarfilter-idshift

@@ -1,111 +1,58 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-
+if [ ! -e /mmdebstrap-testenv ]; then
-trap "rm -f /tmp/mkpaxtar.pl /tmp/orig.tar /tmp/file /tmp/expected /tmp/filtered.tar" EXIT INT TERM
+	echo "this test modifies the system and should only be run inside a container" >&2
-
+	exit 1
-cat <<'END' >/tmp/mkpaxtar.pl
+fi
-#!/usr/bin/env perl
+trap "rm -f /tmp/debian-chroot.tar /tmp/debian-chroot-shifted.tar /tmp/debian-chroot.txt /tmp/debian-chroot-shiftedback.tar /tmp/expected; rm -rf /tmp/debian-chroot" EXIT INT TERM
-
+useradd --home-dir /home/user --create-home user
-use strict;
+echo user:100000:65536 | cmp /etc/subuid -
-use warnings;
+echo user:100000:65536 | cmp /etc/subgid -
-
+# include iputils-ping so that we can verify that tarfilter does not remove
-my @entries = (
+# extended attributes
-    # filename            mode      type content
+# run through tarshift no-op to create a tarball that should be bit-by-bit
-    ['./PaxHeaders/file', oct(644), 'x', "57 SCHILY.xattr.security.capability=\x01\0\0\x02\0\x20\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x0a"],
+# identical to a round trip through "tarfilter --idshift X" and "tarfilter --idshift -X"
-    ['./file',              oct(644), 0,   'test'],
+runuser -u user -- {{ CMD }} --mode=unshare --variant=apt --include=iputils-ping {{ DIST }} - {{ MIRROR }} \
-);
+	| ./tarfilter --idshift 0 > /tmp/debian-chroot.tar
-
-my $num_entries = 0;
-
-foreach my $file (@entries) {
-    my ($fname, $mode, $type, $content) = @{$file};
-    my $entry = pack(
-        'a100 a8 a8 a8 a12 a12 A8 a1 a100 a6 a2 a32 a32 a8 a8 a155 x12',
-        $fname,
-        sprintf('%07o',  $mode),
-        sprintf('%07o',  0),                  # uid
-        sprintf('%07o',  0),                  # gid
-        sprintf('%011o', length $content),    # size
-        sprintf('%011o', 0),                  # mtime
-        '',                                   # checksum
-        $type,
-        '',                                   # linkname
-        "ustar",                              # magic
-        "00",                                 # version
-        '',                                   # username
-        '',                                   # groupname
-        '',                                   # dev major
-        '',                                   # dev minor
-        '',                                   # prefix
-    );
-
-    # compute and insert checksum
-    substr($entry, 148, 7)
-      = sprintf("%06o\0", unpack("%16C*", $entry));
-    print $entry;
-    $num_entries += 1;
-
-    if (length $content) {
-        print(pack 'a512', $content);
-        $num_entries += 1;
-    }
-}
-
-# https://www.gnu.org/software/tar/manual/html_node/Standard.html
-#
-# Physically, an archive consists of a series of file entries terminated by an
-# end-of-archive entry, which consists of two 512 blocks of zero bytes. At the
-# end of the archive file there are two 512-byte blocks filled with binary
-# zeros as an end-of-file marker.
-
-print(pack 'a512', '');
-print(pack 'a512', '');
-$num_entries += 2;
-
-# https://www.gnu.org/software/tar/manual/html_section/tar_76.html
-#
-# Some devices requires that all write operations be a multiple of a certain
-# size, and so, tar pads the archive out to the next record boundary.
-#
-# The default blocking factor is 20. With a block size of 512 bytes, we get a
-# record size of 10240.
-
-for (my $i = $num_entries ; $i < 20 ; $i++) {
-    print(pack 'a512', '');
-}
-END
-
-MMTARFILTER=
-[ -x /usr/bin/mmtarfilter ] && MMTARFILTER=/usr/bin/mmtarfilter
-[ -x ./tarfilter ] && MMTARFILTER=./tarfilter
-
-perl /tmp/mkpaxtar.pl | "$MMTARFILTER" >/tmp/orig.tar
 # make sure that xattrs are set in the original tarball
-tar --xattrs --xattrs-include='*' --directory /tmp/ -xf /tmp/orig.tar ./file
+mkdir /tmp/debian-chroot
-echo "/tmp/file cap_net_raw=ep" >/tmp/expected
+tar --xattrs --xattrs-include='*' --directory /tmp/debian-chroot -xf /tmp/debian-chroot.tar ./usr/bin/ping
-getcap /tmp/file | diff -u /tmp/expected - >&2
+echo "/tmp/debian-chroot/usr/bin/ping cap_net_raw=ep" > /tmp/expected
-# make sure that the file content is as expected
+getcap /tmp/debian-chroot/usr/bin/ping | diff -u /tmp/expected - >&2
-printf test | diff -u /tmp/file - >&2
+rm /tmp/debian-chroot/usr/bin/ping
-# make sure that uid/gid are as expected in the original tarball
+rmdir /tmp/debian-chroot/usr/bin
-echo "0 0 644" >/tmp/expected
+rmdir /tmp/debian-chroot/usr
-stat --format="%u %g %a" /tmp/file | diff -u /tmp/expected - >&2
+rmdir /tmp/debian-chroot
-rm /tmp/file
+# shift the uid/gid forward by 100000 and backward by 100000
-# tarball must be bit by-bit-identical after round-trip
+./tarfilter --idshift 100000 < /tmp/debian-chroot.tar > /tmp/debian-chroot-shifted.tar
-"$MMTARFILTER" --idshift 0 </tmp/orig.tar >/tmp/filtered.tar
+./tarfilter --idshift -100000 < /tmp/debian-chroot-shifted.tar > /tmp/debian-chroot-shiftedback.tar
-cmp /tmp/orig.tar /tmp/filtered.tar
+# the tarball before and after the roundtrip through tarfilter should be bit
-
+# by bit identical
-# now shift uid/gid
+cmp /tmp/debian-chroot.tar /tmp/debian-chroot-shiftedback.tar
-"$MMTARFILTER" --idshift 100000 </tmp/orig.tar >/tmp/filtered.tar
+# manually adjust uid/gid and compare "tar -t" output
-# make sure that uid/gid are as expected in the filtered tarball
+tar --numeric-owner -tvf /tmp/debian-chroot.tar \
-tar --xattrs --xattrs-include='*' --directory /tmp/ -xf /tmp/filtered.tar ./file
+	| sed 's# 42/0 # 100042/100000 #' \
-echo "100000 100000 644" >/tmp/expected
+	| sed 's# 0/0 # 100000/100000 #' \
-stat --format="%u %g %a" /tmp/file | diff -u /tmp/expected - >&2
+	| sed 's# 0/5 # 100000/100005 #' \
-rm /tmp/file
+	| sed 's# 0/8 # 100000/100008 #' \
-
+	| sed 's# 0/42 # 100000/100042 #' \
-# now shift uid/gid back to create a round-trip
+	| sed 's# 0/43 # 100000/100043 #' \
-"$MMTARFILTER" --idshift -100000 </tmp/filtered.tar >/tmp/filtered2.tar
+	| sed 's# 0/50 # 100000/100050 #' \
-
+	| sed 's/ \+/ /g' \
-# the result must be identical to the original and will thus also include the
+	> /tmp/debian-chroot.txt
-# correct xattr information
+tar --numeric-owner -tvf /tmp/debian-chroot-shifted.tar \
-cmp /tmp/orig.tar /tmp/filtered2.tar
+	| sed 's/ \+/ /g' \
+	| diff -u /tmp/debian-chroot.txt - >&2
+mkdir /tmp/debian-chroot
+tar --xattrs --xattrs-include='*' --directory /tmp/debian-chroot -xf /tmp/debian-chroot-shifted.tar
+echo "100000 100000" > /tmp/expected
+stat --format="%u %g" /tmp/debian-chroot/usr/bin/ping | diff -u /tmp/expected - >&2
+echo "/tmp/debian-chroot/usr/bin/ping cap_net_raw=ep" > /tmp/expected
+getcap /tmp/debian-chroot/usr/bin/ping | diff -u /tmp/expected - >&2
+echo "0 0" > /tmp/expected
+runuser -u user -- {{ CMD }} --unshare-helper /usr/sbin/chroot /tmp/debian-chroot stat --format="%u %g" /usr/bin/ping \
+	| diff -u /tmp/expected - >&2
+echo "/usr/bin/ping cap_net_raw=ep" > /tmp/expected
+runuser -u user -- {{ CMD }} --unshare-helper /usr/sbin/chroot /tmp/debian-chroot getcap /usr/bin/ping \
+	| diff -u /tmp/expected - >&2

tests/verbose

@@ -7,7 +7,9 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 
 # we use variant standard in verbose mode to see the maximum number of packages
 # that was chosen in case of USE_HOST_APT_CONFIG=yes
-case {{ VARIANT }} in standard) : ;; *) exit 1 ;; esac
+# we use variant important on arches where variant standard is not bit-by-bit
+# reproducible due to #1031276
+case {{ VARIANT }} in standard|-) : ;; *) exit 1;; esac
 
 {{ CMD }} --variant={{ VARIANT }} --verbose {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 

tests/without-etc-resolv-conf-and-etc-hostname

@@ -8,8 +8,7 @@ fi
 trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 rm /etc/resolv.conf /etc/hostname
 {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-{
+{ tar -tf /tmp/debian-chroot.tar;
-  tar -tf /tmp/debian-chroot.tar
+  printf "./etc/hostname\n";
-  printf "./etc/hostname\n"
+  printf "./etc/resolv.conf\n";
-  printf "./etc/resolv.conf\n"
 } | sort | diff -u tar1.txt -

tests/zombie-reaping

@@ -1,38 +0,0 @@
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-
-[ {{ MODE }} = "unshare" ]
-
-trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
-
-prefix=
-if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
-  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
-    if [ ! -e /mmdebstrap-testenv ]; then
-      echo "this test modifies the system and should only be run inside a container" >&2
-      exit 1
-    fi
-    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
-  fi
-  prefix="runuser -u ${SUDO_USER:-user} --"
-fi
-
-MMTARFILTER=
-[ -x /usr/bin/mmtarfilter ] && MMTARFILTER=/usr/bin/mmtarfilter
-[ -x ./tarfilter ] && MMTARFILTER=./tarfilter
-
-$prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
-  --skip=output/dev \
-  --customize-hook='chroot "$1" sh -c "sleep 1m > /dev/null" &' \
-  {{ DIST }} - {{ MIRROR }} \
-  | "$MMTARFILTER" --path-exclude="/dev" \
-    >/tmp/debian-chroot.tar
-
-origfilter() {
-  "$MMTARFILTER" <./cache/mmdebstrap-{{ DIST }}-apt.tar --path-exclude="/dev/*" --path-exclude="/dev"
-}
-
-origfilter | cmp - /tmp/debian-chroot.tar \
-  || origfilter | diffoscope - /tmp/debian-chroot.tar