release 1.3.7

cherry-picked from
e2cd601b6f

CHANGELOG.md

@@ -1,3 +1,13 @@
+1.3.7 (2023-06-21)
+------------------
+
+ - add hooks/copy-host-apt-sources-and-preferences
+
+1.3.6 (2023-06-16)
+------------------
+
+ - bugfix release
+
 1.3.5 (2023-03-20)
 ------------------
 

README.md

@@ -23,6 +23,11 @@ For the full documentation use:
 
     pod2man ./mmdebstrap | man -l -
 
+Or read a HTML version of the man page in either of these locations:
+
+ - https://gitlab.mister-muffin.de/josch/mmdebstrap/wiki
+ - https://manpages.debian.org/unstable/mmdebstrap/mmdebstrap.1.en.html
+
 The sales pitch in comparison to debootstrap
 --------------------------------------------
 
@@ -159,6 +164,7 @@ Contributors
  - Josh Triplett
  - Konstantin Demin
  - David Kalnischkies
+ - Emilio Pozuelo Monfort
  - Jakub Wilk
  - Joe Groocock
  - Nicolas Vigier

caching_proxy.py

@@ -57,6 +57,8 @@ class ProxyRequestHandler(http.server.BaseHTTPRequestHandler):
             self.send_header("Content-Length", oldpath.stat().st_size)
             self.end_headers()
             with oldpath.open(mode="rb") as old, newpath.open(mode="wb") as new:
+                # we are not using shutil.copyfileobj() because we want to
+                # write to two file objects simultaneously
                 while True:
                     buf = old.read(64 * 1024)  # same as shutil uses
                     if not buf:
@@ -81,6 +83,9 @@ class ProxyRequestHandler(http.server.BaseHTTPRequestHandler):
                 self.send_header(k, v)
             self.end_headers()
             with newpath.open(mode="wb") as f:
+                # we are not using shutil.copyfileobj() because we want to
+                # write to two file objects simultaneously and throttle the
+                # writing speed to 1024 kB/s
                 while True:
                     buf = res.read(64 * 1024)  # same as shutil uses
                     if not buf:

coverage.sh

@@ -45,10 +45,11 @@ rm -f shared/cover_db.img
 
 if [ "$HAVE_QEMU" = "yes" ]; then
 	# prepare image for cover_db
-	guestfish -N shared/cover_db.img=disk:64M -- mkfs vfat /dev/sda
+	fallocate -l 64M shared/cover_db.img
+	/usr/sbin/mkfs.vfat shared/cover_db.img
 
-	if [ ! -e "./shared/cache/debian-$DEFAULT_DIST.qcow" ]; then
-		echo "./shared/cache/debian-$DEFAULT_DIST.qcow does not exist" >&2
+	if [ ! -e "./shared/cache/debian-$DEFAULT_DIST.ext4" ]; then
+		echo "./shared/cache/debian-$DEFAULT_DIST.ext4 does not exist" >&2
 		exit 1
 	fi
 fi

coverage.txt

@@ -3,21 +3,18 @@ Dists: any
 Variants: minbase buildd -
 Needs-Root: true
 Needs-APT-Config: true
-Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: check-against-debootstrap-dist
 Dists: any
 Variants: minbase buildd -
 Needs-Root: true
 Needs-APT-Config: true
-Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: as-debootstrap-unshare-wrapper
 Modes: unshare
 Needs-Root: true
 Variants: minbase -
 Needs-APT-Config: true
-Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: help
 
@@ -65,10 +62,7 @@ Modes: root
 Formats: tar squashfs ext2
 Variants: essential apt minbase buildd - standard
 Skip-If:
- variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
- variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
- fmt == "squashfs" and dist == "oldstable" # squashfs-tools-ng is not available
- fmt == "ext2" and dist == "oldstable" # genext2fs does not support SOURCE_DATE_EPOCH
+ variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
  mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
  variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
@@ -77,16 +71,12 @@ Modes: unshare fakechroot
 Formats: tar squashfs ext2
 Variants: essential apt minbase buildd - standard
 Skip-If:
- variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
- variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
- fmt == "squashfs" and dist == "oldstable" # squashfs-tools-ng is not available
- fmt == "ext2" and dist == "oldstable" # genext2fs does not support SOURCE_DATE_EPOCH
+ variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
  mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
  variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
 Test: tarfilter-idshift
 Needs-QEMU: true
-Skip-If: dist == "oldstable" # python3 tarfile module does not preserve xattrs
 
 Test: progress-bars-on-fake-tty
 
@@ -321,8 +311,7 @@ Test: compare-output-with-pre-seeded-var-cache-apt-archives
 Needs-QEMU: true
 Variants: any
 Skip-If:
- variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
- variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
+ variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
 
 Test: create-directory-dry-run
 Modes: root
@@ -346,20 +335,20 @@ Variants: essential
 Modes: chrootless
 Needs-Root: true
 Skip-If:
- dist in ["oldstable", "stable"]
+ dist == "oldstable"
 
 Test: chrootless-fakeroot
 Variants: essential
 Modes: chrootless
 Skip-If:
- dist in ["oldstable", "stable"]
+ dist == "oldstable"
  hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
 
 Test: chrootless-foreign
 Variants: essential
 Modes: chrootless
 Skip-If:
- dist in ["oldstable", "stable"]
+ dist == "oldstable"
  hostarch not in ["amd64", "arm64"]
  not run_ma_same_tests
 Needs-QEMU: true

gpgvnoexpkeysig

@@ -1,6 +1,7 @@
 #!/bin/sh
 #
-# This script is in the public domain
+# No copyright is claimed.  This code is in the public domain; do with
+# it what you wish.
 #
 # Author: Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
 #

hooks/copy-host-apt-sources-and-preferences/customize00.pl

@@ -0,0 +1,44 @@
+#!/usr/bin/perl
+#
+# This script makes sure that all packages that are installed both locally as
+# well as inside the chroot have the same version.
+#
+# It is implemented in Perl because there are no associative arrays in POSIX
+# shell.
+
+use strict;
+use warnings;
+
+sub get_pkgs {
+    my $root = shift;
+    my %pkgs = ();
+    open(my $fh, '-|', 'dpkg-query', "--root=$root", '--showformat',
+        '${binary:Package}=${Version}\n', '--show')
+      // die "cannot exec dpkg-query";
+    while (my $line = <$fh>) {
+        my ($pkg, $ver) = split(/=/, $line, 2);
+        $pkgs{$pkg} = $ver;
+    }
+    close $fh;
+    if ($? != 0) { die "failed to run dpkg-query" }
+    return %pkgs;
+}
+
+my %pkgs_local  = get_pkgs('/');
+my %pkgs_chroot = get_pkgs($ARGV[0]);
+
+my @diff = ();
+foreach my $pkg (keys %pkgs_chroot) {
+    next unless exists $pkgs_local{$pkg};
+    if ($pkgs_local{$pkg} ne $pkgs_chroot{$pkg}) {
+        push @diff, $pkg;
+    }
+}
+
+if (scalar @diff > 0) {
+    print STDERR "E: packages from the host and the chroot differ:\n";
+    foreach my $pkg (@diff) {
+	print STDERR "E: $pkg $pkgs_local{$pkg} $pkgs_chroot{$pkg}\n";
+    }
+    exit 1;
+}

hooks/copy-host-apt-sources-and-preferences/setup00.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
+
+if [ -n "${MMDEBSTRAP_SUITE:-}" ]; then
+	if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 1 ]; then
+		echo "W: using a non-empty suite name $MMDEBSTRAP_SUITE does not make sense with this hook and might select the wrong Essential:yes package set" >&2
+	fi
+fi
+
+rootdir="$1"
+
+SOURCELIST="/etc/apt/sources.list"
+eval "$(apt-config shell SOURCELIST Dir::Etc::SourceList/f)"
+SOURCEPARTS="/etc/apt/sources.d/"
+eval "$(apt-config shell SOURCEPARTS Dir::Etc::SourceParts/d)"
+PREFERENCES="/etc/apt/preferences"
+eval "$(apt-config shell PREFERENCES Dir::Etc::Preferences/f)"
+PREFERENCESPARTS="/etc/apt/preferences.d/"
+eval "$(apt-config shell PREFERENCESPARTS Dir::Etc::PreferencesParts/d)"
+
+for f in "$SOURCELIST" \
+	"$SOURCEPARTS"/*.list \
+	"$SOURCEPARTS"/*.sources \
+	"$PREFERENCES" \
+	"$PREFERENCESPARTS"/*; do
+	[ -e "$f" ] || continue
+	if [ -e "$rootdir/$f" ]; then
+		if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 2 ]; then
+			echo "I: $f already exists in chroot, appending..." >&2
+		fi
+		echo >> "$rootdir/$f"
+	fi
+	cat "$f" >> "$rootdir/$f"
+	if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+		echo "D: contents of $f inside the chroot:" >&2
+		cat "$rootdir/$f" >&2
+	fi
+done

hooks/eatmydata/customize.sh

@@ -17,10 +17,10 @@ libdir="/usr/lib/$(dpkg-architecture -a "$chrootarch" -q DEB_HOST_MULTIARCH)"
 
 # if eatmydata was actually installed properly, then we are not removing
 # anything here
-if ! chroot "$rootdir" dpkg-query --list eatmydata; then
+if ! chroot "$rootdir" dpkg-query --show eatmydata; then
 	rm "$rootdir/usr/bin/eatmydata"
 fi
-if ! chroot "$rootdir" dpkg-query --list libeatmydata1; then
+if ! chroot "$rootdir" dpkg-query --show libeatmydata1; then
 	rm "$rootdir$libdir"/libeatmydata.so*
 fi
 

make_mirror.sh

@@ -56,24 +56,16 @@ deletecache() {
 			fi
 			;;
 		esac
-		case "$dist" in
-			oldstable)
-				if [ -e "$dir/debian-security/dists/$dist/updates" ]; then
-					rm --one-file-system --recursive "$dir/debian-security/dists/$dist/updates"
-				else
-					echo "does not exist: $dir/debian-security/dists/$dist/updates" >&2
-				fi
-				;;
-			stable)
-				if [ -e "$dir/debian-security/dists/$dist-security" ]; then
-					rm --one-file-system --recursive "$dir/debian-security/dists/$dist-security"
-				else
-					echo "does not exist: $dir/debian-security/dists/$dist-security" >&2
-				fi
-				;;
+		case "$dist" in oldstable|stable)
+			if [ -e "$dir/debian-security/dists/$dist-security" ]; then
+				rm --one-file-system --recursive "$dir/debian-security/dists/$dist-security"
+			else
+				echo "does not exist: $dir/debian-security/dists/$dist-security" >&2
+			fi
+			;;
 		esac
 	done
-	for f in "$dir/debian-"*.qcow; do
+	for f in "$dir/debian-"*.ext4; do
 		if [ -e "$f" ]; then
 			rm --one-file-system "$f"
 		fi
@@ -112,7 +104,6 @@ deletecache() {
 }
 
 cleanup_newcachedir() {
-	kill "$PROXYPID" || :
 	echo "running cleanup_newcachedir"
 	deletecache "$newcachedir"
 }
@@ -163,7 +154,7 @@ update_cache() (
 	# we only set this trap here and overwrite the previous trap, because
 	# the update_cache function is run as part of a pipe and thus in its
 	# own process which will EXIT after it finished
-	trap "cleanupapt" EXIT INT TERM
+	trap 'kill "$PROXYPID" || :;cleanupapt' EXIT INT TERM
 
 	for p in /etc/apt/apt.conf.d /etc/apt/sources.list.d /etc/apt/preferences.d /var/cache/apt/archives /var/lib/apt/lists/partial /var/lib/dpkg; do
 		mkdir -p "$rootdir/$p"
@@ -225,7 +216,7 @@ END
 		cat "$rootdir/$f" >&2
 	done
 
-	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update
+	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update --error-on=any
 
 	pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \
 		--format '$(FILENAME)' 'Created-By: Packages' "Architecture: $nativearch" \
@@ -240,7 +231,7 @@ END
 	# we need usr-is-merged to simulate debootstrap behaviour for all dists
 	# starting from Debian 12 (Bullseye)
 	case "$dist" in
-		oldstable|stable) : ;;
+		oldstable) : ;;
 		*) pkgs="$pkgs usr-is-merged usrmerge" ;;
 	esac
 
@@ -322,6 +313,7 @@ fi
 
 ./caching_proxy.py "$oldcachedir" "$newcachedir" &
 PROXYPID=$!
+trap 'kill "$PROXYPID" || :' EXIT INT TERM
 
 for i in $(seq 10); do
 	curl --proxy "http://127.0.0.1:8080/" --silent -o /dev/null "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" && break
@@ -333,7 +325,7 @@ if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
 	exit 1
 fi
 
-trap "cleanup_newcachedir" EXIT INT TERM
+trap 'kill "$PROXYPID" || :;cleanup_newcachedir' EXIT INT TERM
 
 mkdir -p "$newcachedir"
 touch "$newcachedir/mmdebstrapcache"
@@ -369,16 +361,8 @@ for dist in oldstable stable testing unstable; do
 		echo "deb [arch=$nativearch] $mirror $dist $components" | update_cache "$dist" "$nativearch"
 		# we need to include the base mirror again or otherwise
 		# packages like build-essential will be missing
-		case "$dist" in
-			oldstable)
-				cat << END | update_cache "$dist" "$nativearch"
-deb [arch=$nativearch] $mirror $dist $components
-deb [arch=$nativearch] $mirror $dist-updates main
-deb [arch=$nativearch] $security_mirror $dist/updates main
-END
-				;;
-			stable)
-				cat << END | update_cache "$dist" "$nativearch"
+		case "$dist" in oldstable|stable)
+			cat << END | update_cache "$dist" "$nativearch"
 deb [arch=$nativearch] $mirror $dist $components
 deb [arch=$nativearch] $mirror $dist-updates main
 deb [arch=$nativearch] $security_mirror $dist-security main
@@ -417,10 +401,7 @@ cleanuptmpdir() {
 	if [ ! -e "$tmpdir" ]; then
 		return
 	fi
-	for f in "$tmpdir/worker.sh" \
-		"$tmpdir/mini-httpd" "$tmpdir/hosts" \
-		"$tmpdir/debian-chroot.tar" \
-		"$tmpdir/mmdebstrap.service"; do
+	for f in "$tmpdir/worker.sh" "$tmpdir/mmdebstrap.service"; do
 		if [ ! -e "$f" ]; then
 			echo "does not exist: $f" >&2
 			continue
@@ -434,17 +415,6 @@ SOURCE_DATE_EPOCH="$(date --date="$(grep-dctrl -s Date -n '' "$newmirrordir/dist
 export SOURCE_DATE_EPOCH
 
 if [ "$HAVE_QEMU" = "yes" ]; then
-	case "$HOSTARCH" in
-		amd64|i386|arm64)
-			# okay
-			;;
-		*)
-			echo "qemu support is only available on amd64, i386 and arm64" >&2
-			echo "because grub is only available on those arches" >&2
-			exit 1
-			;;
-	esac
-
 	# we use the caching proxy again when building the qemu image
 	#  - we can re-use the packages that were already downloaded earlier
 	#  - we make sure that the qemu image uses the same Release file even
@@ -464,52 +434,26 @@ if [ "$HAVE_QEMU" = "yes" ]; then
 		exit 1
 	fi
 
-	# We must not use any --dpkgopt here because any dpkg options still
-	# leak into the chroot with chrootless mode.
-	# We do not use our own package cache here because
-	#   - it doesn't (and shouldn't) contain the extra packages
-	#   - it doesn't matter if the base system is from a different mirror timestamp
-	# procps is needed for /sbin/sysctl
 	tmpdir="$(mktemp -d)"
-	trap "cleanuptmpdir; cleanup_newcachedir" EXIT INT TERM
+	trap 'kill "$PROXYPID" || :;cleanuptmpdir; cleanup_newcachedir' EXIT INT TERM
 
-	pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,grub-efi
-	if [ "$DEFAULT_DIST" != "oldstable" ]; then
-		pkgs="$pkgs,squashfs-tools-ng,genext2fs"
-	fi
+	pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,disorderfs,squashfs-tools-ng,genext2fs,linux-image-generic
 	if [ ! -e ./mmdebstrap ]; then
 		pkgs="$pkgs,mmdebstrap"
 	fi
-	case "$HOSTARCH" in
-		amd64|arm64)
-			pkgs="$pkgs,linux-image-$HOSTARCH"
-			;;
-		i386)
-			pkgs="$pkgs,linux-image-686"
-			;;
-		ppc64el)
-			pkgs="$pkgs,linux-image-powerpc64le"
-			;;
-		*)
-			echo "no kernel image for $HOSTARCH" >&2
-			exit 1
-			;;
-	esac
-	if [ "$HOSTARCH" = amd64 ] && [ "$RUN_MA_SAME_TESTS" = "yes" ]; then
-		arches=amd64,arm64
-		pkgs="$pkgs,libfakechroot:arm64,libfakeroot:arm64"
-	elif [ "$HOSTARCH" = arm64 ] && [ "$RUN_MA_SAME_TESTS" = "yes" ]; then
-		arches=arm64,amd64
-		pkgs="$pkgs,libfakechroot:amd64,libfakeroot:amd64"
-	else
-		arches=$HOSTARCH
+	arches=$HOSTARCH
+	if [ "$RUN_MA_SAME_TESTS" = "yes" ]; then
+		case "$HOSTARCH" in
+			amd64)
+				arches=amd64,arm64
+				pkgs="$pkgs,libfakechroot:arm64,libfakeroot:arm64"
+				;;
+			arm64)
+				arches=arm64,amd64
+				pkgs="$pkgs,libfakechroot:amd64,libfakeroot:amd64"
+				;;
+		esac
 	fi
-	$CMD --variant=apt --architectures="$arches" --include="$pkgs" \
-		--setup-hook='echo "Acquire::http::Proxy \"http://127.0.0.1:8080/\";" > "$1/etc/apt/apt.conf.d/00proxy"' \
-		--customize-hook='rm "$1/etc/apt/apt.conf.d/00proxy"' \
-		"$DEFAULT_DIST" - "$mirror" > "$tmpdir/debian-chroot.tar"
-
-	kill $PROXYPID
 
 	cat << END > "$tmpdir/mmdebstrap.service"
 [Unit]
@@ -569,85 +513,32 @@ umount /mnt
 systemctl poweroff
 END
 	chmod +x "$tmpdir/worker.sh"
-	# initially we serve from the new cache so that debootstrap can grab
-	# the new package repository and not the old
-	cat << END > "$tmpdir/mini-httpd"
-START=1
-DAEMON_OPTS="-h 127.0.0.1 -p 80 -u nobody -dd /mnt/$newcache -i /var/run/mini-httpd.pid -T UTF-8"
-END
-	cat << 'END' > "$tmpdir/hosts"
-127.0.0.1 localhost
-END
-	#libguestfs-test-tool
-	#export LIBGUESTFS_DEBUG=1 LIBGUESTFS_TRACE=1
-	#
-	# In case the rootfs was prepared in fakechroot mode, ldconfig has to
-	# run to populate /etc/ld.so.cache or otherwise fakechroot tests will
-	# fail to run.
-	#
-	# The disk size is sufficient in most cases. Sometimes, gcc will do
-	# an upload with unstripped executables to make tracking down ICEs much
-	# easier (see #872672, #894014). During times with unstripped gcc, the
-	# buildd variant will not be 400MB but 1.3GB large and needs a 10G
-	# disk.
 	if [ -z ${DISK_SIZE+x} ]; then
 		DISK_SIZE=10G
 	fi
-	case "$HOSTARCH" in
-		amd64) GRUB_TARGET=x86_64-efi;;
-		i386) GRUB_TARGET=i386-efi;;
-		arm64) GRUB_TARGET=arm64-efi;;
-	esac
-	case "$HOSTARCH" in
-		arm64) SERIAL="loglevel=3 console=tty0 console=ttyAMA0,115200n8" ;;
-		*) SERIAL="loglevel=3 console=tty0 console=ttyS0,115200n8" ;;
-	esac
-	guestfish -- \
-		disk-create "$newcachedir/debian-$DEFAULT_DIST.qcow" qcow2 "$DISK_SIZE" : \
-		add-drive "$newcachedir/debian-$DEFAULT_DIST.qcow" format:qcow2 : \
-		launch : \
-		part-init /dev/sda gpt : \
-		part-add /dev/sda primary 8192 262144 : \
-		part-add /dev/sda primary 262145 -34 : \
-		part-set-gpt-type /dev/sda 1 C12A7328-F81F-11D2-BA4B-00A0C93EC93B  : \
-		mkfs ext2 /dev/sda2 : \
-		mount /dev/sda2 / : \
-		tar-in "$tmpdir/debian-chroot.tar" / xattrs:true : \
-		mkdir-p /boot/efi : \
-		mkfs vfat /dev/sda1 : \
-		mount /dev/sda1 /boot/efi : \
-		command /sbin/ldconfig : \
-		mkdir-p /etc/systemd/system/multi-user.target.wants : \
-		ln-s ../mmdebstrap.service /etc/systemd/system/multi-user.target.wants/mmdebstrap.service : \
-		copy-in "$tmpdir/mmdebstrap.service" /etc/systemd/system/ : \
-		copy-in "$tmpdir/worker.sh" / : \
-		copy-in "$tmpdir/mini-httpd" /etc/default : \
-		copy-in "$tmpdir/hosts" /etc/ : \
-		touch /mmdebstrap-testenv : \
-		command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda2) / ext4 errors=remount-ro 0 1 > /etc/fstab'" : \
-		command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda1) /boot/efi vfat errors=remount-ro 0 2 >> /etc/fstab'" : \
-		command "sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=/GRUB_CMDLINE_LINUX_DEFAULT=\"biosdevname=0 net.ifnames=0 consoleblank=0 rw $SERIAL\"/' /etc/default/grub" : \
-		command "update-initramfs -u" : \
-		command "grub-mkconfig -o /boot/grub/grub.cfg" : \
-		command "grub-install /dev/sda --target=$GRUB_TARGET --no-nvram --force-extra-removable --no-floppy --modules=part_gpt --grub-mkdevicemap=/boot/grub/device.map" : \
-		sync : \
-		umount /boot/efi : \
-		umount / : \
-		shutdown
+	# set PATH to pick up the correct mmdebstrap variant
+	env PATH="$(dirname "$(realpath --canonicalize-existing "$CMD")"):$PATH" \
+		debvm-create --skip=usrmerge --size="$DISK_SIZE" \
+		--release="$DEFAULT_DIST" --skip=usrmerge \
+		--output="$newcachedir/debian-$DEFAULT_DIST.ext4" -- \
+		--architectures="$arches" --include="$pkgs" \
+		--setup-hook='echo "Acquire::http::Proxy \"http://127.0.0.1:8080/\";" > "$1/etc/apt/apt.conf.d/00proxy"' \
+		--hook-dir=/usr/share/mmdebstrap/hooks/maybe-merged-usr \
+		--customize-hook='rm "$1/etc/apt/apt.conf.d/00proxy"' \
+		--customize-hook='mkdir -p "$1/etc/systemd/system/multi-user.target.wants"' \
+		--customize-hook='ln -s ../mmdebstrap.service "$1/etc/systemd/system/multi-user.target.wants/mmdebstrap.service"' \
+		--customize-hook='touch "$1/mmdebstrap-testenv"' \
+		--customize-hook='copy-in "'"$tmpdir"'/mmdebstrap.service" /etc/systemd/system/' \
+		--customize-hook='copy-in "'"$tmpdir"'/worker.sh" /' \
+		--customize-hook='printf 127.0.0.1 localhost > "$1/etc/hosts"' \
+		--customize-hook='printf "START=1\nDAEMON_OPTS=\"-h 127.0.0.1 -p 80 -u nobody -dd /mnt/cache -i /var/run/mini-httpd.pid -T UTF-8\"\n" > "$1/etc/default/mini-httpd"' \
+		"$mirror"
+
+	kill $PROXYPID
 	cleanuptmpdir
 	trap "cleanup_newcachedir" EXIT INT TERM
 fi
 
-if [ "$HAVE_QEMU" = "yes" ]; then
-	# now replace the minihttpd config with one that serves the new repository
-	guestfish -a "$newcachedir/debian-$DEFAULT_DIST.qcow" -i <<EOF
-upload -<<END /etc/default/mini-httpd
-START=1
-DAEMON_OPTS="-h 127.0.0.1 -p 80 -u nobody -dd /mnt/cache -i /var/run/mini-httpd.pid -T UTF-8"
-END
-EOF
-fi
-
 # delete possibly leftover symlink
 if [ -e ./shared/cache.tmp ]; then
 	rm ./shared/cache.tmp

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '1.3.5';
+our $VERSION = '1.3.7';
 
 use English;
 use Getopt::Long;
@@ -1613,13 +1613,16 @@ sub run_hooks {
     if (length $ENV{APT_CONFIG}) {
         push @env_opts, "MMDEBSTRAP_APT_CONFIG=$ENV{APT_CONFIG}";
     }
-    # I hook script that wants to call mmdebstrap with --hook-helper needs to
+    # A hook script that wants to call mmdebstrap with --hook-helper needs to
     # know how mmdebstrap was executed
     push @env_opts, "MMDEBSTRAP_ARGV0=$PROGRAM_NAME";
     # Storing the mode is important for hook scripts to potentially change
     # their behavior depending on the mode. It's also important for when the
     # hook wants to use the mmdebstrap --hook-helper.
     push @env_opts, "MMDEBSTRAP_MODE=$options->{mode}";
+    if (defined $options->{suite}) {
+        push @env_opts, "MMDEBSTRAP_SUITE=$options->{suite}";
+    }
     # Storing the hook name is important for hook scripts to potentially change
     # their behavior depending on the hook. It's also important for when the
     # hook wants to use the mmdebstrap --hook-helper.
@@ -6985,7 +6988,7 @@ the numerical verbosity level (0 for no output, 1 for normal, 2 for verbose and
 3 for debug output). The C<MMDEBSTRAP_INCLUDE> variable stores the list of
 packages, apt patterns or file paths given by the B<--include> option,
 separated by a comma and with commas and percent signs in the option values
-urlencoded.
+urlencoded. If I<SUITE> name was supplied, it's stored in C<MMDEBSTRAP_SUITE>.
 
 In special hooks, the paths inside the chroot are relative to the root
 directory of the chroot.  The path on the outside is relative to current
@@ -7630,6 +7633,27 @@ To change compression specific options, either use the respecitve environment
 variables like B<XZ_OPT> or send B<mmdebstrap> output to your compressor of
 choice with a pipe.
 
+=head1 WRAPPERS
+
+=head2 debvm
+
+B<debvm> helps create and run virtual machines for various Debian releases and
+architectures.  The tool B<debvm-create> can be used to create a virtual
+machine image and the tool B<debvm-run> can be used to run such a machine
+image. Their purpose primarily is testing software using qemu as a containment
+technology.  These are relatively thin wrappers around B<mmdebstrap> and
+B<qemu>.
+
+=head2 bdebstrap
+
+B<bdebstrap> is a YAML config based multi-mirror Debian chroot creation tool.
+B<bdebstrap> is an alternative to B<debootstrap> and a wrapper around
+B<mmdebstrap> to support YAML based configuration files. It inherits all
+benefits from B<mmdebstrap>. The support for configuration allows storing all
+customization in a YAML file instead of having to use a very long one-liner
+call to B<mmdebstrap>. It also layering multiple customizations on top of each
+other, e.g. to support flavors of an image.
+
 =head1 BUGS
 
 https://gitlab.mister-muffin.de/josch/mmdebstrap/issues
@@ -7654,7 +7678,7 @@ update" as an error.
 
 =head1 SEE ALSO
 
-    debootstrap(8)
+    debootstrap(8), debvm(1), bdebstrap(1)
 
 =cut
 

mmdebstrap-autopkgtest-build-qemu

@@ -1,255 +0,0 @@
-#!/bin/sh
-#
-# © 2022 Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to
-# deal in the Software without restriction, including without limitation the
-# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-# sell copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# The software is provided "as is", without warranty of any kind, express or
-# implied, including but not limited to the warranties of merchantability,
-# fitness for a particular purpose and noninfringement. In no event shall the
-# authors or copyright holders be liable for any claim, damages or other
-# liability, whether in an action of contract, tort or otherwise, arising
-# from, out of or in connection with the software or the use or other dealings
-# in the software.
-
-set -eu
-
-# This script creates debian-$RELEASE.qcow2 in the current directory which can
-# then be used by the autopkgtest qemu backend.
-#
-# Thanks to Francesco Poli for providing ideas and testing this.
-#
-# Thanks to Lars Wirzenius of vmdb2 where the grub and efi magic comes from.
-#
-# Only the native architecture is supported because guestfish doesn't support
-# foreign architectures.
-
-usage() {
-	echo "Usage: $0 [--size=SIZE] [--boot=BOOT] RELEASE IMAGE" >&2
-	echo >&2
-	echo "RELEASE is a Debian release like unstable" >&2
-	echo "IMAGE will be stored in qcow2 format" >&2
-	echo "SIZE is 25G by default" >&2
-	echo "BOOT is either auto (the default), bios, efi or ieee1275" >&2
-}
-
-nativearch="$(dpkg --print-architecture)"
-
-SIZE="25G" # default from autopkgtest-build-qemu
-BOOT="auto"
-if [ "$#" -lt 2 ]; then
-	echo "Error: Insufficient number of arguments" >&2
-	usage
-	exit 1
-elif [ "$#" -eq 2 ]; then
-	RELEASE=$1
-	IMAGE=$2
-else
-	# parse options
-	OPTS=$(getopt -n "$0" -o h --long size:,boot:,architecture:,help -- "$@")
-	if [ "$?" -ne 0 ]; then
-		echo "Error: Cannot parse arguments" >&2
-		usage
-		exit 1
-	fi
-	eval set -- "$OPTS"
-	while true; do
-		case "$1" in
-			--size) SIZE="$2"; shift 2; continue;;
-			--boot) BOOT="$2"; shift 2; continue;;
-			--help) usage; exit 1;;
-			--architecture)
-				echo "Error: cannot (yet) create foreign architecture images" >&2
-				exit 1
-				;;
-			--) shift; break;;
-			*)
-				echo "Error: unknown option $1" >&2
-				usage
-				exit 1
-				;;
-		esac
-	done
-	RELEASE=$1
-	IMAGE=$2
-fi
-
-# By default with --boot=auto (the default), bios boot is chosen for
-# amd64 and i386. Compare /usr/share/autopkgtest/lib/autopkgtest_qemu.py
-# But in practice, amd64 and i386 also support efi boot. But then
-# autopkgtest-virt-qemu has to be run with --boot=efi
-case "$BOOT" in
-	auto)
-		case "$nativearch" in
-			amd64|i386) BOOT=bios;;
-			armhf|arm64) BOOT=efi;;
-			ppc64el) BOOT=ieee1275;;
-		esac
-		;;
-	bios)
-		case "$nativearch" in amd64|i386);;
-			*)
-				echo "bios booting only possible on amd64 and i386" >&2
-				exit 1
-				;;
-		esac
-		;;
-	efi)
-		case "$nativearch" in amd64|i386|armhf|arm64);;
-			*)
-				echo "efi booting only possible on amd64, i386, armhf and arm64" >&2
-				exit 1
-				;;
-		esac
-		;;
-	ieee1275)
-		if [ "$nativearch" != "ppc64el" ]; then
-			echo "ieee1275 booting only possible on ppc64el" >&2
-			exit 1
-		fi
-		;;
-	*)
-		echo "invalid value for --boot" >&2;;
-esac
-
-case "$nativearch" in
-	amd64)
-		[ $BOOT = bios ] || [ $BOOT = efi ]
-		if [ $BOOT = bios ]; then
-			include="linux-image-amd64 grub-pc"
-			grub_target="i386-pc"
-		elif [ $BOOT = efi ]; then
-			include="linux-image-amd64 grub-efi"
-			grub_target="x86_64-efi"
-		fi
-		;;
-	arm64)
-		[ $BOOT = efi ]
-		include="linux-image-arm64 grub-efi"
-		grub_target="arm64-efi"
-		;;
-	armhf)
-		[ $BOOT = efi ]
-		include="linux-image-armmp-lpae grub-efi"
-		grub_target="arm-efi"
-		;;
-	i386)
-		[ $BOOT = bios ] || [ $BOOT = efi ]
-		if [ $BOOT = bios ]; then
-			include="linux-image-686-pae grub-pc"
-			grub_target="i386-pc"
-		elif [ $BOOT = efi ]; then
-			include="linux-image-686-pae grub-efi"
-			grub_target="i386-efi"
-		fi
-		;;
-	ppc64el)
-		[ $BOOT = ieee1275 ]
-		include="linux-image-powerpc64le grub-ieee1275"
-		grub_target="powerpc-ieee1275"
-		;;
-	*)
-		echo "architecture $nativearch not yet supported" >&2
-		exit 1
-		;;
-esac
-
-case "$nativearch" in
-	arm64|armhf) serial="loglevel=3 console=tty0 console=ttyAMA0,115200n8" ;;
-	ppc64el) serial="loglevel=3 console=tty0 console=hvc0,115200n8" ;;
-	*) serial="loglevel=3 console=tty0 console=ttyS0,115200n8" ;;
-esac
-
-if ! command -v guestfish >/dev/null; then
-	echo "Error: requires guestfish being installed" >&2
-	exit 1
-fi
-
-if [ ! -e /usr/share/autopkgtest/setup-commands/setup-testbed ]; then
-	echo "Error: requires autopkgtest being installed" >&2
-	exit 1
-fi
-
-run_mmdebstrap() {
-	mmdebstrap --variant=important --include="$include" \
-	--customize-hook='chroot "$1" passwd --delete root' \
-	--customize-hook='chroot "$1" useradd --home-dir /home/user --create-home user' \
-	--customize-hook='chroot "$1" passwd --delete user' \
-	--customize-hook='echo host > "$1/etc/hostname"' \
-	--customize-hook='echo "127.0.0.1 localhost host" > "$1/etc/hosts"' \
-	--customize-hook='env AUTOPKGTEST_BUILD_QEMU=1 /usr/share/autopkgtest/setup-commands/setup-testbed "$1"' \
-	"$RELEASE" -
-}
-
-guestfish_bios() {
-	guestfish -- \
-	disk-create "$IMAGE" qcow2 "$SIZE" : \
-	add-drive "$IMAGE" format:qcow2 : \
-	launch : \
-	part-disk /dev/sda mbr : \
-	part-set-bootable /dev/sda 1 true : \
-	mkfs ext4 /dev/sda1 : mount /dev/sda1 / : \
-	tar-in - / xattrs:true : \
-	command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda1) / ext4 errors=remount-ro 0 1 > /etc/fstab'" : \
-	command "update-initramfs -u" : \
-	command "grub-mkconfig -o /boot/grub/grub.cfg" : \
-	command "grub-install /dev/sda --target=$grub_target --no-nvram --force-extra-removable --no-floppy --modules=part_gpt --grub-mkdevicemap=/boot/grub/device.map" : \
-	sync : umount / : shutdown
-}
-
-guestfish_efi() {
-	guestfish -- \
-	disk-create "$IMAGE" qcow2 "$SIZE" : \
-	add-drive "$IMAGE" format:qcow2 : \
-	launch : \
-	part-init /dev/sda gpt : \
-	part-add /dev/sda primary 8192 262144 : \
-	part-add /dev/sda primary 262145 -34 : \
-	part-set-gpt-type /dev/sda 1 C12A7328-F81F-11D2-BA4B-00A0C93EC93B  : \
-	mkfs ext4 /dev/sda2 : mount /dev/sda2 / : \
-	tar-in - / xattrs:true : \
-	mkdir-p /boot/efi : \
-	mkfs vfat /dev/sda1 : mount /dev/sda1 /boot/efi : \
-	command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda2) / ext4 errors=remount-ro 0 1 > /etc/fstab'" : \
-	command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda1) /boot/efi vfat errors=remount-ro 0 2 >> /etc/fstab'" : \
-	command "sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=/GRUB_CMDLINE_LINUX_DEFAULT=\"biosdevname=0 net.ifnames=0 consoleblank=0 rw $serial\"/' /etc/default/grub" : \
-	command "update-initramfs -u" : \
-	command "grub-mkconfig -o /boot/grub/grub.cfg" : \
-	command "grub-install /dev/sda --target=$grub_target --no-nvram --force-extra-removable --no-floppy --modules=part_gpt --grub-mkdevicemap=/boot/grub/device.map" : \
-	sync : umount /boot/efi : umount / : shutdown
-}
-
-guestfish_ieee1275() {
-	guestfish -- \
-	disk-create "$IMAGE" qcow2 "$SIZE" : \
-	add-drive "$IMAGE" format:qcow2 : \
-	launch : \
-	part-init /dev/sda gpt : \
-	part-add /dev/sda primary 8192 20480 : \
-	part-add /dev/sda primary 20481 -34 : \
-	part-set-gpt-type /dev/sda 1 9E1A2D38-C612-4316-AA26-8B49521E5A8B : \
-	mkfs ext4 /dev/sda2 : mount /dev/sda2 / : \
-	tar-in - / xattrs:true : \
-	command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda2) / ext4 errors=remount-ro 0 1 > /etc/fstab'" : \
-	command "sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=/GRUB_CMDLINE_LINUX_DEFAULT=\"biosdevname=0 net.ifnames=0 consoleblank=0 rw $serial\"/' /etc/default/grub" : \
-	command "update-initramfs -u" : \
-	command "grub-mkconfig -o /boot/grub/grub.cfg" : \
-	command "grub-install /dev/sda --target=$grub_target --no-nvram --force-extra-removable --no-floppy --modules=part_gpt --grub-mkdevicemap=/boot/grub/device.map" : \
-	sync : umount / : shutdown
-}
-
-case "$BOOT" in
-	bios) run_mmdebstrap | guestfish_bios;;
-	efi) run_mmdebstrap | guestfish_efi;;
-	ieee1275) run_mmdebstrap | guestfish_ieee1275;;
-esac
-
-echo "Success! The image is stored as $IMAGE" >&2

run_qemu.sh

@@ -8,7 +8,6 @@ tmpdir="$(mktemp -d)"
 
 cleanup() {
 	rv=$?
-	rm -f "$tmpdir/debian-$DEFAULT_DIST-overlay.qcow"
 	rm -f "$tmpdir/log"
 	[ -e "$tmpdir" ] && rmdir "$tmpdir"
 	if [ -n "${TAIL_PID:-}" ]; then
@@ -26,26 +25,6 @@ cleanup() {
 
 trap cleanup INT TERM EXIT
 
-ARCH=$(dpkg --print-architecture)
-case $ARCH in
-	i386)
-		MACHINE="accel=kvm:tcg"
-		CODE="/usr/share/OVMF/OVMF32_CODE_4M.secboot.fd"
-		QEMUARCH="i386"
-		;;
-	amd64)
-		MACHINE="accel=kvm:tcg"
-		CODE="/usr/share/OVMF/OVMF_CODE.fd"
-		QEMUARCH="x86_64"
-		;;
-	arm64)
-		MACHINE="type=virt,gic-version=host,accel=kvm"
-		CODE="/usr/share/AAVMF/AAVMF_CODE.fd"
-		QEMUARCH="aarch64"
-		;;
-	*) echo "qemu kvm not supported on $ARCH" >&2;;
-esac
-
 echo 1 > shared/exitstatus.txt
 if [ -e shared/output.txt ]; then
 	rm shared/output.txt
@@ -54,27 +33,18 @@ touch shared/output.txt
 tail -f shared/output.txt &
 TAIL_PID=$!
 
-# the path to debian-$DEFAULT_DIST.qcow must be absolute or otherwise qemu will
-# look for the path relative to debian-$DEFAULT_DIST-overlay.qcow
-qemu-img create -f qcow2 -b "$(realpath "$cachedir")/debian-$DEFAULT_DIST.qcow" -F qcow2 "$tmpdir/debian-$DEFAULT_DIST-overlay.qcow"
 # to connect to serial use:
 #   minicom -D 'unix#/tmp/ttyS0'
 #
 # or this (quit with ctrl+q):
 #   socat stdin,raw,echo=0,escape=0x11 unix-connect:/tmp/ttyS0
 ret=0
-timeout --foreground 40m qemu-system-"$QEMUARCH" \
-	-cpu host \
-	-no-user-config \
-	-M "$MACHINE" -m 4G -nographic \
-	-object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \
+timeout --foreground 40m debvm-run --image="$(realpath "$cachedir")/debian-$DEFAULT_DIST.ext4" -- \
+	-m 4G -snapshot \
 	-monitor unix:/tmp/monitor,server,nowait \
 	-serial unix:/tmp/ttyS0,server,nowait \
 	-serial unix:/tmp/ttyS1,server,nowait \
-	-net nic,model=virtio -net user \
-	-drive if=pflash,format=raw,unit=0,read-only=on,file="$CODE" \
 	-virtfs local,id=mmdebstrap,path="$(pwd)/shared",security_model=none,mount_tag=mmdebstrap \
-	-drive file="$tmpdir/debian-$DEFAULT_DIST-overlay.qcow",cache=unsafe,index=0,if=virtio \
 	>"$tmpdir/log" 2>&1 || ret=$?
 if [ "$ret" -ne 0 ]; then
 	cat "$tmpdir/log"

tests/check-against-debootstrap-dist

@@ -11,8 +11,8 @@ echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
 # https://bugs.debian.org/969631
 # we cannot use useradd because passwd is not Essential:yes
 {{ CMD }} --variant={{ VARIANT }} --mode={{ MODE }} \
-	--essential-hook='case {{ DIST }} in oldstable|stable) if [ {{ VARIANT }} = - ]; then echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd; fi;; esac' \
-	"$(case {{ DIST }} in oldstable|stable) echo --merged-usr ;; *) echo --hook-dir=./hooks/merged-usr ;; esac)" \
+	--essential-hook='[ {{ DIST }} = oldstable ] && [ {{ VARIANT }} = - ] && echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd || :' \
+	"$(if [ {{ DIST }} = oldstable ]; then echo --merged-usr; else echo --hook-dir=./hooks/merged-usr; fi)" \
 	{{ DIST }} /tmp/debian-{{ DIST }}-mm.tar {{ MIRROR }}
 
 mkdir /tmp/debian-{{ DIST }}-mm
@@ -108,9 +108,6 @@ if [ "{{ VARIANT }}" = "-" ]; then
 
 	cap=$(chroot /tmp/debian-{{ DIST }}-debootstrap /sbin/getcap /bin/ping)
 	expected="/bin/ping cap_net_raw=ep"
-	if [ "{{ DIST }}" = oldstable ]; then
-		expected="/bin/ping = cap_net_raw+ep"
-	fi
 	if [ "$cap" != "$expected" ]; then
 		echo "expected bin/ping to have capabilities $expected" >&2
 		echo "but debootstrap produced: $cap" >&2
@@ -148,14 +145,12 @@ done
 # Because of unreproducible uids (#969631) we created the _apt user ourselves
 # and because passwd is not Essential:yes we didn't use useradd. But newer
 # versions of adduser and shadow will create a different /etc/shadow
-if [ "{{ VARIANT }}" = "-" ]; then
-case {{ DIST }} in oldstable|stable)
-for f in shadow shadow-; do
-	if grep -q '^_apt:!:' /tmp/debian-{{ DIST }}-debootstrap/etc/$f; then
-		sed -i 's/^_apt:\*:\([^:]\+\):0:99999:7:::$/_apt:!:\1::::::/' /tmp/debian-{{ DIST }}-mm/etc/$f
-	fi
-done;;
-esac
+if [ "{{ VARIANT }}" = "-" ] && [ "{{ DIST}}" = oldstable ]; then
+	for f in shadow shadow-; do
+		if grep -q '^_apt:!:' /tmp/debian-{{ DIST }}-debootstrap/etc/$f; then
+			sed -i 's/^_apt:\*:\([^:]\+\):0:99999:7:::$/_apt:!:\1::::::/' /tmp/debian-{{ DIST }}-mm/etc/$f
+		fi
+	done
 fi
 
 for log in faillog lastlog; do
@@ -171,16 +166,12 @@ done
 # the order in which systemd and cron get installed differ and thus the order
 # of lines in /etc/group and /etc/gshadow differs
 if [ "{{ VARIANT }}" = "-" ]; then
-	case {{ DIST }} in testing|unstable)
-		for f in group group- gshadow gshadow-; do
-			cmp /tmp/debian-{{ DIST }}-mm/etc/$f /tmp/debian-{{ DIST }}-debootstrap/etc/$f 2>/dev/null && exit 1
-			for d in mm debootstrap; do
-				sort /tmp/debian-{{ DIST }}-$d/etc/$f > /tmp/debian-{{ DIST }}-$d/etc/$f.bak
-				mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
-			done
+	for f in group group- gshadow gshadow-; do
+		for d in mm debootstrap; do
+			sort /tmp/debian-{{ DIST }}-$d/etc/$f > /tmp/debian-{{ DIST }}-$d/etc/$f.bak
+			mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
 		done
-		;;
-	esac
+	done
 fi
 
 # check if the file content differs

tests/debootstrap

@@ -5,13 +5,6 @@ export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
 tmpdir="$(mktemp -d)"
 chmod 755 "$tmpdir"
-case "{{ DIST }}" in
-	oldstable|stable)
-		debootstrap --no-merged-usr --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
-		;;
-	*)
-		debootstrap --merged-usr --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
-		;;
-esac
+debootstrap "$([ "{{ DIST }}" = oldstable ] && echo --no-merged-usr || echo --merged-usr)" --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
 tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system --xattrs -C "$tmpdir" -c . > "./cache/debian-{{ DIST }}-{{ VARIANT }}.tar"
 rm -r "$tmpdir"

tests/eatmydata-via-hook-dir

@@ -34,6 +34,10 @@ elfheader="$elfheader\\001\\000"
 	--hook-dir=./hooks/eatmydata \
 	--customize-hook='printf "'"$elfheader"'" | cmp --bytes=8 - "$1"/usr/bin/dpkg' \
 	 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+ tar -C /tmp/debian-chroot --one-file-system -c . \
+	 | tar -t \
+	 | sort \
+	 | grep -v '^\./var/lib/dpkg/diversions\(-old\)\?$' \
+	 | diff -u tar1.txt -
 rm /tmp/checkeatmydata.sh
 rm -r /tmp/debian-chroot

tests/include

@@ -3,7 +3,7 @@ set -eu
 export LC_ALL=C.UTF-8
 trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
 {{ CMD }} --mode=root --variant=apt --include=doc-debian {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
 rm /tmp/debian-chroot/var/lib/apt/extended_states

tests/install-doc-debian

@@ -24,7 +24,7 @@ tar -C /tmp/debian-chroot --owner=0 --group=0 --numeric-owner --sort=name --clam
 tar tvf /tmp/debian-chroot.tar > doc-debian.tar.list
 rm /tmp/debian-chroot.tar
 # delete contents of doc-debian
-rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
 # delete real files

tests/install-doc-debian-and-test-hooks

@@ -27,7 +27,7 @@ tar -C /tmp/debian-chroot --owner=0 --group=0 --numeric-owner --sort=name --clam
 tar tvf /tmp/debian-chroot.tar | grep -v ' ./dev' | diff -u doc-debian.tar.list -
 rm /tmp/debian-chroot.tar
 # delete contents of doc-debian
-rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
 # delete real files

tests/jessie-or-older

@@ -29,6 +29,7 @@ filter() {
 		--path-exclude=/var/cache/debconf/config.dat-old \
 		--path-exclude=/var/cache/debconf/templates.dat-old \
 		--path-exclude=/var/lib/dpkg/available \
+		--path-exclude=/var/lib/dpkg/diversions \
 		--path-exclude=/var/lib/dpkg/cmethopt \
 		--path-exclude=/var/lib/dpkg/status-old \
 		--path-exclude=/var/lib/shells.state

tests/multiple-include

@@ -3,7 +3,7 @@ set -eu
 export LC_ALL=C.UTF-8
 trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
 {{ CMD }} --mode=root --variant=apt --include=doc-debian --include=tzdata {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
 rm /tmp/debian-chroot/usr/share/lintian/overrides/tzdata

tests/unpack-doc-debian

@@ -21,7 +21,7 @@ fi
 [ "{{ MODE }}" = "fakechroot" ] && prefix="$prefix fakechroot fakeroot"
 $prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --include=doc-debian {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 # delete contents of doc-debian
-rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
 # delete real files