release 1.1.0

- implements the same as debootstrap in
   https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/71
 - builds a temporary usr-is-merged package and upgrades to the real one
 - create merged-/usr chroots for unstable and testing (which will
   become Debian 12 Bookworm)
 - add a dedicated merged-/usr section to the manual page

CHANGELOG.md

@@ -1,3 +1,10 @@
+1.1.0 (2022-07-26)
+----------------
+
+ - mount a new /dev/pts instance into the chroot to make posix_openpt work
+ - adjust merged-/usr hook to work the same way as debootstrap
+ - add no-merged-usr hook
+
 1.0.1 (2022-05-29)
 ------------------
 

README.md

@@ -137,6 +137,11 @@ By default, `coverage.sh` will skip running a single test which tries creating
 a Ubuntu Focal chroot. To not skip that test, run `coverage.sh` with the
 environment variable `ONLINE=yes`.
 
+If a test fails you can run individual tests by executing `coverage.py` with
+the test name and optionally limit it to a specific distribution like so:
+
+    CMD=./mmdebstrap ./coverage.py --dist unstable check-against-debootstrap-dist
+
 Bugs
 ====
 

hooks/merged-usr/essential00.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -eu
+
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+        set -x
+fi
+
+TARGET="$1"
+
+APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$TARGET" usr-is-merged
+
+chroot "$TARGET" dpkg-query --showformat '${db:Status-Status}\n' --show usr-is-merged | grep -q '^installed$'
+chroot "$TARGET" dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
+dpkg --compare-versions "1" "lt" "$(chroot "$TARGET" dpkg-query --showformat '${Version}\n' --show usr-is-merged)"

hooks/merged-usr/setup00.sh

@@ -78,3 +78,34 @@ else
 		mkdir -p "$TARGET/usr/$dir"
 	done
 fi
+
+# now install an empty "usr-is-merged" package to avoid installing the
+# usrmerge package on this system even after init-system-helpers starts
+# depending on "usrmerge | usr-is-merged".
+#
+# This package will not end up in the final chroot because the essential
+# hook replaces it with the actual usr-is-merged package from src:usrmerge.
+
+tmpdir=$(mktemp --directory --tmpdir="$TARGET/tmp")
+mkdir -p "$tmpdir/usr-is-merged/DEBIAN"
+
+cat << END > "$tmpdir/usr-is-merged/DEBIAN/control"
+Package: usr-is-merged
+Priority: optional
+Section: oldlibs
+Maintainer: Johannes Schauer Marin Rodrigues <josch@debian.org>
+Architecture: all
+Multi-Arch: foreign
+Source: mmdebstrap-dummy-usr-is-merged
+Version: 1
+Description: dummy package created by mmdebstrap merged-usr setup hook
+ This package was generated and installed by the mmdebstrap merged-usr
+ setup hook at /usr/share/mmdebstrap/hooks/merged-usr.
+ .
+ If this package is installed in the final chroot, then this is a bug
+ in mmdebstrap. Please report: https://gitlab.mister-muffin.de/josch/mmdebstrap
+END
+dpkg-deb --build "$tmpdir/usr-is-merged" "$tmpdir/usr-is-merged.deb"
+dpkg --root="$TARGET" --log="$TARGET/var/log/dpkg.log" --install "$tmpdir/usr-is-merged.deb"
+rm "$tmpdir/usr-is-merged.deb" "$tmpdir/usr-is-merged/DEBIAN/control"
+rmdir "$tmpdir/usr-is-merged/DEBIAN" "$tmpdir/usr-is-merged" "$tmpdir"

hooks/no-merged-usr/essential00.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -eu
+
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+        set -x
+fi
+
+TARGET="$1"
+
+APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$TARGET" usr-is-merged
+
+chroot "$TARGET" dpkg-query --showformat '${db:Status-Status}\n' --show usr-is-merged | grep -q '^installed$'
+chroot "$TARGET" dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
+dpkg --compare-versions "1" "lt" "$(chroot "$TARGET" dpkg-query --showformat '${Version}\n' --show usr-is-merged)"

hooks/no-merged-usr/setup00.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+#
+# mmdebstrap does have a --no-merged-usr option but only as a no-op for
+# debootstrap compatibility
+#
+# Using this hook script, you can emulate what debootstrap does to set up
+# a system without merged-/usr even after the essential init-system-helpers
+# package added a dependency on "usrmerge | usr-is-merged". By installing
+# a dummy usr-is-merged package, it avoids pulling in the dependencies of
+# the usrmerge package.
+
+set -eu
+
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+        set -x
+fi
+
+TARGET="$1"
+
+echo "Warning: starting with Debian 12 (Bookworm), systems without merged-/usr are not supported anymore" >&2
+
+echo "this system will not be supported in the future" > "$TARGET/etc/unsupported-skip-usrmerge-conversion"
+
+# now install an empty "usr-is-merged" package to avoid installing the
+# usrmerge package on this system even after init-system-helpers starts
+# depending on "usrmerge | usr-is-merged".
+#
+# This package will not end up in the final chroot because the essential
+# hook replaces it with the actual usr-is-merged package from src:usrmerge.
+
+tmpdir=$(mktemp --directory --tmpdir="$TARGET/tmp")
+mkdir -p "$tmpdir/usr-is-merged/DEBIAN"
+
+cat << END > "$tmpdir/usr-is-merged/DEBIAN/control"
+Package: usr-is-merged
+Priority: optional
+Section: oldlibs
+Maintainer: Johannes Schauer Marin Rodrigues <josch@debian.org>
+Architecture: all
+Multi-Arch: foreign
+Source: mmdebstrap-dummy-usr-is-merged
+Version: 1
+Description: dummy package created by mmdebstrap no-merged-usr setup hook
+ This package was generated and installed by the mmdebstrap no-merged-usr
+ setup hook at /usr/share/mmdebstrap/hooks/no-merged-usr.
+ .
+ If this package is installed in the final chroot, then this is a bug
+ in mmdebstrap. Please report: https://gitlab.mister-muffin.de/josch/mmdebstrap
+END
+dpkg-deb --build "$tmpdir/usr-is-merged" "$tmpdir/usr-is-merged.deb"
+dpkg --root="$TARGET" --log="$TARGET/var/log/dpkg.log" --install "$tmpdir/usr-is-merged.deb"
+rm "$tmpdir/usr-is-merged.deb" "$tmpdir/usr-is-merged/DEBIAN/control"
+rmdir "$tmpdir/usr-is-merged/DEBIAN" "$tmpdir/usr-is-merged" "$tmpdir"

make_mirror.sh

@@ -266,6 +266,13 @@ END
 
 	pkgs="$(echo $pkgs) build-essential busybox gpg eatmydata"
 
+	# we need usr-is-merged to simulate debootstrap behaviour for all dists
+	# starting from Debian 12 (Bullseye)
+	case "$dist" in
+		oldstable|stable) : ;;
+		*) pkgs="$pkgs usr-is-merged" ;;
+	esac
+
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs
 
 	# to be able to also test gpg verification, we need to create a mirror
@@ -622,7 +629,7 @@ END
 	# buildd variant will not be 400MB but 1.3GB large and needs a 10G
 	# disk.
 	if [ -z ${DISK_SIZE+x} ]; then
-		DISK_SIZE=3G
+		DISK_SIZE=10G
 	fi
 	guestfish -N "$tmpdir/debian-$DEFAULT_DIST.img"=disk:$DISK_SIZE -- \
 		part-disk /dev/sda mbr : \
@@ -654,7 +661,7 @@ fi
 mirror="http://127.0.0.1/debian"
 for dist in oldstable stable testing unstable; do
 	for variant in minbase buildd -; do
-		echo "running debootstrap --no-merged-usr --variant=$variant $dist \${TEMPDIR} $mirror"
+		echo "running debootstrap --variant=$variant $dist \${TEMPDIR} $mirror"
 		cat << END > shared/test.sh
 #!/bin/sh
 set -eu
@@ -663,7 +670,14 @@ export SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
 echo "SOURCE_DATE_EPOCH=\$SOURCE_DATE_EPOCH"
 tmpdir="\$(mktemp -d)"
 chmod 755 "\$tmpdir"
-debootstrap --no-merged-usr --variant=$variant $dist "\$tmpdir" $mirror
+case "$dist" in
+	oldstable|stable)
+		debootstrap --no-merged-usr --variant=$variant $dist "\$tmpdir" $mirror
+		;;
+	*)
+		debootstrap --merged-usr --variant=$variant $dist "\$tmpdir" $mirror
+		;;
+esac
 tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system --xattrs -C "\$tmpdir" -c . > "$newcache/debian-$dist-$variant.tar"
 rm -r "\$tmpdir"
 END

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '1.0.1';
+our $VERSION = '1.1.0';
 
 use English;
 use Getopt::Long;
@@ -2236,7 +2236,7 @@ sub run_download() {
                   . " -- use copy:// instead of file:// or a bind-mount. You"
                   . " can also try using --hook-dir=/usr/share/mmdebstrap/"
                   . "hooks/file-mirror-automount to automatically create"
-                  . " the necessary bind-mounts.";
+                  . " bind-mounts or copy the files as necessary.";
             }
             push @essential_pkgs, $deb;
             next;
@@ -4451,7 +4451,7 @@ sub main() {
         );
         close $fh;
         if (    $? == 0
-            and $content =~ /^apt ([0-9]+\.[0-9]+\.[0-9]+) \([a-z0-9-]+\)$/m) {
+            and $content =~ /^apt ([0-9]+\.[0-9]+\.[0-9]+)[a-z0-9]* \([a-z0-9-]+\)$/m) {
             $aptversion = version->new($1);
         }
         if ($aptversion < "2.3.14") {
@@ -5785,6 +5785,8 @@ sub main() {
                         error("failed to run " . (join " ", @argv));
                     }
                 } else {
+                    # somehow, when running under qemu, writing to a virtio
+                    # device will not result in a ENOSPC but just stall forever
                     if (!copy($rfh, $options->{target})) {
                         error "cannot copy to $options->{target}: $!";
                     }
@@ -6098,10 +6100,6 @@ package list is free of duplicates. So the following are equivalent:
     --include=pkg1/stable,pkg2=1.0,pkg3-
     --incl=pkg1/stable --incl="pkg2=1.0 pkg3-" --incl=pkg2=1.0,pkg3-
 
-Example: setting up merged-/usr via the usrmerge package
-
-    --include=usrmerge
-
 =item B<--components>=I<comp1>[,I<comp2>,...]
 
 Comma or whitespace separated list of components like main, contrib and
@@ -6148,18 +6146,7 @@ installed. At that point, the chroot directory does not contain any
 executables and thus cannot be chroot-ed into. See section B<HOOKS> for more
 information.
 
-Example 1: Setup merged-/usr via symlinks, omitting the architecture specific
+Example: Setup chroot for installing a sub-essential busybox-based chroot
-symlinks
-
-    --setup-hook='for d in bin sbin lib; do ln -s usr/$d "$1/$d";
-        mkdir -p "$1/usr/$d"; done'
-
-Example 2: Setup merged-/usr using debootstrap-method which takes care of the
-architecture specific symlinks
-
-    --setup-hook=/usr/share/mmdebstrap/hooks/merged-usr/setup00.sh
-
-Example 3: Setup chroot for installing a sub-essential busybox-based chroot
 with --variant=custom
 --include=dpkg,busybox,libc-bin,base-files,base-passwd,debianutils
 
@@ -6170,8 +6157,8 @@ with --variant=custom
     --setup-hook='echo root:x:0:0:root:/root:/bin/sh > "$1/etc/passwd"'
     --setup-hook='printf "root:x:0:\nmail:x:8:\nutmp:x:43:\n" > "$1/etc/group"'
 
-For a more elegant way to setup merged-/usr via symlinks and for setting up a
+For a more elegant way for setting up a sub-essential busybox-based chroot, see
-sub-essential busybox-based chroot, see the B<--hook-dir> option below.
+the B<--hook-dir> option below.
 
 =item B<--extract-hook>=I<command>
 
@@ -6252,12 +6239,7 @@ Example 2: Setup chroot for installing a sub-essential busybox-based chroot
 
     --hook-dir=/usr/share/mmdebstrap/hooks/busybox
 
-Example 3: Setup merged-/usr using debootstrap-method which takes care of the
+Example 3: Automatically mount all directories referenced by C<file://> mirrors
-architecture specific symlinks
-
-    --hook-dir=/usr/share/mmdebstrap/hooks/merged-usr
-
-Example 4: Automatically mount all directories referenced by C<file://> mirrors
 into the chroot
 
     --hook-dir=/usr/share/mmdebstrap/hooks/file-mirror-automount
@@ -7095,6 +7077,33 @@ I<--merged-usr> and I<--no-merged-usr>
 
 =back
 
+=head1 MERGED-/USR
+
+B<mmdebstrap> will create a merged-/usr chroot or not depending on whether
+packages setting up merged-/usr (i.e. the B<usrmerge> package) are installed or
+not. In Debian, the essential package B<init-system-helpers> is will depend
+on the B<usrmerge> package, starting with Debian 12 (Bookworm).
+
+Before Debian 12 (Bookworm), to force B<mmdebstrap> to create a chroot with
+merged-/usr using symlinks, either explicitly install the B<usrmerge> package:
+
+    --include=usrmerge
+
+or setup merged-/usr using the debootstrap-method which takes care of the
+architecture specific symlinks and installs the B<usr-is-merged> package.
+
+    --hook-dir=/usr/share/mmdebstrap/hooks/merged-usr
+
+To force B<mmdebstrap> to create a chroot without merged-/usr even after the
+Debian 12 (Bookworm) release, you can use the following hook:
+
+    --hook-dir=/usr/share/mmdebstrap/hooks/no-merged-usr
+
+This will write "this system will not be supported in the future" into
+F</etc/unsupported-skip-usrmerge-conversion> inside the chroot and install the
+B<usr-is-merged> package to avoid the installation of the B<usrmerge> package
+and its dependencies.
+
 =head1 COMPRESSION
 
 B<mmdebstrap> will choose a suitable compressor for the output tarball

tests/as-debootstrap-unshare-wrapper

@@ -8,7 +8,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 fi
 sysctl -w kernel.unprivileged_userns_clone=1
 adduser --gecos user --disabled-password user
-runuser -u user -- {{ CMD }} --variant=custom --mode=unshare --setup-hook='env container=lxc debootstrap --no-merged-usr unstable "$1" {{ MIRROR }}' - /tmp/debian-mm.tar {{ MIRROR }}
+runuser -u user -- {{ CMD }} --variant=custom --mode=unshare --setup-hook='env container=lxc debootstrap unstable "$1" {{ MIRROR }}' - /tmp/debian-mm.tar {{ MIRROR }}
 
 mkdir /tmp/debian-mm
 tar --xattrs --xattrs-include='*' -C /tmp/debian-mm -xf /tmp/debian-mm.tar

tests/check-against-debootstrap-dist

@@ -10,8 +10,14 @@ echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
 # order in comparison to the systemd users
 # https://bugs.debian.org/969631
 # we cannot use useradd because passwd is not Essential:yes
+#
+# with cron 3.0pl1-142 and the introduction of cron-daemon-common, installation
+# order of cron and systemd started to differ between debootstrap and
+# mmdebstrap, resulting in different gid values
 {{ CMD }} --variant={{ VARIANT }} --mode={{ MODE }} \
 	--essential-hook='if [ {{ VARIANT }} = - ]; then echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd; fi' \
+	--essential-hook='if [ {{ VARIANT }} = - ] && [ {{ DIST }} = unstable -o {{ DIST }} = testing ]; then printf "systemd-journal:x:101:\nsystemd-network:x:102:\nsystemd-resolve:x:103:\ncrontab:x:104:" >> "$1"/etc/group; fi' \
+	$(case {{ DIST }} in oldstable|stable) : ;; *) echo --hook-dir=./hooks/merged-usr ;; esac) \
 	{{ DIST }} /tmp/debian-{{ DIST }}-mm.tar {{ MIRROR }}
 
 mkdir /tmp/debian-{{ DIST }}-mm
@@ -129,8 +135,21 @@ rm /tmp/debian-{{ DIST }}-mm/var/lib/apt/extended_states
 rm /tmp/debian-{{ DIST }}-mm/var/lib/apt/lists/lock
 
 # the list of shells might be sorted wrongly
-for f in "/tmp/debian-{{ DIST }}-debootstrap/etc/shells" "/tmp/debian-{{ DIST }}-mm/etc/shells"; do
+# /var/lib/dpkg/triggers/File might be sorted wrongly
-	sort -o "$f" "$f"
+for f in "/var/lib/dpkg/triggers/File" "/etc/shells"; do
+	f1="/tmp/debian-{{ DIST }}-debootstrap/$f"
+	f2="/tmp/debian-{{ DIST }}-mm/$f"
+	# both chroots must have the file
+	if [ ! -e "$f1" ] || [ ! -e "$f2" ]; then
+		continue
+	fi
+	# the file must be different
+	if cmp "$f1" "$f2"; then
+		continue
+	fi
+	# then sort both
+	sort -o "$f1" "$f1"
+	sort -o "$f2" "$f2"
 done
 
 # Because of unreproducible uids (#969631) we created the _apt user ourselves
@@ -141,6 +160,14 @@ for f in shadow shadow-; do
 		sed -i 's/^_apt:\*:\([^:]\+\):0:99999:7:::$/_apt:!:\1::::::/' /tmp/debian-{{ DIST }}-mm/etc/$f
 	fi
 done
+# same as above but for cron and systemd groups
+for f in gshadow gshadow-; do
+	for group in systemd-journal systemd-network systemd-resolve crontab; do
+		if grep -q '^'"$group"':!:' /tmp/debian-{{ DIST }}-debootstrap/etc/$f; then
+			sed -i 's/^'"$group"':x::/'"$group"':!::/' /tmp/debian-{{ DIST }}-mm/etc/$f
+		fi
+	done
+done
 
 # workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
 if ! cmp /tmp/debian-{{ DIST }}-debootstrap/etc/shadow /tmp/debian-{{ DIST }}-mm/etc/shadow; then

tests/merged-usr-via-setup-hook

@@ -3,6 +3,7 @@ set -eu
 export LC_ALL=C.UTF-8
 trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
 {{ CMD }} --mode=root --variant=apt \
+	--essential-hook=./hooks/merged-usr/essential00.sh \
 	--setup-hook=./hooks/merged-usr/setup00.sh \
 	--customize-hook='[ -L "$1"/bin -a -L "$1"/sbin -a -L "$1"/lib ]' \
 	{{ DIST }} /tmp/debian-chroot {{ MIRROR }}
@@ -40,4 +41,10 @@ tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort > tar2.txt
 		echo ./usr/lib32/;
 		;;
 	esac
+	echo ./usr/share/doc/usr-is-merged/
+	echo ./usr/share/doc/usr-is-merged/changelog.gz
+	echo ./usr/share/doc/usr-is-merged/copyright
+	echo ./var/lib/dpkg/info/usr-is-merged.list
+	echo ./var/lib/dpkg/info/usr-is-merged.md5sums
+	echo ./var/lib/dpkg/info/usr-is-merged.preinst
 } | sort -u | diff -u - tar2.txt