Adjust merged-/usr as it's done by debootstrap

- implements the same as debootstrap in
   https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/71
 - builds a temporary usr-is-merged package and upgrades to the real one
 - create merged-/usr chroots for unstable and testing (which will
   become Debian 12 Bookworm)
 - add a dedicated merged-/usr section to the manual page

hooks/merged-usr/essential00.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -eu
+
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+        set -x
+fi
+
+TARGET="$1"
+
+APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$TARGET" usr-is-merged
+
+chroot "$TARGET" dpkg-query --showformat '${db:Status-Status}\n' --show usr-is-merged | grep -q '^installed$'
+chroot "$TARGET" dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
+dpkg --compare-versions "1" "lt" "$(chroot "$TARGET" dpkg-query --showformat '${Version}\n' --show usr-is-merged)"

hooks/merged-usr/setup00.sh

@@ -78,3 +78,34 @@ else
 		mkdir -p "$TARGET/usr/$dir"
 	done
 fi
+
+# now install an empty "usr-is-merged" package to avoid installing the
+# usrmerge package on this system even after init-system-helpers starts
+# depending on "usrmerge | usr-is-merged".
+#
+# This package will not end up in the final chroot because the essential
+# hook replaces it with the actual usr-is-merged package from src:usrmerge.
+
+tmpdir=$(mktemp --directory --tmpdir="$TARGET/tmp")
+mkdir -p "$tmpdir/usr-is-merged/DEBIAN"
+
+cat << END > "$tmpdir/usr-is-merged/DEBIAN/control"
+Package: usr-is-merged
+Priority: optional
+Section: oldlibs
+Maintainer: Johannes Schauer Marin Rodrigues <josch@debian.org>
+Architecture: all
+Multi-Arch: foreign
+Source: mmdebstrap-dummy-usr-is-merged
+Version: 1
+Description: dummy package created by mmdebstrap merged-usr setup hook
+ This package was generated and installed by the mmdebstrap merged-usr
+ setup hook at /usr/share/mmdebstrap/hooks/merged-usr.
+ .
+ If this package is installed in the final chroot, then this is a bug
+ in mmdebstrap. Please report: https://gitlab.mister-muffin.de/josch/mmdebstrap
+END
+dpkg-deb --build "$tmpdir/usr-is-merged" "$tmpdir/usr-is-merged.deb"
+dpkg --root="$TARGET" --log="$TARGET/var/log/dpkg.log" --install "$tmpdir/usr-is-merged.deb"
+rm "$tmpdir/usr-is-merged.deb" "$tmpdir/usr-is-merged/DEBIAN/control"
+rmdir "$tmpdir/usr-is-merged/DEBIAN" "$tmpdir/usr-is-merged" "$tmpdir"

hooks/no-merged-usr/essential00.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -eu
+
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+        set -x
+fi
+
+TARGET="$1"
+
+APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$TARGET" usr-is-merged
+
+chroot "$TARGET" dpkg-query --showformat '${db:Status-Status}\n' --show usr-is-merged | grep -q '^installed$'
+chroot "$TARGET" dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
+dpkg --compare-versions "1" "lt" "$(chroot "$TARGET" dpkg-query --showformat '${Version}\n' --show usr-is-merged)"

hooks/no-merged-usr/setup00.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+#
+# mmdebstrap does have a --no-merged-usr option but only as a no-op for
+# debootstrap compatibility
+#
+# Using this hook script, you can emulate what debootstrap does to set up
+# a system without merged-/usr even after the essential init-system-helpers
+# package added a dependency on "usrmerge | usr-is-merged". By installing
+# a dummy usr-is-merged package, it avoids pulling in the dependencies of
+# the usrmerge package.
+
+set -eu
+
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+        set -x
+fi
+
+TARGET="$1"
+
+echo "Warning: starting with Debian 12 (Bookworm), systems without merged-/usr are not supported anymore" >&2
+
+echo "this system will not be supported in the future" > "$TARGET/etc/unsupported-skip-usrmerge-conversion"
+
+# now install an empty "usr-is-merged" package to avoid installing the
+# usrmerge package on this system even after init-system-helpers starts
+# depending on "usrmerge | usr-is-merged".
+#
+# This package will not end up in the final chroot because the essential
+# hook replaces it with the actual usr-is-merged package from src:usrmerge.
+
+tmpdir=$(mktemp --directory --tmpdir="$TARGET/tmp")
+mkdir -p "$tmpdir/usr-is-merged/DEBIAN"
+
+cat << END > "$tmpdir/usr-is-merged/DEBIAN/control"
+Package: usr-is-merged
+Priority: optional
+Section: oldlibs
+Maintainer: Johannes Schauer Marin Rodrigues <josch@debian.org>
+Architecture: all
+Multi-Arch: foreign
+Source: mmdebstrap-dummy-usr-is-merged
+Version: 1
+Description: dummy package created by mmdebstrap no-merged-usr setup hook
+ This package was generated and installed by the mmdebstrap no-merged-usr
+ setup hook at /usr/share/mmdebstrap/hooks/no-merged-usr.
+ .
+ If this package is installed in the final chroot, then this is a bug
+ in mmdebstrap. Please report: https://gitlab.mister-muffin.de/josch/mmdebstrap
+END
+dpkg-deb --build "$tmpdir/usr-is-merged" "$tmpdir/usr-is-merged.deb"
+dpkg --root="$TARGET" --log="$TARGET/var/log/dpkg.log" --install "$tmpdir/usr-is-merged.deb"
+rm "$tmpdir/usr-is-merged.deb" "$tmpdir/usr-is-merged/DEBIAN/control"
+rmdir "$tmpdir/usr-is-merged/DEBIAN" "$tmpdir/usr-is-merged" "$tmpdir"

make_mirror.sh

@@ -266,6 +266,13 @@ END
 
 	pkgs="$(echo $pkgs) build-essential busybox gpg eatmydata"
 
+	# we need usr-is-merged to simulate debootstrap behaviour for all dists
+	# starting from Debian 12 (Bullseye)
+	case "$dist" in
+		oldstable|stable) : ;;
+		*) pkgs="$pkgs usr-is-merged" ;;
+	esac
+
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs
 
 	# to be able to also test gpg verification, we need to create a mirror
@@ -654,7 +661,7 @@ fi
 mirror="http://127.0.0.1/debian"
 for dist in oldstable stable testing unstable; do
 	for variant in minbase buildd -; do
-		echo "running debootstrap --no-merged-usr --variant=$variant $dist \${TEMPDIR} $mirror"
+		echo "running debootstrap --variant=$variant $dist \${TEMPDIR} $mirror"
 		cat << END > shared/test.sh
 #!/bin/sh
 set -eu
@@ -663,7 +670,14 @@ export SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
 echo "SOURCE_DATE_EPOCH=\$SOURCE_DATE_EPOCH"
 tmpdir="\$(mktemp -d)"
 chmod 755 "\$tmpdir"
-debootstrap --no-merged-usr --variant=$variant $dist "\$tmpdir" $mirror
+case "$dist" in
+	oldstable|stable)
+		debootstrap --no-merged-usr --variant=$variant $dist "\$tmpdir" $mirror
+		;;
+	*)
+		debootstrap --merged-usr --variant=$variant $dist "\$tmpdir" $mirror
+		;;
+esac
 tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system --xattrs -C "\$tmpdir" -c . > "$newcache/debian-$dist-$variant.tar"
 rm -r "\$tmpdir"
 END

mmdebstrap

@@ -6098,10 +6098,6 @@ package list is free of duplicates. So the following are equivalent:
     --include=pkg1/stable,pkg2=1.0,pkg3-
     --incl=pkg1/stable --incl="pkg2=1.0 pkg3-" --incl=pkg2=1.0,pkg3-
 
-Example: setting up merged-/usr via the usrmerge package
-
-    --include=usrmerge
-
 =item B<--components>=I<comp1>[,I<comp2>,...]
 
 Comma or whitespace separated list of components like main, contrib and
@@ -6148,18 +6144,7 @@ installed. At that point, the chroot directory does not contain any
 executables and thus cannot be chroot-ed into. See section B<HOOKS> for more
 information.
 
-Example 1: Setup merged-/usr via symlinks, omitting the architecture specific
+Example: Setup chroot for installing a sub-essential busybox-based chroot
-symlinks
-
-    --setup-hook='for d in bin sbin lib; do ln -s usr/$d "$1/$d";
-        mkdir -p "$1/usr/$d"; done'
-
-Example 2: Setup merged-/usr using debootstrap-method which takes care of the
-architecture specific symlinks
-
-    --setup-hook=/usr/share/mmdebstrap/hooks/merged-usr/setup00.sh
-
-Example 3: Setup chroot for installing a sub-essential busybox-based chroot
 with --variant=custom
 --include=dpkg,busybox,libc-bin,base-files,base-passwd,debianutils
 
@@ -6170,8 +6155,8 @@ with --variant=custom
     --setup-hook='echo root:x:0:0:root:/root:/bin/sh > "$1/etc/passwd"'
     --setup-hook='printf "root:x:0:\nmail:x:8:\nutmp:x:43:\n" > "$1/etc/group"'
 
-For a more elegant way to setup merged-/usr via symlinks and for setting up a
+For a more elegant way for setting up a sub-essential busybox-based chroot, see
-sub-essential busybox-based chroot, see the B<--hook-dir> option below.
+the B<--hook-dir> option below.
 
 =item B<--extract-hook>=I<command>
 
@@ -6252,12 +6237,7 @@ Example 2: Setup chroot for installing a sub-essential busybox-based chroot
 
     --hook-dir=/usr/share/mmdebstrap/hooks/busybox
 
-Example 3: Setup merged-/usr using debootstrap-method which takes care of the
+Example 3: Automatically mount all directories referenced by C<file://> mirrors
-architecture specific symlinks
-
-    --hook-dir=/usr/share/mmdebstrap/hooks/merged-usr
-
-Example 4: Automatically mount all directories referenced by C<file://> mirrors
 into the chroot
 
     --hook-dir=/usr/share/mmdebstrap/hooks/file-mirror-automount
@@ -7095,6 +7075,33 @@ I<--merged-usr> and I<--no-merged-usr>
 
 =back
 
+=head1 MERGED-/USR
+
+B<mmdebstrap> will create a merged-/usr chroot or not depending on whether
+packages setting up merged-/usr (i.e. the B<usrmerge> package) are installed or
+not. In Debian, the essential package B<init-system-helpers> is will depend
+on the B<usrmerge> package, starting with Debian 12 (Bookworm).
+
+Before Debian 12 (Bookworm), to force B<mmdebstrap> to create a chroot with
+merged-/usr using symlinks, either explicitly install the B<usrmerge> package:
+
+    --include=usrmerge
+
+or setup merged-/usr using the debootstrap-method which takes care of the
+architecture specific symlinks and installs the B<usr-is-merged> package.
+
+    --hook-dir=/usr/share/mmdebstrap/hooks/merged-usr
+
+To force B<mmdebstrap> to create a chroot without merged-/usr even after the
+Debian 12 (Bookworm) release, you can use the following hook:
+
+    --hook-dir=/usr/share/mmdebstrap/hooks/no-merged-usr
+
+This will write "this system will not be supported in the future" into
+F</etc/unsupported-skip-usrmerge-conversion> inside the chroot and install the
+B<usr-is-merged> package to avoid the installation of the B<usrmerge> package
+and its dependencies.
+
 =head1 COMPRESSION
 
 B<mmdebstrap> will choose a suitable compressor for the output tarball

tests/as-debootstrap-unshare-wrapper

@@ -8,7 +8,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 fi
 sysctl -w kernel.unprivileged_userns_clone=1
 adduser --gecos user --disabled-password user
-runuser -u user -- {{ CMD }} --variant=custom --mode=unshare --setup-hook='env container=lxc debootstrap --no-merged-usr unstable "$1" {{ MIRROR }}' - /tmp/debian-mm.tar {{ MIRROR }}
+runuser -u user -- {{ CMD }} --variant=custom --mode=unshare --setup-hook='env container=lxc debootstrap unstable "$1" {{ MIRROR }}' - /tmp/debian-mm.tar {{ MIRROR }}
 
 mkdir /tmp/debian-mm
 tar --xattrs --xattrs-include='*' -C /tmp/debian-mm -xf /tmp/debian-mm.tar

tests/check-against-debootstrap-dist

@@ -12,6 +12,7 @@ echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
 # we cannot use useradd because passwd is not Essential:yes
 {{ CMD }} --variant={{ VARIANT }} --mode={{ MODE }} \
 	--essential-hook='if [ {{ VARIANT }} = - ]; then echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd; fi' \
+	$(case {{ DIST }} in oldstable|stable) : ;; *) echo --hook-dir=./hooks/merged-usr ;; esac) \
 	{{ DIST }} /tmp/debian-{{ DIST }}-mm.tar {{ MIRROR }}
 
 mkdir /tmp/debian-{{ DIST }}-mm

tests/merged-usr-via-setup-hook

@@ -3,6 +3,7 @@ set -eu
 export LC_ALL=C.UTF-8
 trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
 {{ CMD }} --mode=root --variant=apt \
+	--essential-hook=./hooks/merged-usr/essential00.sh \
 	--setup-hook=./hooks/merged-usr/setup00.sh \
 	--customize-hook='[ -L "$1"/bin -a -L "$1"/sbin -a -L "$1"/lib ]' \
 	{{ DIST }} /tmp/debian-chroot {{ MIRROR }}
@@ -40,4 +41,10 @@ tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort > tar2.txt
 		echo ./usr/lib32/;
 		;;
 	esac
+	echo ./usr/share/doc/usr-is-merged/
+	echo ./usr/share/doc/usr-is-merged/changelog.gz
+	echo ./usr/share/doc/usr-is-merged/copyright
+	echo ./var/lib/dpkg/info/usr-is-merged.list
+	echo ./var/lib/dpkg/info/usr-is-merged.md5sums
+	echo ./var/lib/dpkg/info/usr-is-merged.preinst
 } | sort -u | diff -u - tar2.txt