coverage.py: move print of failed tests to the end

coverage.py

@@ -432,11 +432,6 @@ def main():
             print(f"skipped because of {reason}:", file=sys.stderr)
             for t in l:
                 print(f"    {t}", file=sys.stderr)
-    if failed:
-        print("failed %d:" % len(failed), file=sys.stderr)
-        for f in failed:
-            print(f, file=sys.stderr)
-        exit(1)
     if len(time_per_test) > 1:
         print_time_per_test(time_per_test)
     if len(acc_time_per_test) > 1:
@@ -447,6 +442,11 @@ def main():
             },
             "accumulated test",
         )
+    if failed:
+        print("failed %d:" % len(failed), file=sys.stderr)
+        for f in failed:
+            print(f, file=sys.stderr)
+        exit(1)
 
 
 if __name__ == "__main__":

coverage.sh

@@ -95,4 +95,11 @@ END
 	echo
 fi
 
+# check if the wiki has to be updated with pod2markdown output
+if [ "${DEBEMAIL:-}" = "josch@debian.org" ]; then
+	bash -exc "diff -u <(curl --silent https://gitlab.mister-muffin.de/josch/mmdebstrap/wiki/raw/Home | dos2unix) <(pod2markdown < mmdebstrap)" || :
+fi
+
 rm -f shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/tarfilter shared/proxysolver
+
+echo "$0 finished successfully" >&2

hooks/maybe-merged-usr/essential00.sh

@@ -26,7 +26,7 @@ esac
 #  2. using ./hooks
 #  3. using /usr/share/mmdebstrap/hooks/
 for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
-	if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
+	if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/extract00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
 		"$p/merged-usr/essential00.sh" "$1"
 		exit 0
 	fi

hooks/maybe-merged-usr/extract00.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -eu
+
+env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get update --error-on=any
+
+# if the usr-is-merged package cannot be installed with apt, do nothing
+if ! env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
+	echo "no package called usr-is-merged found -- not running merged-usr extract hook" >&2
+	exit 0
+else
+	echo "package usr-is-merged found -- running merged-usr extract hook" >&2
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/extract00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
+		"$p/merged-usr/extract00.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find merged-usr hook anywhere" >&2
+exit 1

hooks/maybe-merged-usr/setup00.sh

@@ -17,7 +17,7 @@ fi
 #  2. using ./hooks
 #  3. using /usr/share/mmdebstrap/hooks/
 for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
-	if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
+	if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/extract00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
 		"$p/merged-usr/setup00.sh" "$1"
 		exit 0
 	fi

hooks/merged-usr/extract00.sh

@@ -0,0 +1,85 @@
+#!/bin/sh
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
+
+TARGET="$1"
+
+# can_usrmerge_symlink() and can_usrmerge_symlink() are
+# Copyright 2023 Helmut Grohne <helmut@subdivi.de>
+# and part of the debootstrap source in /usr/share/debootstrap/functions
+# https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/96
+# https://bugs.debian.org/104989
+can_usrmerge_symlink() {
+	# Absolute symlinks can be relocated without problems.
+	test "${2#/}" = "$2" || return 0
+	while :; do
+		if test "${2#/}" != "$2"; then
+			# Handle double-slashes.
+			set -- "$1" "${2#/}"
+		elif test "${2#./}" != "$2"; then
+			# Handle ./ inside a link target.
+			set -- "$1" "${2#./}"
+		elif test "$2" = ..; then
+			# A parent directory symlink is ok if it does not
+			# cross the top level directory.
+			test "${1%/*/*}" != "$1" -a -n "${1%/*/*}"
+			return $?
+		elif test "${2#../}" != "$2"; then
+			# Symbolic link crossing / cannot be moved safely.
+			# This is prohibited by Debian Policy 10.5.
+			test "${1%/*/*}" = "$1" -o -z "${1%/*/*}" && return 1
+			set -- "${1%/*}" "${2#../}"
+		else
+			# Consider the symlink ok if its target does not
+			# contain a parent directory. When we fail here,
+			# the link target is non-minimal and doesn't happen
+			# in the archive.
+			test "${2#*/../}" = "$2"
+			return $?
+		fi
+	done
+}
+
+merge_usr_entry() {
+	# shellcheck disable=SC3043
+	local entry canon
+	canon="$TARGET/usr/${1#"$TARGET/"}"
+	test -h "$canon" &&
+		error 1 USRMERGEFAIL "cannot move %s as its destination exists as a symlink" "${1#"$TARGET"}"
+	if ! test -e "$canon"; then
+		mv "$1" "$canon"
+		return 0
+	fi
+	test -d "$1" ||
+		error 1 USRMERGEFAIL "cannot move non-directory %s as its destination exists" "${1#"$TARGET"}"
+	test -d "$canon" ||
+		error 1 USRMERGEFAIL "cannot move directory %s as its destination is not a directory" "${1#"$TARGET"}"
+	for entry in "$1/"* "$1/."*; do
+		# Some shells return . and .. on dot globs.
+		test "${entry%/.}" != "${entry%/..}" && continue
+		if test -h "$entry" && ! can_usrmerge_symlink "${entry#"$TARGET"}" "$(readlink "$entry")"; then
+			error 1 USRMERGEFAIL "cannot move relative symlink crossing top-level directory" "${entry#"$TARGET"}"
+		fi
+		# Ignore glob match failures
+		if test "${entry%'/*'}" != "${entry%'/.*'}" && ! test -e "$entry"; then
+			continue
+		fi
+		merge_usr_entry "$entry"
+	done
+	rmdir "$1"
+}
+
+# This is list includes all possible multilib directories. It must be
+# updated when new multilib directories are being added. Hopefully,
+# all new architectures use multiarch instead, so we never get to
+# update this.
+for dir in bin lib lib32 lib64 libo32 libx32 sbin; do
+	test -h "$TARGET/$dir" && continue
+	test -e "$TARGET/$dir" || continue
+	merge_usr_entry "$TARGET/$dir"
+	ln -s "usr/$dir" "$TARGET/$dir"
+done

hooks/merged-usr/setup00.sh

@@ -47,40 +47,6 @@ fi
 
 TARGET="$1"
 
-ARCH=$(dpkg --print-architecture)
-eval "$(APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-config shell ARCH Apt::Architecture)"
-
-if [ -e /usr/share/debootstrap/functions ]; then
-	# shellcheck disable=SC1091
-	. /usr/share/debootstrap/functions
-	doing_variant () { [ "$1" != "buildd" ]; }
-	# shellcheck disable=SC2034
-	MERGED_USR="yes"
-	setup_merged_usr
-else
-	link_dir=""
-	case $ARCH in
-	    hurd-*) exit 0;;
-	    amd64) link_dir="lib32 lib64 libx32" ;;
-	    i386) link_dir="lib64 libx32" ;;
-	    mips|mipsel) link_dir="lib32 lib64" ;;
-	    mips64*|mipsn32*) link_dir="lib32 lib64 libo32" ;;
-	    powerpc) link_dir="lib64" ;;
-	    ppc64) link_dir="lib32 lib64" ;;
-	    ppc64el) link_dir="lib64" ;;
-	    s390x) link_dir="lib32" ;;
-	    sparc) link_dir="lib64" ;;
-	    sparc64) link_dir="lib32 lib64" ;;
-	    x32) link_dir="lib32 lib64 libx32" ;;
-	esac
-	link_dir="bin sbin lib $link_dir"
-
-	for dir in $link_dir; do
-		ln -s usr/"$dir" "$TARGET/$dir"
-		mkdir -p "$TARGET/usr/$dir"
-	done
-fi
-
 # now install an empty "usr-is-merged" package to avoid installing the
 # usrmerge package on this system even after init-system-helpers starts
 # depending on "usrmerge | usr-is-merged".

hooks/no-merged-usr/setup00.sh

@@ -18,6 +18,7 @@ fi
 TARGET="$1"
 
 echo "Warning: starting with Debian 12 (Bookworm), systems without merged-/usr are not supported anymore" >&2
+echo "Warning: starting with Debian 13 (Trixie), merged-/usr symlinks are shipped by packages in the essential-set making this hook ineffective" >&2
 
 echo "this system will not be supported in the future" > "$TARGET/etc/unsupported-skip-usrmerge-conversion"
 

make_mirror.sh

@@ -518,8 +518,7 @@ END
 	fi
 	# set PATH to pick up the correct mmdebstrap variant
 	env PATH="$(dirname "$(realpath --canonicalize-existing "$CMD")"):$PATH" \
-		debvm-create --skip=usrmerge --size="$DISK_SIZE" \
+		debvm-create --skip=usrmerge --size="$DISK_SIZE" --release="$DEFAULT_DIST" \
-		--release="$DEFAULT_DIST" --skip=usrmerge \
 		--output="$newcachedir/debian-$DEFAULT_DIST.ext4" -- \
 		--architectures="$arches" --include="$pkgs" \
 		--setup-hook='echo "Acquire::http::Proxy \"http://127.0.0.1:8080/\";" > "$1/etc/apt/apt.conf.d/00proxy"' \

mmdebstrap

@@ -5814,7 +5814,8 @@ sub main() {
         '--format=pax',
         '--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime',
         '-c',
-        '--exclude=./dev'
+        '--exclude=./dev',
+        '--exclude=./lost+found'
     );
     # tar2sqfs and genext2fs do not support extended attributes
     if ($format eq "squashfs") {

tests/ascii-armored-keys

@@ -13,9 +13,7 @@ rmdir /etc/apt/trusted.gpg.d
 mkdir /etc/apt/trusted.gpg.d
 for f in /usr/share/keyrings/*.gpg; do
 	name=$(basename "$f" .gpg)
-	gpg --enarmor < "/usr/share/keyrings/$name.gpg" \
+	gpg --no-default-keyring --keyring="/usr/share/keyrings/$name.gpg" --armor --output="/etc/apt/trusted.gpg.d/$name.asc" --export
-		| sed 's/ PGP ARMORED FILE/ PGP PUBLIC KEY BLOCK/;/^Comment: /d' \
-		> "/etc/apt/trusted.gpg.d/$name.asc"
 	rm "/usr/share/keyrings/$name.gpg"
 done
 {{ CMD }} --mode=root --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}