tests: skip debootstrap tests variant - on oldstable because of #917773

caching_proxy.py

@@ -46,7 +46,12 @@ class ProxyRequestHandler(http.server.BaseHTTPRequestHandler):
             newpath = pathlib.Path("/dev/null")
 
         # copy from oldpath to newpath and send back to client
-        if oldpath.exists():
+        # Only take files from the old cache if they are .deb files or Packages
+        # files in the by-hash directory as only those are unique by their path
+        # name. Other files like InRelease files have to be downloaded afresh.
+        if oldpath.exists() and (
+            oldpath.suffix == ".deb" or "by-hash" in oldpath.parts
+        ):
             print(f"proxy cached: {self.path}", file=sys.stderr)
             self.wfile.write(b"HTTP/1.1 200 OK\r\n")
             self.send_header("Content-Length", oldpath.stat().st_size)

coverage.txt

@@ -3,18 +3,21 @@ Dists: any
 Variants: minbase buildd -
 Needs-Root: true
 Needs-APT-Config: true
+Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: check-against-debootstrap-dist
 Dists: any
 Variants: minbase buildd -
 Needs-Root: true
 Needs-APT-Config: true
+Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: as-debootstrap-unshare-wrapper
 Modes: unshare
 Needs-Root: true
 Variants: minbase -
 Needs-APT-Config: true
+Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: help
 
@@ -56,8 +59,20 @@ Needs-Root: true
 Test: mount-is-missing
 Needs-QEMU: true
 
-Test: check-for-bit-by-bit-identical-format-output
+Test: mmdebstrap
 Needs-Root: true
+Modes: root
+Formats: tar squashfs ext2
+Variants: essential apt minbase buildd - standard
+Skip-If:
+ variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
+ variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
+ fmt == "squashfs" and dist == "oldstable" # squashfs-tools-ng is not available
+ fmt == "ext2" and dist == "oldstable" # genext2fs does not support SOURCE_DATE_EPOCH
+ mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
+ variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
+
+Test: check-for-bit-by-bit-identical-format-output
 Modes: unshare fakechroot
 Formats: tar squashfs ext2
 Variants: essential apt minbase buildd - standard
@@ -191,12 +206,14 @@ Skip-If:
 
 Test: include-libmagic-mgc-arm64
 Needs-Root: true
+Needs-APT-Config: true
 Skip-If:
  hostarch != "amd64"
  not run_ma_same_tests
 
 Test: include-libmagic-mgc-arm64-with-multiple-arch-options
 Needs-Root: true
+Needs-APT-Config: true
 Skip-If:
  hostarch != "amd64"
  not run_ma_same_tests

make_mirror.sh

@@ -174,10 +174,22 @@ END
 		for f in /etc/apt/sources.list /etc/apt/sources.list.d/*; do
 			[ -e "$f" ] || continue
 			[ -e "$rootdir/$f" ] && echo >> "$rootdir/$f"
-			# we do not add entries from deb.debian.org or
+			# Filter out file:// repositories as they are added
-			# otherwise tests will fail if mirror pushes happen
+			# to each mmdebstrap call verbatim by
-			# while the script is running
+			# debian/tests/copy_host_apt_config
-			grep -v deb.debian.org/debian "$f" >> "$rootdir/$f" || :
+			# Also filter out all mirrors that are not of suite
+			# $DEFAULT_DIST, except experimental if the suite
+			# is unstable. This prevents packages from
+			# unstable entering a testing mirror.
+			if [ "$dist" = unstable ]; then
+				grep -v ' file://' "$f" \
+					| grep -E " (unstable|experimental) " \
+					>> "$rootdir/$f" || :
+			else
+				grep -v ' file://' "$f" \
+					| grep " $DEFAULT_DIST " \
+					>> "$rootdir/$f" || :
+			fi
 		done
 		for f in /etc/apt/preferences.d/*; do
 			[ -e "$f" ] || continue
@@ -186,6 +198,13 @@ END
 		done
 	fi
 
+	echo "creating mirror for $dist" >&2
+	for f in /etc/apt/sources.list /etc/apt/sources.list.d/* /etc/apt/preferences.d/*; do
+		[ -e "$rootdir/$f" ] || continue
+		echo "contents of $f:" >&2
+		cat "$rootdir/$f" >&2
+	done
+
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update
 
 	pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \

mmdebstrap

@@ -6481,16 +6481,16 @@ comma or whitespace.
 
 =item B<--components>=I<comp1>[,I<comp2>,...]
 
-Comma or whitespace separated list of components like main, contrib and
+Comma or whitespace separated list of components like main, contrib, non-free
-non-free which will be used for all URI-only I<MIRROR> arguments. The option
+and non-free-firmware which will be used for all URI-only I<MIRROR> arguments.
-can be specified multiple times and the components are concatenated in the
+The option can be specified multiple times and the components are concatenated
-order in which they are given on the command line. If later list items are
+in the order in which they are given on the command line. If later list items
-repeated, then they get dropped so that the resulting component list is free
+are repeated, then they get dropped so that the resulting component list is
-of duplicates. So the following are equivalent:
+free of duplicates. So the following are equivalent:
 
-    --components="main contrib non-free"
+    --components="main contrib non-free non-free-firmware"
-    --components=main,contrib,non-free
+    --components=main,contrib,non-free,non-free-firmware
-    --comp=main --comp="contrib non-free" --comp="main,non-free"
+    --comp=main --comp="contrib non-free" --comp="main,non-free-firmware"
 
 =item B<--architectures>=I<native>[,I<foreign1>,...]
 

tests/as-debootstrap-unshare-wrapper

@@ -89,7 +89,6 @@ rm /tmp/debian-mm/var/cache/apt/archives/lock
 rm /tmp/debian-mm/var/lib/apt/lists/lock
 rm /tmp/debian-mm/var/lib/dpkg/arch
 
-# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
 # also needed for users that are created by systemd-sysusers before systemd 252
 # https://github.com/systemd/systemd/pull/24534
 for f in shadow shadow-; do

tests/check-against-debootstrap-dist

@@ -183,23 +183,6 @@ if [ "{{ VARIANT }}" = "-" ]; then
 	esac
 fi
 
-# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
-case {{ DIST }} in oldstable|stable)
-for f in shadow shadow-; do
-	if [ ! -e /tmp/debian-{{ DIST }}-mm/etc/$f ]; then
-		continue
-	fi
-	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/etc/$f /tmp/debian-{{ DIST }}-mm/etc/$f >&2; then
-		echo patching /etc/$f on {{ DIST }} {{ VARIANT }} >&2
-		awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-{{ DIST }}-mm/etc/$f > /tmp/debian-{{ DIST }}-mm/etc/$f.bak
-		cat /tmp/debian-{{ DIST }}-mm/etc/$f.bak > /tmp/debian-{{ DIST }}-mm/etc/$f
-		rm /tmp/debian-{{ DIST }}-mm/etc/$f.bak
-	else
-		echo no difference for /etc/$f on {{ DIST }} {{ VARIANT }} >&2
-	fi
-done;;
-esac
-
 # check if the file content differs
 diff --unified --no-dereference --recursive /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm >&2
 

tests/check-for-bit-by-bit-identical-format-output

@@ -3,9 +3,7 @@ set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
-trap "rm -f /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} /tmp/debian-chroot-root.{{ FORMAT }}" EXIT INT TERM
+trap "rm -f /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }}" EXIT INT TERM
-
-[ "$(id -u)" -eq 0 ]
 
 if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
 	if [ ! -e /mmdebstrap-testenv ]; then
@@ -14,23 +12,9 @@ if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
 	fi
 	useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 fi
-prefix="runuser -u ${SUDO_USER:-user} --"
+runuser -u "${SUDO_USER:-user}" -- {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} {{ MIRROR }}
-
+cmp ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} \
-{{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-root.{{ FORMAT }} {{ MIRROR }}
+	|| diffoscope ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }}
-if [ "{{ FORMAT }}" = tar ]; then
-	printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 /tmp/debian-chroot-root.tar -
-elif [ "{{ FORMAT }}" = squashfs ]; then
-	printf 'hsqs' | cmp --bytes=4 /tmp/debian-chroot-root.squashfs -
-elif [ "{{ FORMAT }}" = ext2 ]; then
-	printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 /tmp/debian-chroot-root.ext2 -
-else
-	echo "unknown format: {{ FORMAT }}" >&2
-	exit 1
-fi
-
-$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} {{ MIRROR }}
-cmp /tmp/debian-chroot-root.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} \
-	|| diffoscope /tmp/debian-chroot-root.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }}
 
 # we cannot test chrootless mode here, because mmdebstrap relies on the
 # usrmerge package to set up merged-/usr and that doesn't work in chrootless

tests/mmdebstrap

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+
+[ "$(id -u)" -eq 0 ]
+[ {{ MODE }} = "root" ]
+case {{ FORMAT }} in tar|squashfs|ext2) : ;; *) exit 1;; esac
+
+{{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} {{ MIRROR }}
+if [ "{{ FORMAT }}" = tar ]; then
+	printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar -
+elif [ "{{ FORMAT }}" = squashfs ]; then
+	printf 'hsqs' | cmp --bytes=4 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.squashfs -
+elif [ "{{ FORMAT }}" = ext2 ]; then
+	printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext2 -
+else
+	echo "unknown format: {{ FORMAT }}" >&2
+	exit 1
+fi