caching_proxy.py

@@ -46,12 +46,7 @@ class ProxyRequestHandler(http.server.BaseHTTPRequestHandler):
             newpath = pathlib.Path("/dev/null")
 
         # copy from oldpath to newpath and send back to client
-        # Only take files from the old cache if they are .deb files or Packages
+        if oldpath.exists():
-        # files in the by-hash directory as only those are unique by their path
-        # name. Other files like InRelease files have to be downloaded afresh.
-        if oldpath.exists() and (
-            oldpath.suffix == ".deb" or "by-hash" in oldpath.parts
-        ):
             print(f"proxy cached: {self.path}", file=sys.stderr)
             self.wfile.write(b"HTTP/1.1 200 OK\r\n")
             self.send_header("Content-Length", oldpath.stat().st_size)

coverage.txt

@@ -3,21 +3,18 @@ Dists: any
 Variants: minbase buildd -
 Needs-Root: true
 Needs-APT-Config: true
-Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: check-against-debootstrap-dist
 Dists: any
 Variants: minbase buildd -
 Needs-Root: true
 Needs-APT-Config: true
-Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: as-debootstrap-unshare-wrapper
 Modes: unshare
 Needs-Root: true
 Variants: minbase -
 Needs-APT-Config: true
-Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: help
 
@@ -59,20 +56,8 @@ Needs-Root: true
 Test: mount-is-missing
 Needs-QEMU: true
 
-Test: mmdebstrap
-Needs-Root: true
-Modes: root
-Formats: tar squashfs ext2
-Variants: essential apt minbase buildd - standard
-Skip-If:
- variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
- variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
- fmt == "squashfs" and dist == "oldstable" # squashfs-tools-ng is not available
- fmt == "ext2" and dist == "oldstable" # genext2fs does not support SOURCE_DATE_EPOCH
- mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
- variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
-
 Test: check-for-bit-by-bit-identical-format-output
+Needs-Root: true
 Modes: unshare fakechroot
 Formats: tar squashfs ext2
 Variants: essential apt minbase buildd - standard
@@ -206,14 +191,12 @@ Skip-If:
 
 Test: include-libmagic-mgc-arm64
 Needs-Root: true
-Needs-APT-Config: true
 Skip-If:
  hostarch != "amd64"
  not run_ma_same_tests
 
 Test: include-libmagic-mgc-arm64-with-multiple-arch-options
 Needs-Root: true
-Needs-APT-Config: true
 Skip-If:
  hostarch != "amd64"
  not run_ma_same_tests

make_mirror.sh

@@ -174,22 +174,10 @@ END
 		for f in /etc/apt/sources.list /etc/apt/sources.list.d/*; do
 			[ -e "$f" ] || continue
 			[ -e "$rootdir/$f" ] && echo >> "$rootdir/$f"
-			# Filter out file:// repositories as they are added
+			# we do not add entries from deb.debian.org or
-			# to each mmdebstrap call verbatim by
+			# otherwise tests will fail if mirror pushes happen
-			# debian/tests/copy_host_apt_config
+			# while the script is running
-			# Also filter out all mirrors that are not of suite
+			grep -v deb.debian.org/debian "$f" >> "$rootdir/$f" || :
-			# $DEFAULT_DIST, except experimental if the suite
-			# is unstable. This prevents packages from
-			# unstable entering a testing mirror.
-			if [ "$dist" = unstable ]; then
-				grep -v ' file://' "$f" \
-					| grep -E " (unstable|experimental) " \
-					>> "$rootdir/$f" || :
-			else
-				grep -v ' file://' "$f" \
-					| grep " $DEFAULT_DIST " \
-					>> "$rootdir/$f" || :
-			fi
 		done
 		for f in /etc/apt/preferences.d/*; do
 			[ -e "$f" ] || continue
@@ -198,13 +186,6 @@ END
 		done
 	fi
 
-	echo "creating mirror for $dist" >&2
-	for f in /etc/apt/sources.list /etc/apt/sources.list.d/* /etc/apt/preferences.d/*; do
-		[ -e "$rootdir/$f" ] || continue
-		echo "contents of $f:" >&2
-		cat "$rootdir/$f" >&2
-	done
-
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update
 
 	pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \

mmdebstrap

@@ -6481,16 +6481,16 @@ comma or whitespace.
 
 =item B<--components>=I<comp1>[,I<comp2>,...]
 
-Comma or whitespace separated list of components like main, contrib, non-free
+Comma or whitespace separated list of components like main, contrib and
-and non-free-firmware which will be used for all URI-only I<MIRROR> arguments.
+non-free which will be used for all URI-only I<MIRROR> arguments. The option
-The option can be specified multiple times and the components are concatenated
+can be specified multiple times and the components are concatenated in the
-in the order in which they are given on the command line. If later list items
+order in which they are given on the command line. If later list items are
-are repeated, then they get dropped so that the resulting component list is
+repeated, then they get dropped so that the resulting component list is free
-free of duplicates. So the following are equivalent:
+of duplicates. So the following are equivalent:
 
-    --components="main contrib non-free non-free-firmware"
+    --components="main contrib non-free"
-    --components=main,contrib,non-free,non-free-firmware
+    --components=main,contrib,non-free
-    --comp=main --comp="contrib non-free" --comp="main,non-free-firmware"
+    --comp=main --comp="contrib non-free" --comp="main,non-free"
 
 =item B<--architectures>=I<native>[,I<foreign1>,...]
 

tests/as-debootstrap-unshare-wrapper

@@ -89,6 +89,7 @@ rm /tmp/debian-mm/var/cache/apt/archives/lock
 rm /tmp/debian-mm/var/lib/apt/lists/lock
 rm /tmp/debian-mm/var/lib/dpkg/arch
 
+# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
 # also needed for users that are created by systemd-sysusers before systemd 252
 # https://github.com/systemd/systemd/pull/24534
 for f in shadow shadow-; do

tests/check-against-debootstrap-dist

@@ -183,6 +183,23 @@ if [ "{{ VARIANT }}" = "-" ]; then
 	esac
 fi
 
+# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
+case {{ DIST }} in oldstable|stable)
+for f in shadow shadow-; do
+	if [ ! -e /tmp/debian-{{ DIST }}-mm/etc/$f ]; then
+		continue
+	fi
+	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/etc/$f /tmp/debian-{{ DIST }}-mm/etc/$f >&2; then
+		echo patching /etc/$f on {{ DIST }} {{ VARIANT }} >&2
+		awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-{{ DIST }}-mm/etc/$f > /tmp/debian-{{ DIST }}-mm/etc/$f.bak
+		cat /tmp/debian-{{ DIST }}-mm/etc/$f.bak > /tmp/debian-{{ DIST }}-mm/etc/$f
+		rm /tmp/debian-{{ DIST }}-mm/etc/$f.bak
+	else
+		echo no difference for /etc/$f on {{ DIST }} {{ VARIANT }} >&2
+	fi
+done;;
+esac
+
 # check if the file content differs
 diff --unified --no-dereference --recursive /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm >&2
 

tests/check-for-bit-by-bit-identical-format-output

@@ -3,7 +3,9 @@ set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
-trap "rm -f /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }}" EXIT INT TERM
+trap "rm -f /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} /tmp/debian-chroot-root.{{ FORMAT }}" EXIT INT TERM
+
+[ "$(id -u)" -eq 0 ]
 
 if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
 	if [ ! -e /mmdebstrap-testenv ]; then
@@ -12,9 +14,23 @@ if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
 	fi
 	useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 fi
-runuser -u "${SUDO_USER:-user}" -- {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} {{ MIRROR }}
+prefix="runuser -u ${SUDO_USER:-user} --"
-cmp ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} \
+
-	|| diffoscope ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }}
+{{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-root.{{ FORMAT }} {{ MIRROR }}
+if [ "{{ FORMAT }}" = tar ]; then
+	printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 /tmp/debian-chroot-root.tar -
+elif [ "{{ FORMAT }}" = squashfs ]; then
+	printf 'hsqs' | cmp --bytes=4 /tmp/debian-chroot-root.squashfs -
+elif [ "{{ FORMAT }}" = ext2 ]; then
+	printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 /tmp/debian-chroot-root.ext2 -
+else
+	echo "unknown format: {{ FORMAT }}" >&2
+	exit 1
+fi
+
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} {{ MIRROR }}
+cmp /tmp/debian-chroot-root.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} \
+	|| diffoscope /tmp/debian-chroot-root.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }}
 
 # we cannot test chrootless mode here, because mmdebstrap relies on the
 # usrmerge package to set up merged-/usr and that doesn't work in chrootless

tests/mmdebstrap

@@ -1,20 +0,0 @@
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-
-[ "$(id -u)" -eq 0 ]
-[ {{ MODE }} = "root" ]
-case {{ FORMAT }} in tar|squashfs|ext2) : ;; *) exit 1;; esac
-
-{{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} {{ MIRROR }}
-if [ "{{ FORMAT }}" = tar ]; then
-	printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar -
-elif [ "{{ FORMAT }}" = squashfs ]; then
-	printf 'hsqs' | cmp --bytes=4 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.squashfs -
-elif [ "{{ FORMAT }}" = ext2 ]; then
-	printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext2 -
-else
-	echo "unknown format: {{ FORMAT }}" >&2
-	exit 1
-fi