coverage.py: support USE_HOST_APT_CONFIG and new Needs-APT-Config field

coverage.py

@@ -15,6 +15,7 @@ from itertools import product
 have_qemu = os.getenv("HAVE_QEMU", "yes") == "yes"
 have_binfmt = os.getenv("HAVE_BINFMT", "yes") == "yes"
 run_ma_same_tests = os.getenv("RUN_MA_SAME_TESTS", "yes") == "yes"
+use_host_apt_config = os.getenv("USE_HOST_APT_CONFIG", "no") == "yes"
 cmd = os.getenv("CMD", "./mmdebstrap")
 
 default_dist = os.getenv("DEFAULT_DIST", "unstable")
@@ -93,6 +94,7 @@ def parse_config(confname):
                     "Skip-If",
                     "Needs-QEMU",
                     "Needs-Root",
+                    "Needs-APT-Config",
                 ]:
                     print(f"Unknown field name {k} in test {name}")
                     exit(1)
@@ -266,6 +268,10 @@ def main():
             skipreason = skip(test.get("Skip-If"), dist, mode, variant, fmt)
             if skipreason:
                 tt = ("skip", skipreason)
+            elif (
+                test.get("Needs-APT-Config", "false") == "true" and use_host_apt_config
+            ):
+                tt = ("skip", "test cannot use host apt config")
             elif have_qemu:
                 tt = "qemu"
             elif test.get("Needs-QEMU", "false") == "true":

coverage.txt

@@ -2,16 +2,19 @@ Test: debootstrap
 Dists: any
 Variants: minbase buildd -
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: check-against-debootstrap-dist
 Dists: any
 Variants: minbase buildd -
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: as-debootstrap-unshare-wrapper
 Modes: unshare
 Needs-Root: true
 Variants: minbase -
+Needs-APT-Config: true
 
 Test: help
 
@@ -27,6 +30,7 @@ Needs-Root: true
 
 Test: dist-using-codename
 Dists: any
+Needs-APT-Config: true
 
 Test: fail-without-etc-subuid
 Needs-QEMU: true
@@ -36,12 +40,15 @@ Needs-QEMU: true
 
 Test: unshare-as-root-user-inside-chroot
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: root-mode-inside-chroot
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: root-mode-inside-unshare-chroot
 Modes: unshare
+Needs-APT-Config: true
 
 Test: root-without-cap-sys-admin
 Needs-Root: true
@@ -122,6 +129,7 @@ Test: read-from-stdin-write-to-stdout
 Test: supply-components-manually
 Modes: root
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: stable-default-mirror
 Needs-QEMU: true
@@ -146,19 +154,23 @@ Needs-QEMU: true
 Test: mirror-is-deb
 
 Test: mirror-is-real-file
+Needs-APT-Config: true
 
 Test: deb822-1-2
 Modes: root
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: deb822-2-2
 Modes: root
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: automatic-mirror-from-suite
 Needs-QEMU: true
 
 Test: invalid-mirror
+Needs-APT-Config: true
 
 Test: fail-installing-to-root
 Modes: root
@@ -198,6 +210,7 @@ Needs-QEMU: true
 
 Test: keyring-overwrites
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: signed-by-without-host-keys
 Needs-QEMU: true
@@ -207,6 +220,7 @@ Needs-QEMU: true
 
 Test: signed-by-with-host-keys
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: dpkgopt
 Needs-Root: true
@@ -240,9 +254,11 @@ Needs-Root: true
 
 Test: special-hooks-using-helpers
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: special-hooks-using-helpers-and-env-vars
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: special-hooks-with-mode-mode
 Modes: root unshare fakechroot
@@ -261,6 +277,7 @@ Needs-Root: true
 
 Test: logfile
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: without-etc-resolv-conf-and-etc-hostname
 Needs-QEMU: true
@@ -295,10 +312,12 @@ Modes: any
 Test: unpack-doc-debian
 Modes: root fakechroot
 Variants: extract
+Needs-APT-Config: true
 
 Test: install-doc-debian
 Modes: chrootless
 Variants: custom
+Needs-APT-Config: true
 
 Test: chrootless
 Variants: essential
@@ -325,10 +344,12 @@ Needs-QEMU: true
 Test: install-doc-debian-and-output-tarball
 Variants: custom
 Modes: chrootless
+Needs-APT-Config: true
 
 Test: install-doc-debian-and-test-hooks
 Variants: custom
 Modes: chrootless
+Needs-APT-Config: true
 
 Test: install-libmagic-mgc-on-arm64
 Variants: custom
@@ -358,12 +379,14 @@ Test: error-if-stdout-is-tty
 Test: variant-custom-timeout
 
 Test: include-deb-file
+Needs-APT-Config: true
 
 Test: unshare-include-deb
 Modes: unshare
 
 Test: pivot_root
 Modes: root unshare
+Needs-APT-Config: true
 
 Test: jessie-or-older
 Needs-Root: true
@@ -378,6 +401,7 @@ Test: empty-sources.list
 
 Test: merged-fakechroot-inside-unmerged-chroot
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: auto-mode-as-normal-user
 Modes: auto

make_mirror.sh

@@ -176,6 +176,8 @@ cleanupapt() {
 		"$rootdir/var/lib/dpkg/lock-frontend" \
 		"$rootdir/var/lib/dpkg/lock" \
 		"$rootdir/etc/apt/apt.conf" \
+		"$rootdir/etc/apt/sources.list.d/"* \
+		"$rootdir/etc/apt/preferences.d/"* \
 		"$rootdir/etc/apt/sources.list" \
 		"$rootdir/oldaptnames" \
 		"$rootdir/newaptnames" \
@@ -234,6 +236,24 @@ END
 
 	: > "$rootdir/var/lib/dpkg/status"
 
+	if [ "$dist" = "$DEFAULT_DIST" ] && [ "$nativearch" = "$HOSTARCH" ] && [ "$USE_HOST_APT_CONFIG" = "yes" ]; then
+		# we append sources and settings instead of overwriting after
+		# an empty line
+		for f in /etc/apt/sources.list /etc/apt/sources.list.d/*; do
+			[ -e "$f" ] || continue
+			[ -e "$rootdir/$f" ] && echo >> "$rootdir/$f"
+			# we do not add entries from deb.debian.org or
+			# otherwise tests will fail if mirror pushes happen
+			# while the script is running
+			grep -v deb.debian.org/debian "$f" >> "$rootdir/$f" || :
+		done
+		for f in /etc/apt/preferences.d/*; do
+			[ -e "$f" ] || continue
+			[ -e "$rootdir/$f" ] && echo >> "$rootdir/$f"
+			cat "$f" >> "$rootdir/$f"
+		done
+	fi
+
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update
 
 	# before downloading packages and before replacing the old Packages
@@ -401,6 +421,7 @@ components=main
 : "${RUN_MA_SAME_TESTS:=yes}"
 # by default, use the mmdebstrap executable in the current directory
 : "${CMD:=./mmdebstrap}"
+: "${USE_HOST_APT_CONFIG:=no}"
 
 if [ -e "$oldmirrordir/dists/$DEFAULT_DIST/Release" ]; then
 	http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/Release" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/Release")