release 1.3.3

CHANGELOG.md

@@ -1,3 +1,8 @@
+1.3.3 (2023-02-19)
+------------------
+
+ - testsuite improvements
+
 1.3.2 (2023-02-16)
 ------------------
 

coverage.py

@@ -15,6 +15,7 @@ from itertools import product
 have_qemu = os.getenv("HAVE_QEMU", "yes") == "yes"
 have_binfmt = os.getenv("HAVE_BINFMT", "yes") == "yes"
 run_ma_same_tests = os.getenv("RUN_MA_SAME_TESTS", "yes") == "yes"
+use_host_apt_config = os.getenv("USE_HOST_APT_CONFIG", "no") == "yes"
 cmd = os.getenv("CMD", "./mmdebstrap")
 
 default_dist = os.getenv("DEFAULT_DIST", "unstable")
@@ -93,6 +94,7 @@ def parse_config(confname):
                     "Skip-If",
                     "Needs-QEMU",
                     "Needs-Root",
+                    "Needs-APT-Config",
                 ]:
                     print(f"Unknown field name {k} in test {name}")
                     exit(1)
@@ -266,6 +268,10 @@ def main():
             skipreason = skip(test.get("Skip-If"), dist, mode, variant, fmt)
             if skipreason:
                 tt = ("skip", skipreason)
+            elif (
+                test.get("Needs-APT-Config", "false") == "true" and use_host_apt_config
+            ):
+                tt = ("skip", "test cannot use host apt config")
             elif have_qemu:
                 tt = "qemu"
             elif test.get("Needs-QEMU", "false") == "true":

coverage.sh

@@ -92,4 +92,4 @@ END
 	echo
 fi
 
-rm shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/tarfilter shared/proxysolver
+rm -f shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/tarfilter shared/proxysolver

coverage.txt

@@ -2,16 +2,19 @@ Test: debootstrap
 Dists: any
 Variants: minbase buildd -
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: check-against-debootstrap-dist
 Dists: any
 Variants: minbase buildd -
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: as-debootstrap-unshare-wrapper
 Modes: unshare
 Needs-Root: true
 Variants: minbase -
+Needs-APT-Config: true
 
 Test: help
 
@@ -27,6 +30,7 @@ Needs-Root: true
 
 Test: dist-using-codename
 Dists: any
+Needs-APT-Config: true
 
 Test: fail-without-etc-subuid
 Needs-QEMU: true
@@ -36,12 +40,15 @@ Needs-QEMU: true
 
 Test: unshare-as-root-user-inside-chroot
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: root-mode-inside-chroot
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: root-mode-inside-unshare-chroot
 Modes: unshare
+Needs-APT-Config: true
 
 Test: root-without-cap-sys-admin
 Needs-Root: true
@@ -122,6 +129,7 @@ Test: read-from-stdin-write-to-stdout
 Test: supply-components-manually
 Modes: root
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: stable-default-mirror
 Needs-QEMU: true
@@ -146,19 +154,23 @@ Needs-QEMU: true
 Test: mirror-is-deb
 
 Test: mirror-is-real-file
+Needs-APT-Config: true
 
 Test: deb822-1-2
 Modes: root
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: deb822-2-2
 Modes: root
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: automatic-mirror-from-suite
 Needs-QEMU: true
 
 Test: invalid-mirror
+Needs-APT-Config: true
 
 Test: fail-installing-to-root
 Modes: root
@@ -198,6 +210,7 @@ Needs-QEMU: true
 
 Test: keyring-overwrites
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: signed-by-without-host-keys
 Needs-QEMU: true
@@ -207,6 +220,7 @@ Needs-QEMU: true
 
 Test: signed-by-with-host-keys
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: dpkgopt
 Needs-Root: true
@@ -240,9 +254,11 @@ Needs-Root: true
 
 Test: special-hooks-using-helpers
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: special-hooks-using-helpers-and-env-vars
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: special-hooks-with-mode-mode
 Modes: root unshare fakechroot
@@ -261,6 +277,7 @@ Needs-Root: true
 
 Test: logfile
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: without-etc-resolv-conf-and-etc-hostname
 Needs-QEMU: true
@@ -295,10 +312,12 @@ Modes: any
 Test: unpack-doc-debian
 Modes: root fakechroot
 Variants: extract
+Needs-APT-Config: true
 
 Test: install-doc-debian
 Modes: chrootless
 Variants: custom
+Needs-APT-Config: true
 
 Test: chrootless
 Variants: essential
@@ -312,6 +331,7 @@ Variants: essential
 Modes: chrootless
 Skip-If:
  dist in ["oldstable", "stable"]
+ hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
 
 Test: chrootless-foreign
 Variants: essential
@@ -325,10 +345,12 @@ Needs-QEMU: true
 Test: install-doc-debian-and-output-tarball
 Variants: custom
 Modes: chrootless
+Needs-APT-Config: true
 
 Test: install-doc-debian-and-test-hooks
 Variants: custom
 Modes: chrootless
+Needs-APT-Config: true
 
 Test: install-libmagic-mgc-on-arm64
 Variants: custom
@@ -358,17 +380,20 @@ Test: error-if-stdout-is-tty
 Test: variant-custom-timeout
 
 Test: include-deb-file
+Needs-APT-Config: true
 
 Test: unshare-include-deb
 Modes: unshare
 
 Test: pivot_root
 Modes: root unshare
+Needs-APT-Config: true
 
 Test: jessie-or-older
 Needs-Root: true
 Modes: root unshare fakechroot
 Variants: essential apt minbase
+Skip-If: mode == "fakechroot" and hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
 
 Test: apt-patterns
 
@@ -378,6 +403,8 @@ Test: empty-sources.list
 
 Test: merged-fakechroot-inside-unmerged-chroot
 Needs-Root: true
+Needs-APT-Config: true
+Skip-If: hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
 
 Test: auto-mode-as-normal-user
 Modes: auto

make_mirror.sh

@@ -176,6 +176,8 @@ cleanupapt() {
 		"$rootdir/var/lib/dpkg/lock-frontend" \
 		"$rootdir/var/lib/dpkg/lock" \
 		"$rootdir/etc/apt/apt.conf" \
+		"$rootdir/etc/apt/sources.list.d/"* \
+		"$rootdir/etc/apt/preferences.d/"* \
 		"$rootdir/etc/apt/sources.list" \
 		"$rootdir/oldaptnames" \
 		"$rootdir/newaptnames" \
@@ -234,6 +236,24 @@ END
 
 	: > "$rootdir/var/lib/dpkg/status"
 
+	if [ "$dist" = "$DEFAULT_DIST" ] && [ "$nativearch" = "$HOSTARCH" ] && [ "$USE_HOST_APT_CONFIG" = "yes" ]; then
+		# we append sources and settings instead of overwriting after
+		# an empty line
+		for f in /etc/apt/sources.list /etc/apt/sources.list.d/*; do
+			[ -e "$f" ] || continue
+			[ -e "$rootdir/$f" ] && echo >> "$rootdir/$f"
+			# we do not add entries from deb.debian.org or
+			# otherwise tests will fail if mirror pushes happen
+			# while the script is running
+			grep -v deb.debian.org/debian "$f" >> "$rootdir/$f" || :
+		done
+		for f in /etc/apt/preferences.d/*; do
+			[ -e "$f" ] || continue
+			[ -e "$rootdir/$f" ] && echo >> "$rootdir/$f"
+			cat "$f" >> "$rootdir/$f"
+		done
+	fi
+
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update
 
 	# before downloading packages and before replacing the old Packages
@@ -397,10 +417,13 @@ security_mirror="http://security.debian.org/debian-security"
 components=main
 
 : "${DEFAULT_DIST:=unstable}"
+: "${ONLY_DEFAULT_DIST:=no}"
+: "${ONLY_HOSTARCH:=no}"
 : "${HAVE_QEMU:=yes}"
 : "${RUN_MA_SAME_TESTS:=yes}"
 # by default, use the mmdebstrap executable in the current directory
 : "${CMD:=./mmdebstrap}"
+: "${USE_HOST_APT_CONFIG:=no}"
 
 if [ -e "$oldmirrordir/dists/$DEFAULT_DIST/Release" ]; then
 	http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/Release" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/Release")
@@ -430,6 +453,13 @@ for nativearch in $arches; do
 		if [ "$nativearch" != "$HOSTARCH" ] && [ "$DEFAULT_DIST" != "$dist" ]; then
 			continue
 		fi
+		# if ONLY_DEFAULT_DIST is set, only download DEFAULT_DIST
+		if [ "$ONLY_DEFAULT_DIST" = "yes" ] && [ "$DEFAULT_DIST" != "$dist" ]; then
+			continue
+		fi
+		if [ "$ONLY_HOSTARCH" = "yes" ] && [ "$nativearch" != "$HOSTARCH" ]; then
+			continue
+		fi
 		# we need a first pass without updates and security patches
 		# because otherwise, old package versions needed by
 		# debootstrap will not get included

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '1.3.2';
+our $VERSION = '1.3.3';
 
 use English;
 use Getopt::Long;

tests/create-directory

@@ -1,7 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
 {{ CMD }} --mode=root --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 chroot /tmp/debian-chroot dpkg-query --showformat '${binary:Package}\n' --show > pkglist.txt
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort > tar1.txt
-rm -r /tmp/debian-chroot

tests/fail-installing-to-existing-file

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap "rm -f /tmp/exists" EXIT INT TERM
+
 touch /tmp/exists
 ret=0
 {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/exists {{ MIRROR }} || ret=$?

tests/fail-with-path-with-quotes

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap 'rm -rf /tmp/quoted\"path' EXIT INT TERM
+
 ret=0
 {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/quoted\"path {{ MIRROR }} || ret=$?
 if [ "$ret" = 0 ]; then

tests/include-deb-file

@@ -3,6 +3,8 @@
 set -eu
 export LC_ALL=C.UTF-8
 
+trap "rm -rf /tmp/dummypkg.deb /tmp/dummypkg" EXIT INT TERM
+
 # instead of obtaining a .deb from our cache, we create a new package because
 # otherwise apt might decide to download the package with the same name and
 # version from the cache instead of using the local .deb

tests/install-busybox-based-sub-essential-system

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
 pkgs=base-files,base-passwd,busybox,debianutils,dpkg,libc-bin,mawk,tar
 # busybox --install -s will install symbolic links into the rootfs, leaving
 # existing files untouched. It has to run after extraction (otherwise there is
@@ -31,4 +34,3 @@ chroot /tmp/debian-chroot echo foobar \
 	| chroot /tmp/debian-chroot tee /dev/null \
 	| chroot /tmp/debian-chroot sed 's/foobar/blubber/' \
 	| chroot /tmp/debian-chroot grep blubber >/dev/null
-rm -r /tmp/debian-chroot

tests/install-doc-debian

@@ -5,6 +5,8 @@ export LC_ALL=C.UTF-8
 [ {{ VARIANT }} = "custom" ]
 [ {{ MODE }} = "chrootless" ]
 
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
 prefix=
 if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
 	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then

tests/install-doc-debian-and-test-hooks

@@ -6,6 +6,8 @@ export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 [ {{ VARIANT }} = "custom" ]
 [ {{ MODE }} = "chrootless" ]
 
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
 prefix=
 if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
 	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then

tests/logfile

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap "rm -rf /tmp/debian-chroot /tmp/log /tmp/trimmed" EXIT INT TERM
+
 # we check the full log to also prevent debug printfs to accidentally make it into a commit
 {{ CMD }} --mode=root --variant=apt --logfile=/tmp/log {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 # omit the last line which should contain the runtime
@@ -17,5 +20,3 @@ I: cleaning package lists and apt cache...
 LOG
 tail --lines=1 /tmp/log | grep '^I: success in .* seconds$'
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
-rm -r /tmp/debian-chroot
-rm /tmp/log /tmp/trimmed

tests/multiple-include

@@ -8,6 +8,7 @@ rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
 rm /tmp/debian-chroot/usr/share/lintian/overrides/tzdata
 rm /tmp/debian-chroot/etc/localtime
+rm /tmp/debian-chroot/etc/timezone
 rm -r /tmp/debian-chroot/usr/share/doc/tzdata
 rm -r /tmp/debian-chroot/usr/share/zoneinfo
 rm /tmp/debian-chroot/var/lib/apt/extended_states

tests/root-mode-inside-chroot

@@ -6,9 +6,12 @@
 set -eu
 export LC_ALL=C.UTF-8
 [ "$(whoami)" = "root" ]
+
+trap "rm -f /tmp/debian-chroot.tar script.sh" EXIT INT TERM
+
 cat << 'SCRIPT' > script.sh
 #!/bin/sh
-set -eu
+set -exu
 rootfs="$1"
 mkdir -p "$rootfs/mnt"
 [ -e /usr/bin/mmdebstrap ] && cp -aT /usr/bin/mmdebstrap "$rootfs/usr/bin/mmdebstrap"
@@ -23,4 +26,3 @@ chmod +x script.sh
 	--customize-hook="download /tmp/debian-chroot.tar /tmp/debian-chroot.tar" \
 	{{ DIST }} /dev/null {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
-rm /tmp/debian-chroot.tar script.sh

tests/unpack-doc-debian

@@ -2,6 +2,8 @@
 set -eu
 export LC_ALL=C.UTF-8
 
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
 [ {{ VARIANT }} = extract ]
 
 prefix=

tests/unshare-include-deb

@@ -5,6 +5,8 @@ export LC_ALL=C.UTF-8
 
 [ "{{ MODE }}" = unshare ]
 
+trap "rm -rf /tmp/dummypkg.deb /tmp/dummypkg" EXIT INT TERM
+
 prefix=
 if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
 	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then