remove workarounds for #1010957

This allows running the hook from older mmdebstrap versions.

coverage.py

@@ -10,6 +10,7 @@ import argparse
 import time
 from datetime import timedelta
 from collections import defaultdict
+from itertools import product
 
 have_qemu = os.getenv("HAVE_QEMU", "yes") == "yes"
 have_unshare = os.getenv("HAVE_UNSHARE", "yes") == "yes"
@@ -66,6 +67,71 @@ def skip(condition, dist, mode, variant, fmt):
     return ""
 
 
+def parse_config(confname):
+    config_dict = defaultdict(dict)
+    config_order = list()
+    all_vals = {
+        "Dists": all_dists,
+        "Modes": all_modes,
+        "Variants": all_variants,
+        "Formats": all_formats,
+    }
+    with open(confname) as f:
+        for test in Deb822.iter_paragraphs(f):
+            if "Test" not in test.keys():
+                print("Test without name", file=sys.stderr)
+                exit(1)
+            name = test["Test"]
+            config_order.append(name)
+            for k in test.keys():
+                v = test[k]
+                if k not in [
+                    "Test",
+                    "Dists",
+                    "Modes",
+                    "Variants",
+                    "Formats",
+                    "Skip-If",
+                    "Needs-QEMU",
+                    "Needs-Root",
+                ]:
+                    print(f"Unknown field name {k} in test {name}")
+                    exit(1)
+                if k in all_vals.keys():
+                    if v == "default":
+                        print(
+                            f"Setting {k} to default in Test {name} is redundant",
+                            file=sys.stderr,
+                        )
+                        exit(1)
+                    if v == "any":
+                        v = all_vals[k]
+                    else:
+                        # else, split the value by whitespace
+                        v = v.split()
+                        for i in v:
+                            if i not in all_vals[k]:
+                                print(
+                                    f"{i} is not a valid value for {k}", file=sys.stderr
+                                )
+                                exit(1)
+                config_dict[name][k] = v
+    return config_order, config_dict
+
+
+def format_failed(num, total, name, dist, mode, variant, fmt, config_dict):
+    ret = f"({num}/{total}) {name}"
+    if len(config_dict[name].get("Dists", [])) > 0:
+        ret += f" --dist={dist}"
+    if len(config_dict[name].get("Modes", [])) > 0:
+        ret += f" --mode={mode}"
+    if len(config_dict[name].get("Variants", [])) > 0:
+        ret += f" --variant={variant}"
+    if len(config_dict[name].get("Formats", [])) > 0:
+        ret += f" --format={fmt}"
+    return ret
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("test", nargs="*", help="only run these tests")
@@ -86,10 +152,25 @@ def main():
         default=0,
         help="exit after first num failures or errors.",
     )
-    parser.add_argument("--mode", metavar="mode", help="only run tests with this mode")
-    parser.add_argument("--dist", metavar="dist", help="only run tests with this dist")
     parser.add_argument(
-        "--variant", metavar="variant", help="only run tests with this variant"
+        "--mode",
+        metavar="mode",
+        help=f"only run tests with this mode (Default = {default_mode})",
+    )
+    parser.add_argument(
+        "--dist",
+        metavar="dist",
+        help=f"only run tests with this dist (Default = {default_dist})",
+    )
+    parser.add_argument(
+        "--variant",
+        metavar="variant",
+        help=f"only run tests with this variant (Default = {default_variant})",
+    )
+    parser.add_argument(
+        "--format",
+        metavar="format",
+        help=f"only run tests with this format (Default = {default_format})",
     )
     args = parser.parse_args()
 
@@ -120,62 +201,38 @@ def main():
             "/usr/share/mmdebstrap/hooks", "shared/hooks", dirs_exist_ok=True
         )
 
+    # parse coverage.txt
+    config_order, config_dict = parse_config("coverage.txt")
+
+    # produce the list of tests using the cartesian product of all allowed
+    # dists, modes, variants and formats of a given test
     tests = []
-    with open("coverage.txt") as f:
-        for test in Deb822.iter_paragraphs(f):
-            name = test["Test"]
-            dists = test.get("Dists", default_dist)
-            if dists == "any":
-                dists = all_dists
-            elif dists == "default":
-                dists = [default_dist]
+    for name in config_order:
+        test = config_dict[name]
+        for dist, mode, variant, fmt in product(
+            test.get("Dists", [default_dist]),
+            test.get("Modes", [default_mode]),
+            test.get("Variants", [default_variant]),
+            test.get("Formats", [default_format]),
+        ):
+            skipreason = skip(test.get("Skip-If"), dist, mode, variant, fmt)
+            if skipreason:
+                tt = ("skip", skipreason)
+            elif have_qemu:
+                tt = "qemu"
+            elif test.get("Needs-QEMU", "false") == "true":
+                tt = ("skip", "test needs QEMU")
+            elif test.get("Needs-Root", "false") == "true":
+                tt = "sudo"
+            elif mode == "auto" and not have_unshare:
+                tt = "sudo"
+            elif mode == "root":
+                tt = "sudo"
+            elif mode == "unshare" and not have_unshare:
+                tt = ("skip", "test needs unshare")
             else:
-                dists = dists.split()
-            modes = test.get("Modes", default_mode)
-            if modes == "any":
-                modes = all_modes
-            elif modes == "default":
-                modes = [default_mode]
-            else:
-                modes = modes.split()
-            variants = test.get("Variants", default_variant)
-            if variants == "any":
-                variants = all_variants
-            elif variants == "default":
-                variants = [default_variant]
-            else:
-                variants = variants.split()
-            formats = test.get("Formats", default_format)
-            if formats == "any":
-                formats = all_formats
-            elif formats == "default":
-                formats = [default_format]
-            else:
-                formats = formats.split()
-            for dist in dists:
-                for mode in modes:
-                    for variant in variants:
-                        for fmt in formats:
-                            skipreason = skip(
-                                test.get("Skip-If"), dist, mode, variant, fmt
-                            )
-                            if skipreason:
-                                tt = ("skip", skipreason)
-                            elif have_qemu:
-                                tt = "qemu"
-                            elif test.get("Needs-QEMU", "false") == "true":
-                                tt = ("skip", "test needs QEMU")
-                            elif test.get("Needs-Root", "false") == "true":
-                                tt = "sudo"
-                            elif mode == "auto" and not have_unshare:
-                                tt = "sudo"
-                            elif mode == "root":
-                                tt = "sudo"
-                            elif mode == "unshare" and not have_unshare:
-                                tt = ("skip", "test needs unshare")
-                            else:
-                                tt = "null"
-                            tests.append((tt, name, dist, mode, variant, fmt))
+                tt = "null"
+            tests.append((tt, name, dist, mode, variant, fmt))
 
     torun = []
     num_tests = len(tests)
@@ -262,6 +319,9 @@ def main():
         if args.variant and args.variant != variant:
             print(f"skipping because of --variant={args.variant}", file=sys.stderr)
             continue
+        if args.format and args.format != fmt:
+            print(f"skipping because of --format={args.format}", file=sys.stderr)
+            continue
         proc = subprocess.Popen(argv)
         try:
             proc.wait()
@@ -272,7 +332,9 @@ def main():
         print(separator, file=sys.stderr)
         if proc.returncode != 0:
             failed.append(
-                ("(%d/%d) %s" % (i + 1, len(tests), name), dist, mode, variant, fmt)
+                format_failed(
+                    i + 1, len(tests), name, dist, mode, variant, fmt, config_dict
+                )
             )
             print("result: FAILURE", file=sys.stderr)
         else:
@@ -292,8 +354,8 @@ def main():
                 print(f"    {t}", file=sys.stderr)
     if failed:
         print("failed %d:" % len(failed), file=sys.stderr)
-        for t in failed:
-            print(f"    {t}", file=sys.stderr)
+        for f in failed:
+            print(f, file=sys.stderr)
         exit(1)
 
 

coverage.txt

@@ -45,7 +45,7 @@ Needs-QEMU: true
 Test: check-for-bit-by-bit-identical-format-output
 Needs-QEMU: true
 Formats: tar squashfs ext2
-Variants: essential apt minbase buildd important standard
+Variants: essential apt minbase buildd - standard
 Skip-If:
  variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
  variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
@@ -207,9 +207,6 @@ Needs-Root: true
 Test: include-with-multiple-apt-sources
 Needs-Root: true
 
-Test: merged-usr-via-setup-hook
-Needs-Root: true
-
 Test: essential-hook
 Needs-Root: true
 

hooks/busybox/extract00.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
 	set -x
 fi
 

hooks/busybox/setup00.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
 	set -x
 fi
 

hooks/eatmydata/customize.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
 	set -x
 fi
 

hooks/eatmydata/extract.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
 	set -x
 fi
 

hooks/file-mirror-automount/customize00.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
 	set -x
 fi
 

hooks/file-mirror-automount/setup00.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
 	set -x
 fi
 

hooks/merged-usr/essential00.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
         set -x
 fi
 

hooks/merged-usr/setup00.sh

@@ -41,7 +41,7 @@
 
 set -eu
 
-if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
 	set -x
 fi
 

hooks/no-merged-usr/essential00.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
         set -x
 fi
 

hooks/no-merged-usr/setup00.sh

@@ -11,7 +11,7 @@
 
 set -eu
 
-if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
         set -x
 fi
 

run_qemu.sh

@@ -30,6 +30,9 @@ trap cleanup INT TERM EXIT
 qemu-img create -f qcow2 -b "$(realpath $cachedir)/debian-$DEFAULT_DIST.qcow" -F qcow2 "$tmpdir/debian-$DEFAULT_DIST-overlay.qcow"
 # to connect to serial use:
 #   minicom -D 'unix#/tmp/ttyS0'
+#
+# or this (quit with ctrl+q):
+#   socat stdin,raw,echo=0,escape=0x11 unix-connect:/tmp/ttyS0
 ret=0
 timeout --foreground 20m qemu-system-x86_64 \
 	-cpu host \

tests/as-debootstrap-unshare-wrapper

@@ -45,6 +45,7 @@ rm /tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_mai
 rm /tmp/debian-debootstrap/etc/machine-id /tmp/debian-mm/etc/machine-id
 rm /tmp/debian-mm/var/cache/apt/archives/lock
 rm /tmp/debian-mm/var/lib/apt/lists/lock
+rm /tmp/debian-mm/var/lib/dpkg/arch
 
 # check if the file content differs
 diff --no-dereference --recursive /tmp/debian-debootstrap /tmp/debian-mm >&2

tests/check-against-debootstrap-dist

@@ -77,6 +77,8 @@ rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status-old \
 # remove dpkg files
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/available
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/cmethopt
+# remove /var/lib/dpkg/arch
+rm /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/arch
 # since we installed packages directly from the .deb files, Priorities differ
 # thus we first check for equality and then remove the files
 chroot /tmp/debian-{{ DIST }}-debootstrap dpkg --list > dpkg1

tests/check-for-bit-by-bit-identical-format-output

@@ -8,7 +8,6 @@ fi
 adduser --gecos user --disabled-password user
 sysctl -w kernel.unprivileged_userns_clone=1
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-mount -o size=4G -t tmpfs tmpfs /tmp # workaround for #1010957
 {{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-root.{{ FORMAT }} {{ MIRROR }}
 if [ "{{ FORMAT }}" = tar ]; then
 	printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 /tmp/debian-chroot-root.tar -

tests/compare-output-with-pre-seeded-var-cache-apt-archives

@@ -14,7 +14,6 @@ include="--include=doc-debian"
 if [ "{{ VARIANT }}" = "custom" ]; then
 	include="$include,base-files,base-passwd,coreutils,dash,diffutils,dpkg,libc-bin,sed"
 fi
-mount -o size=4G -t tmpfs tmpfs /tmp # workaround for #1010957
 {{ CMD }} $include --mode={{ MODE }} --variant={{ VARIANT }} \
 	--setup-hook='mkdir -p "$1"/var/cache/apt/archives/partial' \
 	--setup-hook='touch "$1"/var/cache/apt/archives/lock' \

tests/create-directory-dry-run

@@ -24,5 +24,6 @@ rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
 rm /tmp/debian-chroot/var/lib/dpkg/status
+rm /tmp/debian-chroot/var/lib/dpkg/arch
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir

tests/include-libmagic-mgc-arm64

@@ -10,7 +10,6 @@ set -eu
 export LC_ALL=C.UTF-8
 {{ CMD }} --mode=root --variant=apt --architectures=amd64,arm64 --include=libmagic-mgc:arm64 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 { echo "amd64"; echo "arm64"; } | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
-rm /tmp/debian-chroot/var/lib/dpkg/arch
 rm /tmp/debian-chroot/var/lib/apt/extended_states
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums

tests/include-libmagic-mgc-arm64-with-multiple-arch-options

@@ -3,7 +3,6 @@ set -eu
 export LC_ALL=C.UTF-8
 {{ CMD }} --mode=root --variant=apt --architectures=amd64 --architectures=arm64 --include=libmagic-mgc:arm64 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 { echo "amd64"; echo "arm64"; } | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
-rm /tmp/debian-chroot/var/lib/dpkg/arch
 rm /tmp/debian-chroot/var/lib/apt/extended_states
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums

tests/install-doc-debian

@@ -24,6 +24,7 @@ rm /tmp/debian-chroot/etc/fstab
 rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/dpkg/status
+rm /tmp/debian-chroot/var/lib/dpkg/arch
 rm /tmp/debian-chroot/var/cache/apt/archives/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock-frontend

tests/install-doc-debian-and-test-hooks

@@ -27,6 +27,7 @@ rm /tmp/debian-chroot/etc/fstab
 rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/dpkg/status
+rm /tmp/debian-chroot/var/lib/dpkg/arch
 rm /tmp/debian-chroot/var/cache/apt/archives/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock-frontend

tests/merged-usr-via-setup-hook

@@ -1,50 +0,0 @@
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
-{{ CMD }} --mode=root --variant=apt \
-	--essential-hook=./hooks/merged-usr/essential00.sh \
-	--setup-hook=./hooks/merged-usr/setup00.sh \
-	--customize-hook='[ -L "$1"/bin -a -L "$1"/sbin -a -L "$1"/lib ]' \
-	{{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort > tar2.txt
-{
-	sed -e 's/^\.\/bin\//.\/usr\/bin\//;s/^\.\/lib\//.\/usr\/lib\//;s/^\.\/sbin\//.\/usr\/sbin\//;' tar1.txt | {
-		case {{ HOSTARCH }} in
-		amd64) sed -e 's/^\.\/lib32\//.\/usr\/lib32\//;s/^\.\/lib64\//.\/usr\/lib64\//;s/^\.\/libx32\//.\/usr\/libx32\//;';;
-		ppc64el) sed -e 's/^\.\/lib64\//.\/usr\/lib64\//;';;
-		*) cat;;
-		esac
-	};
-	echo ./bin;
-	echo ./lib;
-	echo ./sbin;
-	case {{ HOSTARCH }} in
-	amd64)
-		echo ./lib32;
-		echo ./lib64;
-		echo ./libx32;
-		echo ./usr/lib32/;
-		echo ./usr/libx32/;
-		;;
-	i386)
-		echo ./lib64;
-		echo ./libx32;
-		echo ./usr/lib64/;
-		echo ./usr/libx32/;
-		;;
-	ppc64el)
-		echo ./lib64;
-		;;
-	s390x)
-		echo ./lib32;
-		echo ./usr/lib32/;
-		;;
-	esac
-	echo ./usr/share/doc/usr-is-merged/
-	echo ./usr/share/doc/usr-is-merged/changelog.gz
-	echo ./usr/share/doc/usr-is-merged/copyright
-	echo ./var/lib/dpkg/info/usr-is-merged.list
-	echo ./var/lib/dpkg/info/usr-is-merged.md5sums
-	echo ./var/lib/dpkg/info/usr-is-merged.preinst
-} | sort -u | diff -u - tar2.txt

tests/preserve-mode-of-etc-resolv-conf-and-etc-hostname

@@ -38,6 +38,7 @@ rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
 rm /tmp/debian-chroot/var/lib/dpkg/status
+rm /tmp/debian-chroot/var/lib/dpkg/arch
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir
 for f in /etc/resolv.conf /etc/hostname; do
@@ -66,6 +67,7 @@ rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
 rm /tmp/debian-chroot/var/lib/dpkg/status
+rm /tmp/debian-chroot/var/lib/dpkg/arch
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir
 for f in /etc/resolv.conf /etc/hostname; do
@@ -95,5 +97,6 @@ rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
 rm /tmp/debian-chroot/var/lib/dpkg/status
+rm /tmp/debian-chroot/var/lib/dpkg/arch
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir

tests/unpack-doc-debian

@@ -29,6 +29,7 @@ rm /tmp/debian-chroot/etc/fstab
 rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/dpkg/status
+rm /tmp/debian-chroot/var/lib/dpkg/arch
 rm /tmp/debian-chroot/var/cache/apt/archives/lock
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
 ## delete merged usr symlinks