coverage.py

@@ -10,7 +10,6 @@ import argparse
 import time
 from datetime import timedelta
 from collections import defaultdict
-from itertools import product
 
 have_qemu = os.getenv("HAVE_QEMU", "yes") == "yes"
 have_unshare = os.getenv("HAVE_UNSHARE", "yes") == "yes"
@@ -67,71 +66,6 @@ def skip(condition, dist, mode, variant, fmt):
     return ""
 
 
-def parse_config(confname):
-    config_dict = defaultdict(dict)
-    config_order = list()
-    all_vals = {
-        "Dists": all_dists,
-        "Modes": all_modes,
-        "Variants": all_variants,
-        "Formats": all_formats,
-    }
-    with open(confname) as f:
-        for test in Deb822.iter_paragraphs(f):
-            if "Test" not in test.keys():
-                print("Test without name", file=sys.stderr)
-                exit(1)
-            name = test["Test"]
-            config_order.append(name)
-            for k in test.keys():
-                v = test[k]
-                if k not in [
-                    "Test",
-                    "Dists",
-                    "Modes",
-                    "Variants",
-                    "Formats",
-                    "Skip-If",
-                    "Needs-QEMU",
-                    "Needs-Root",
-                ]:
-                    print(f"Unknown field name {k} in test {name}")
-                    exit(1)
-                if k in all_vals.keys():
-                    if v == "default":
-                        print(
-                            f"Setting {k} to default in Test {name} is redundant",
-                            file=sys.stderr,
-                        )
-                        exit(1)
-                    if v == "any":
-                        v = all_vals[k]
-                    else:
-                        # else, split the value by whitespace
-                        v = v.split()
-                        for i in v:
-                            if i not in all_vals[k]:
-                                print(
-                                    f"{i} is not a valid value for {k}", file=sys.stderr
-                                )
-                                exit(1)
-                config_dict[name][k] = v
-    return config_order, config_dict
-
-
-def format_failed(num, total, name, dist, mode, variant, fmt, config_dict):
-    ret = f"({num}/{total}) {name}"
-    if len(config_dict[name].get("Dists", [])) > 0:
-        ret += f" --dist={dist}"
-    if len(config_dict[name].get("Modes", [])) > 0:
-        ret += f" --mode={mode}"
-    if len(config_dict[name].get("Variants", [])) > 0:
-        ret += f" --variant={variant}"
-    if len(config_dict[name].get("Formats", [])) > 0:
-        ret += f" --format={fmt}"
-    return ret
-
-
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("test", nargs="*", help="only run these tests")
@@ -152,25 +86,10 @@ def main():
         default=0,
         help="exit after first num failures or errors.",
     )
+    parser.add_argument("--mode", metavar="mode", help="only run tests with this mode")
+    parser.add_argument("--dist", metavar="dist", help="only run tests with this dist")
     parser.add_argument(
-        "--mode",
+        "--variant", metavar="variant", help="only run tests with this variant"
-        metavar="mode",
-        help=f"only run tests with this mode (Default = {default_mode})",
-    )
-    parser.add_argument(
-        "--dist",
-        metavar="dist",
-        help=f"only run tests with this dist (Default = {default_dist})",
-    )
-    parser.add_argument(
-        "--variant",
-        metavar="variant",
-        help=f"only run tests with this variant (Default = {default_variant})",
-    )
-    parser.add_argument(
-        "--format",
-        metavar="format",
-        help=f"only run tests with this format (Default = {default_format})",
     )
     args = parser.parse_args()
 
@@ -201,38 +120,62 @@ def main():
             "/usr/share/mmdebstrap/hooks", "shared/hooks", dirs_exist_ok=True
         )
 
-    # parse coverage.txt
-    config_order, config_dict = parse_config("coverage.txt")
-
-    # produce the list of tests using the cartesian product of all allowed
-    # dists, modes, variants and formats of a given test
     tests = []
-    for name in config_order:
+    with open("coverage.txt") as f:
-        test = config_dict[name]
+        for test in Deb822.iter_paragraphs(f):
-        for dist, mode, variant, fmt in product(
+            name = test["Test"]
-            test.get("Dists", [default_dist]),
+            dists = test.get("Dists", default_dist)
-            test.get("Modes", [default_mode]),
+            if dists == "any":
-            test.get("Variants", [default_variant]),
+                dists = all_dists
-            test.get("Formats", [default_format]),
+            elif dists == "default":
-        ):
+                dists = [default_dist]
-            skipreason = skip(test.get("Skip-If"), dist, mode, variant, fmt)
-            if skipreason:
-                tt = ("skip", skipreason)
-            elif have_qemu:
-                tt = "qemu"
-            elif test.get("Needs-QEMU", "false") == "true":
-                tt = ("skip", "test needs QEMU")
-            elif test.get("Needs-Root", "false") == "true":
-                tt = "sudo"
-            elif mode == "auto" and not have_unshare:
-                tt = "sudo"
-            elif mode == "root":
-                tt = "sudo"
-            elif mode == "unshare" and not have_unshare:
-                tt = ("skip", "test needs unshare")
             else:
-                tt = "null"
+                dists = dists.split()
-            tests.append((tt, name, dist, mode, variant, fmt))
+            modes = test.get("Modes", default_mode)
+            if modes == "any":
+                modes = all_modes
+            elif modes == "default":
+                modes = [default_mode]
+            else:
+                modes = modes.split()
+            variants = test.get("Variants", default_variant)
+            if variants == "any":
+                variants = all_variants
+            elif variants == "default":
+                variants = [default_variant]
+            else:
+                variants = variants.split()
+            formats = test.get("Formats", default_format)
+            if formats == "any":
+                formats = all_formats
+            elif formats == "default":
+                formats = [default_format]
+            else:
+                formats = formats.split()
+            for dist in dists:
+                for mode in modes:
+                    for variant in variants:
+                        for fmt in formats:
+                            skipreason = skip(
+                                test.get("Skip-If"), dist, mode, variant, fmt
+                            )
+                            if skipreason:
+                                tt = ("skip", skipreason)
+                            elif have_qemu:
+                                tt = "qemu"
+                            elif test.get("Needs-QEMU", "false") == "true":
+                                tt = ("skip", "test needs QEMU")
+                            elif test.get("Needs-Root", "false") == "true":
+                                tt = "sudo"
+                            elif mode == "auto" and not have_unshare:
+                                tt = "sudo"
+                            elif mode == "root":
+                                tt = "sudo"
+                            elif mode == "unshare" and not have_unshare:
+                                tt = ("skip", "test needs unshare")
+                            else:
+                                tt = "null"
+                            tests.append((tt, name, dist, mode, variant, fmt))
 
     torun = []
     num_tests = len(tests)
@@ -319,9 +262,6 @@ def main():
         if args.variant and args.variant != variant:
             print(f"skipping because of --variant={args.variant}", file=sys.stderr)
             continue
-        if args.format and args.format != fmt:
-            print(f"skipping because of --format={args.format}", file=sys.stderr)
-            continue
         proc = subprocess.Popen(argv)
         try:
             proc.wait()
@@ -332,9 +272,7 @@ def main():
         print(separator, file=sys.stderr)
         if proc.returncode != 0:
             failed.append(
-                format_failed(
+                ("(%d/%d) %s" % (i + 1, len(tests), name), dist, mode, variant, fmt)
-                    i + 1, len(tests), name, dist, mode, variant, fmt, config_dict
-                )
             )
             print("result: FAILURE", file=sys.stderr)
         else:
@@ -354,8 +292,8 @@ def main():
                 print(f"    {t}", file=sys.stderr)
     if failed:
         print("failed %d:" % len(failed), file=sys.stderr)
-        for f in failed:
+        for t in failed:
-            print(f, file=sys.stderr)
+            print(f"    {t}", file=sys.stderr)
         exit(1)
 
 

coverage.txt

@@ -45,7 +45,7 @@ Needs-QEMU: true
 Test: check-for-bit-by-bit-identical-format-output
 Needs-QEMU: true
 Formats: tar squashfs ext2
-Variants: essential apt minbase buildd - standard
+Variants: essential apt minbase buildd important standard
 Skip-If:
  variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
  variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
@@ -207,6 +207,9 @@ Needs-Root: true
 Test: include-with-multiple-apt-sources
 Needs-Root: true
 
+Test: merged-usr-via-setup-hook
+Needs-Root: true
+
 Test: essential-hook
 Needs-Root: true
 

hooks/busybox/extract00.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
 	set -x
 fi
 

hooks/busybox/setup00.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
 	set -x
 fi
 

hooks/eatmydata/customize.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
 	set -x
 fi
 

hooks/eatmydata/extract.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
 	set -x
 fi
 

hooks/file-mirror-automount/customize00.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
 	set -x
 fi
 

hooks/file-mirror-automount/setup00.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
 	set -x
 fi
 

hooks/merged-usr/essential00.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
         set -x
 fi
 

hooks/merged-usr/setup00.sh

@@ -41,7 +41,7 @@
 
 set -eu
 
-if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
 	set -x
 fi
 

hooks/no-merged-usr/essential00.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
         set -x
 fi
 

hooks/no-merged-usr/setup00.sh

@@ -11,7 +11,7 @@
 
 set -eu
 
-if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+if [ "$MMDEBSTRAP_VERBOSITY" -ge 3 ]; then
         set -x
 fi
 

run_qemu.sh

@@ -30,9 +30,6 @@ trap cleanup INT TERM EXIT
 qemu-img create -f qcow2 -b "$(realpath $cachedir)/debian-$DEFAULT_DIST.qcow" -F qcow2 "$tmpdir/debian-$DEFAULT_DIST-overlay.qcow"
 # to connect to serial use:
 #   minicom -D 'unix#/tmp/ttyS0'
-#
-# or this (quit with ctrl+q):
-#   socat stdin,raw,echo=0,escape=0x11 unix-connect:/tmp/ttyS0
 ret=0
 timeout --foreground 20m qemu-system-x86_64 \
 	-cpu host \

tests/as-debootstrap-unshare-wrapper

@@ -45,7 +45,6 @@ rm /tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_mai
 rm /tmp/debian-debootstrap/etc/machine-id /tmp/debian-mm/etc/machine-id
 rm /tmp/debian-mm/var/cache/apt/archives/lock
 rm /tmp/debian-mm/var/lib/apt/lists/lock
-rm /tmp/debian-mm/var/lib/dpkg/arch
 
 # check if the file content differs
 diff --no-dereference --recursive /tmp/debian-debootstrap /tmp/debian-mm >&2

tests/check-against-debootstrap-dist

@@ -77,8 +77,6 @@ rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status-old \
 # remove dpkg files
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/available
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/cmethopt
-# remove /var/lib/dpkg/arch
-rm /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/arch
 # since we installed packages directly from the .deb files, Priorities differ
 # thus we first check for equality and then remove the files
 chroot /tmp/debian-{{ DIST }}-debootstrap dpkg --list > dpkg1

tests/check-for-bit-by-bit-identical-format-output

@@ -8,6 +8,7 @@ fi
 adduser --gecos user --disabled-password user
 sysctl -w kernel.unprivileged_userns_clone=1
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+mount -o size=4G -t tmpfs tmpfs /tmp # workaround for #1010957
 {{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-root.{{ FORMAT }} {{ MIRROR }}
 if [ "{{ FORMAT }}" = tar ]; then
 	printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 /tmp/debian-chroot-root.tar -

tests/compare-output-with-pre-seeded-var-cache-apt-archives

@@ -14,6 +14,7 @@ include="--include=doc-debian"
 if [ "{{ VARIANT }}" = "custom" ]; then
 	include="$include,base-files,base-passwd,coreutils,dash,diffutils,dpkg,libc-bin,sed"
 fi
+mount -o size=4G -t tmpfs tmpfs /tmp # workaround for #1010957
 {{ CMD }} $include --mode={{ MODE }} --variant={{ VARIANT }} \
 	--setup-hook='mkdir -p "$1"/var/cache/apt/archives/partial' \
 	--setup-hook='touch "$1"/var/cache/apt/archives/lock' \

tests/create-directory-dry-run

@@ -24,6 +24,5 @@ rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
 rm /tmp/debian-chroot/var/lib/dpkg/status
-rm /tmp/debian-chroot/var/lib/dpkg/arch
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir

tests/include-libmagic-mgc-arm64

@@ -10,6 +10,7 @@ set -eu
 export LC_ALL=C.UTF-8
 {{ CMD }} --mode=root --variant=apt --architectures=amd64,arm64 --include=libmagic-mgc:arm64 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 { echo "amd64"; echo "arm64"; } | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
+rm /tmp/debian-chroot/var/lib/dpkg/arch
 rm /tmp/debian-chroot/var/lib/apt/extended_states
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums

tests/include-libmagic-mgc-arm64-with-multiple-arch-options

@@ -3,6 +3,7 @@ set -eu
 export LC_ALL=C.UTF-8
 {{ CMD }} --mode=root --variant=apt --architectures=amd64 --architectures=arm64 --include=libmagic-mgc:arm64 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 { echo "amd64"; echo "arm64"; } | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
+rm /tmp/debian-chroot/var/lib/dpkg/arch
 rm /tmp/debian-chroot/var/lib/apt/extended_states
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums

tests/install-doc-debian

@@ -24,7 +24,6 @@ rm /tmp/debian-chroot/etc/fstab
 rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/dpkg/status
-rm /tmp/debian-chroot/var/lib/dpkg/arch
 rm /tmp/debian-chroot/var/cache/apt/archives/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock-frontend

tests/install-doc-debian-and-test-hooks

@@ -27,7 +27,6 @@ rm /tmp/debian-chroot/etc/fstab
 rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/dpkg/status
-rm /tmp/debian-chroot/var/lib/dpkg/arch
 rm /tmp/debian-chroot/var/cache/apt/archives/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock-frontend

tests/merged-usr-via-setup-hook

@@ -0,0 +1,50 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+{{ CMD }} --mode=root --variant=apt \
+	--essential-hook=./hooks/merged-usr/essential00.sh \
+	--setup-hook=./hooks/merged-usr/setup00.sh \
+	--customize-hook='[ -L "$1"/bin -a -L "$1"/sbin -a -L "$1"/lib ]' \
+	{{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort > tar2.txt
+{
+	sed -e 's/^\.\/bin\//.\/usr\/bin\//;s/^\.\/lib\//.\/usr\/lib\//;s/^\.\/sbin\//.\/usr\/sbin\//;' tar1.txt | {
+		case {{ HOSTARCH }} in
+		amd64) sed -e 's/^\.\/lib32\//.\/usr\/lib32\//;s/^\.\/lib64\//.\/usr\/lib64\//;s/^\.\/libx32\//.\/usr\/libx32\//;';;
+		ppc64el) sed -e 's/^\.\/lib64\//.\/usr\/lib64\//;';;
+		*) cat;;
+		esac
+	};
+	echo ./bin;
+	echo ./lib;
+	echo ./sbin;
+	case {{ HOSTARCH }} in
+	amd64)
+		echo ./lib32;
+		echo ./lib64;
+		echo ./libx32;
+		echo ./usr/lib32/;
+		echo ./usr/libx32/;
+		;;
+	i386)
+		echo ./lib64;
+		echo ./libx32;
+		echo ./usr/lib64/;
+		echo ./usr/libx32/;
+		;;
+	ppc64el)
+		echo ./lib64;
+		;;
+	s390x)
+		echo ./lib32;
+		echo ./usr/lib32/;
+		;;
+	esac
+	echo ./usr/share/doc/usr-is-merged/
+	echo ./usr/share/doc/usr-is-merged/changelog.gz
+	echo ./usr/share/doc/usr-is-merged/copyright
+	echo ./var/lib/dpkg/info/usr-is-merged.list
+	echo ./var/lib/dpkg/info/usr-is-merged.md5sums
+	echo ./var/lib/dpkg/info/usr-is-merged.preinst
+} | sort -u | diff -u - tar2.txt

tests/preserve-mode-of-etc-resolv-conf-and-etc-hostname

@@ -38,7 +38,6 @@ rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
 rm /tmp/debian-chroot/var/lib/dpkg/status
-rm /tmp/debian-chroot/var/lib/dpkg/arch
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir
 for f in /etc/resolv.conf /etc/hostname; do
@@ -67,7 +66,6 @@ rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
 rm /tmp/debian-chroot/var/lib/dpkg/status
-rm /tmp/debian-chroot/var/lib/dpkg/arch
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir
 for f in /etc/resolv.conf /etc/hostname; do
@@ -97,6 +95,5 @@ rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
 rm /tmp/debian-chroot/var/lib/dpkg/status
-rm /tmp/debian-chroot/var/lib/dpkg/arch
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir

tests/unpack-doc-debian

@@ -29,7 +29,6 @@ rm /tmp/debian-chroot/etc/fstab
 rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/dpkg/status
-rm /tmp/debian-chroot/var/lib/dpkg/arch
 rm /tmp/debian-chroot/var/cache/apt/archives/lock
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
 ## delete merged usr symlinks