release 1.5.4

iputils-ping does not ship /bin/ping with xattrs anymore.

.mailmap

@@ -1,4 +1,6 @@
 Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
 Johannes Schauer Marin Rodrigues <josch@mister-muffin.de> <j.schauer@email.de>
+Johannes Schauer Marin Rodrigues <josch@mister-muffin.de> <josch@debian.org>
+Johannes Schauer Marin Rodrigues <josch@mister-muffin.de> <Johannes Schauer Marin Rodrigues josch@debian.org>
 Helmut Grohne <helmut@subdivi.de> <helmut.grohne@intenta.de>
 Benjamin Drung <benjamin.drung@ionos.com> <benjamin.drung@cloud.ionos.com>

CHANGELOG.md

@@ -1,3 +1,120 @@
+1.5.4 (2024-10-28)
+------------------
+
+ - do not generate apt sources.list entry if SUITE is empty
+
+1.5.3 (2024-09-13)
+------------------
+
+ - tidy up any zombie processes
+ - chrootless hurd-i386
+ - add --skip=cleanup/reproducible/machine-id
+ - m-a-b-q: replace test_installed by dpkg-checkbuilddeps
+
+1.5.2 (2024-06-26)
+------------------
+
+ - mmdebstrap-autopkgtest-build-qemu produces bit-by-bit reproducible output
+
+1.5.1 (2024-06-03)
+------------------
+
+ - in root and unshare mode, run 'mount --make-rprivate /' before bind-mounting
+ - switch apt variant from using 'apt-get dist-upgrade' to apt patterns
+
+1.5.0 (2024-05-14)
+------------------
+
+ - add --format=ext4
+
+1.4.3 (2024-02-01)
+------------------
+
+ - take hard links into account when computing disk usage
+
+1.4.2 (2024-01-29)
+------------------
+
+ - allow for start-stop-daemon to be in either /sbin or /usr/sbin
+ - mmdebstrap-autopkgtest-build-qemu: fix octal mode computation and hostname
+
+1.4.1 (2024-01-09)
+------------------
+
+ - set DPkg::Chroot-Directory in APT_CONFIG to simplify calling apt in hooks
+ - disallow running chrootless as root without fakeroot unless
+   --skip=check/chrootless is used
+ - only print short --help output if wrong args are passed
+ - read files passed as --aptopt and --dpkgopt outside the unshared namespace
+
+1.4.0 (2023-10-24)
+------------------
+
+ - add mmdebstrap-autopkgtest-build-qemu
+ - export container=mmdebstrap-unshare env variable in unshare-mode hooks
+ - add new skip options: output/dev, output/mknod, tar-in/mknod,
+   copy-in/mknod, sync-in/mknod
+ - stop copying qemu-$arch-static binary into the chroot
+ - tarfilter: add --type-exclude option
+ - set MMDEBSTRAP_FORMAT in hooks
+ - do not install priority:required in buildd variant following debootstrap
+
+1.3.8 (2023-08-20)
+------------------
+
+ - hooks/merged-usr: implement post-merging as debootstrap does
+ - exclude ./lost+found from tarball
+
+1.3.7 (2023-06-21)
+------------------
+
+ - add hooks/copy-host-apt-sources-and-preferences
+
+1.3.6 (2023-06-16)
+------------------
+
+ - bugfix release
+
+1.3.5 (2023-03-20)
+------------------
+
+ - bugfix release
+
+1.3.4 (2023-03-16)
+------------------
+
+ - more safeguards before automatically choosing unshare mode
+
+1.3.3 (2023-02-19)
+------------------
+
+ - testsuite improvements
+
+1.3.2 (2023-02-16)
+------------------
+
+ - unshare mode works in privileged docker containers
+
+1.3.1 (2023-01-20)
+------------------
+
+ - bugfix release
+
+1.3.0 (2023-01-16)
+------------------
+
+ - add hooks/maybe-jessie-or-older and hooks/maybe-merged-usr
+ - add --skip=check/signed-by
+ - hooks/jessie-or-older: split into two individual hook files
+ - skip running apt-get update if we are very sure that it was already run
+ - be more verbose when 'apt-get update' failed
+ - warn if a hook is named like one but not executable and if a hook is
+   executable but not named like one
+ - to find signed-by value, run gpg on the individual keys to print better
+   error messages in case it fails (gpg doesn't give an indication which file
+   it was unable to read) and print progress bar
+ - allow empty sources.list entries
+
 1.2.5 (2023-01-04)
 ------------------
 

README.md

@@ -23,6 +23,11 @@ For the full documentation use:
 
     pod2man ./mmdebstrap | man -l -
 
+Or read a HTML version of the man page in either of these locations:
+
+ - https://gitlab.mister-muffin.de/josch/mmdebstrap/wiki
+ - https://manpages.debian.org/unstable/mmdebstrap/mmdebstrap.1.en.html
+
 The sales pitch in comparison to debootstrap
 --------------------------------------------
 
@@ -152,14 +157,20 @@ Contributors
 ============
 
  - Johannes Schauer Marin Rodrigues (main author)
- - Gioele Barabucci
  - Helmut Grohne
- - Benjamin Drung
  - Jochen Sprickerhof
+ - Gioele Barabucci
+ - Benjamin Drung
  - Josh Triplett
  - Konstantin Demin
+ - Chris Hofstaedtler
+ - Colin Watson
  - David Kalnischkies
+ - Emilio Pozuelo Monfort
+ - Francesco Poli
+ - Jakub Wilk
  - Joe Groocock
+ - Max-Julian Pogner
  - Nicolas Vigier
  - Raul Tambre
  - Steve Dodd

caching_proxy.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import time
+import http.client
+import http.server
+from io import StringIO
+import pathlib
+import urllib.parse
+
+oldcachedir = None
+newcachedir = None
+readonly = False
+
+
+class ProxyRequestHandler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        assert int(self.headers.get("Content-Length", 0)) == 0
+        assert self.headers["Host"]
+        pathprefix = "http://" + self.headers["Host"] + "/"
+        assert self.path.startswith(pathprefix)
+        sanitizedpath = urllib.parse.unquote(self.path.removeprefix(pathprefix))
+        oldpath = oldcachedir / sanitizedpath
+        newpath = newcachedir / sanitizedpath
+
+        if not readonly:
+            newpath.parent.mkdir(parents=True, exist_ok=True)
+
+        # just send back to client
+        if newpath.exists():
+            print(f"proxy cached: {self.path}", file=sys.stderr)
+            self.wfile.write(b"HTTP/1.1 200 OK\r\n")
+            self.send_header("Content-Length", newpath.stat().st_size)
+            self.end_headers()
+            with newpath.open(mode="rb") as new:
+                while True:
+                    buf = new.read(64 * 1024)  # same as shutil uses
+                    if not buf:
+                        break
+                    self.wfile.write(buf)
+            self.wfile.flush()
+            return
+
+        if readonly:
+            newpath = pathlib.Path("/dev/null")
+
+        # copy from oldpath to newpath and send back to client
+        # Only take files from the old cache if they are .deb files or Packages
+        # files in the by-hash directory as only those are unique by their path
+        # name. Other files like InRelease files have to be downloaded afresh.
+        if oldpath.exists() and (
+            oldpath.suffix == ".deb" or "by-hash" in oldpath.parts
+        ):
+            print(f"proxy cached: {self.path}", file=sys.stderr)
+            self.wfile.write(b"HTTP/1.1 200 OK\r\n")
+            self.send_header("Content-Length", oldpath.stat().st_size)
+            self.end_headers()
+            with oldpath.open(mode="rb") as old, newpath.open(mode="wb") as new:
+                # we are not using shutil.copyfileobj() because we want to
+                # write to two file objects simultaneously
+                while True:
+                    buf = old.read(64 * 1024)  # same as shutil uses
+                    if not buf:
+                        break
+                    self.wfile.write(buf)
+                    new.write(buf)
+            self.wfile.flush()
+            return
+
+        # download fresh copy
+        try:
+            print(f"\rproxy download: {self.path}", file=sys.stderr)
+            conn = http.client.HTTPConnection(self.headers["Host"], timeout=5)
+            conn.request("GET", self.path, None, dict(self.headers))
+            res = conn.getresponse()
+            assert (res.status, res.reason) == (200, "OK"), (res.status, res.reason)
+            self.wfile.write(b"HTTP/1.1 200 OK\r\n")
+            for k, v in res.getheaders():
+                # do not allow a persistent connection
+                if k == "connection":
+                    continue
+                self.send_header(k, v)
+            self.end_headers()
+            with newpath.open(mode="wb") as f:
+                # we are not using shutil.copyfileobj() because we want to
+                # write to two file objects simultaneously and throttle the
+                # writing speed to 1024 kB/s
+                while True:
+                    buf = res.read(64 * 1024)  # same as shutil uses
+                    if not buf:
+                        break
+                    self.wfile.write(buf)
+                    f.write(buf)
+                    time.sleep(64 / 1024)  # 1024 kB/s
+            self.wfile.flush()
+        except Exception as e:
+            self.send_error(502)
+
+
+def main():
+    global oldcachedir, newcachedir, readonly
+    if sys.argv[1] == "--readonly":
+        readonly = True
+        oldcachedir = pathlib.Path(sys.argv[2])
+        newcachedir = pathlib.Path(sys.argv[3])
+    else:
+        oldcachedir = pathlib.Path(sys.argv[1])
+        newcachedir = pathlib.Path(sys.argv[2])
+    print(f"starting caching proxy for {newcachedir}", file=sys.stderr)
+    httpd = http.server.ThreadingHTTPServer(
+        server_address=("", 8080), RequestHandlerClass=ProxyRequestHandler
+    )
+    httpd.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

coverage.py

@@ -13,14 +13,14 @@ from collections import defaultdict
 from itertools import product
 
 have_qemu = os.getenv("HAVE_QEMU", "yes") == "yes"
-have_unshare = os.getenv("HAVE_UNSHARE", "yes") == "yes"
 have_binfmt = os.getenv("HAVE_BINFMT", "yes") == "yes"
 run_ma_same_tests = os.getenv("RUN_MA_SAME_TESTS", "yes") == "yes"
+use_host_apt_config = os.getenv("USE_HOST_APT_CONFIG", "no") == "yes"
 cmd = os.getenv("CMD", "./mmdebstrap")
 
 default_dist = os.getenv("DEFAULT_DIST", "unstable")
 all_dists = ["oldstable", "stable", "testing", "unstable"]
-default_mode = "auto" if have_unshare else "root"
+default_mode = "auto"
 all_modes = ["auto", "root", "unshare", "fakechroot", "chrootless"]
 default_variant = "apt"
 all_variants = [
@@ -34,12 +34,12 @@ all_variants = [
     "standard",
 ]
 default_format = "auto"
-all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "null"]
+all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "ext4", "null"]
 
 mirror = os.getenv("mirror", "http://127.0.0.1/debian")
 hostarch = subprocess.check_output(["dpkg", "--print-architecture"]).decode().strip()
 
-release_path = f"./shared/cache/debian/dists/{default_dist}/Release"
+release_path = f"./shared/cache/debian/dists/{default_dist}/InRelease"
 if not os.path.exists(release_path):
     print("path doesn't exist:", release_path, file=sys.stderr)
     print("run ./make_mirror.sh first", file=sys.stderr)
@@ -94,6 +94,7 @@ def parse_config(confname):
                     "Skip-If",
                     "Needs-QEMU",
                     "Needs-Root",
+                    "Needs-APT-Config",
                 ]:
                     print(f"Unknown field name {k} in test {name}")
                     exit(1)
@@ -119,7 +120,7 @@ def parse_config(confname):
     return config_order, config_dict
 
 
-def format_failed(num, total, name, dist, mode, variant, fmt, config_dict):
+def format_test(num, total, name, dist, mode, variant, fmt, config_dict):
     ret = f"({num}/{total}) {name}"
     if len(config_dict[name].get("Dists", [])) > 1:
         ret += f" --dist={dist}"
@@ -132,6 +133,30 @@ def format_failed(num, total, name, dist, mode, variant, fmt, config_dict):
     return ret
 
 
+def print_time_per_test(time_per_test, name="test"):
+    print(
+        f"average time per {name}:",
+        sum(time_per_test.values(), start=timedelta()) / len(time_per_test),
+        file=sys.stderr,
+    )
+    print(
+        f"median time per {name}:",
+        sorted(time_per_test.values())[len(time_per_test) // 2],
+        file=sys.stderr,
+    )
+    head_tail_num = 10
+    print(f"{head_tail_num} fastests {name}s:", file=sys.stderr)
+    for k, v in sorted(time_per_test.items(), key=lambda i: i[1])[
+        : min(head_tail_num, len(time_per_test))
+    ]:
+        print(f"    {k}: {v}", file=sys.stderr)
+    print(f"{head_tail_num} slowest {name}s:", file=sys.stderr)
+    for k, v in sorted(time_per_test.items(), key=lambda i: i[1], reverse=True)[
+        : min(head_tail_num, len(time_per_test))
+    ]:
+        print(f"    {k}: {v}", file=sys.stderr)
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("test", nargs="*", help="only run these tests")
@@ -178,7 +203,7 @@ def main():
     args = parser.parse_args()
 
     # copy over files from git or as distributed
-    for (git, dist, target) in [
+    for git, dist, target in [
         ("./mmdebstrap", "/usr/bin/mmdebstrap", "mmdebstrap"),
         ("./tarfilter", "/usr/bin/mmtarfilter", "tarfilter"),
         (
@@ -243,18 +268,18 @@ def main():
             skipreason = skip(test.get("Skip-If"), dist, mode, variant, fmt)
             if skipreason:
                 tt = ("skip", skipreason)
+            elif (
+                test.get("Needs-APT-Config", "false") == "true" and use_host_apt_config
+            ):
+                tt = ("skip", "test cannot use host apt config")
             elif have_qemu:
                 tt = "qemu"
             elif test.get("Needs-QEMU", "false") == "true":
                 tt = ("skip", "test needs QEMU")
             elif test.get("Needs-Root", "false") == "true":
                 tt = "sudo"
-            elif mode == "auto" and not have_unshare:
-                tt = "sudo"
             elif mode == "root":
                 tt = "sudo"
-            elif mode == "unshare" and not have_unshare:
-                tt = ("skip", "test needs unshare")
             else:
                 tt = "null"
             tests.append((tt, name, dist, mode, variant, fmt))
@@ -287,6 +312,8 @@ def main():
     failed = []
     num_success = 0
     num_finished = 0
+    time_per_test = {}
+    acc_time_per_test = defaultdict(list)
     for i, (test, name, dist, mode, variant, fmt) in enumerate(tests):
         if torun and i not in torun:
             continue
@@ -335,6 +362,21 @@ def main():
             check=False,
             stdout=subprocess.PIPE,
         ).stdout.decode()
+        shfmt = subprocess.run(
+            [
+                "shfmt",
+                "--posix",
+                "--binary-next-line",
+                "--case-indent",
+                "--indent",
+                "2",
+                "--simplify",
+                "-d",
+                "shared/test.sh",
+            ],
+            check=False,
+            stdout=subprocess.PIPE,
+        ).stdout.decode()
         argv = None
         match test:
             case "qemu":
@@ -345,7 +387,9 @@ def main():
                 argv = ["./run_null.sh"]
             case ("skip", reason):
                 skipped[reason].append(
-                    ("(%d/%d) %s" % (i + 1, len(tests), name), dist, mode, variant, fmt)
+                    format_test(
+                        i + 1, len(tests), name, dist, mode, variant, fmt, config_dict
+                    )
                 )
                 print(f"skipped because of {reason}", file=sys.stderr)
                 continue
@@ -365,6 +409,7 @@ def main():
         if args.format and args.format != fmt:
             print(f"skipping because of --format={args.format}", file=sys.stderr)
             continue
+        before = time.time()
         proc = subprocess.Popen(argv)
         try:
             proc.wait()
@@ -372,21 +417,28 @@ def main():
             proc.terminate()
             proc.wait()
             break
-        print(separator, file=sys.stderr)
-        if proc.returncode != 0 or shellcheck != "":
-            if shellcheck != "":
-                print(shellcheck)
-            failed.append(
-                format_failed(
+        after = time.time()
+        walltime = timedelta(seconds=int(after - before))
+        formated_test_name = format_test(
             i + 1, len(tests), name, dist, mode, variant, fmt, config_dict
         )
-            )
+        time_per_test[formated_test_name] = walltime
+        acc_time_per_test[name].append(walltime)
+        print(separator, file=sys.stderr)
+        print(f"duration: {walltime}", file=sys.stderr)
+        if proc.returncode != 0 or shellcheck != "" or shfmt != "":
+            if shellcheck != "":
+                print(shellcheck)
+            if shfmt != "":
+                print(shfmt)
+            failed.append(formated_test_name)
             print("result: FAILURE", file=sys.stderr)
         else:
             print("result: SUCCESS", file=sys.stderr)
             num_success += 1
         if args.maxfail and len(failed) >= args.maxfail:
             break
+    print(separator, file=sys.stderr)
     print(
         "successfully ran %d tests" % num_success,
         file=sys.stderr,
@@ -397,10 +449,24 @@ def main():
             print(f"skipped because of {reason}:", file=sys.stderr)
             for t in l:
                 print(f"    {t}", file=sys.stderr)
+    if len(time_per_test) > 1:
+        print_time_per_test(time_per_test)
+    if len(acc_time_per_test) > 1:
+        print_time_per_test(
+            {
+                f"{len(v)}x {k}": sum(v, start=timedelta())
+                for k, v in acc_time_per_test.items()
+            },
+            "accumulated test",
+        )
     if failed:
         print("failed %d:" % len(failed), file=sys.stderr)
         for f in failed:
             print(f, file=sys.stderr)
+    currenttime = time.time()
+    walltime = timedelta(seconds=int(currenttime - starttime))
+    print(f"total runtime: {walltime}", file=sys.stderr)
+    if failed:
         exit(1)
 
 

coverage.sh

@@ -2,11 +2,22 @@
 
 set -eu
 
-if [ -e ./mmdebstrap ]; then
+# by default, use the mmdebstrap executable in the current directory together
+# with perl Devel::Cover but allow to overwrite this
+: "${CMD:=perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap}"
+
+case "$CMD" in
+  "mmdebstrap "* | mmdebstrap | *" mmdebstrap" | *" mmdebstrap "*)
+    MMSCRIPT="$(command -v mmdebstrap 2>/dev/null)"
+    ;;
+  *) MMSCRIPT=./mmdebstrap ;;
+esac
+
+if [ -e "$MMSCRIPT" ]; then
   TMPFILE=$(mktemp)
-	perltidy < ./mmdebstrap > "$TMPFILE"
+  perltidy <"$MMSCRIPT" >"$TMPFILE"
   ret=0
-	diff -u ./mmdebstrap "$TMPFILE" || ret=$?
+  diff -u "$MMSCRIPT" "$TMPFILE" || ret=$?
   if [ "$ret" -ne 0 ]; then
     echo "perltidy failed" >&2
     rm "$TMPFILE"
@@ -14,18 +25,24 @@ if [ -e ./mmdebstrap ]; then
   fi
   rm "$TMPFILE"
 
-	if [ "$(sed -e '/^__END__$/,$d' ./mmdebstrap | wc --max-line-length)" -gt 79 ]; then
+  if [ "$(sed -e '/^__END__$/,$d' "$MMSCRIPT" | wc --max-line-length)" -gt 79 ]; then
     echo "exceeded maximum line length of 79 characters" >&2
     exit 1
   fi
 
-	perlcritic --severity 4 --verbose 8 ./mmdebstrap
+  perlcritic --severity 4 --verbose 8 "$MMSCRIPT"
+
+  pod2man "$MMSCRIPT" >/dev/null
 fi
 
-[ -e ./tarfilter ] && black --check ./tarfilter
-[ -e ./coverage.py ] && black --check ./coverage.py
+for f in tarfilter coverage.py caching_proxy.py; do
+  [ -e "./$f" ] || continue
+  black --check "./$f"
+done
 
-shellcheck --exclude=SC2016 coverage.sh make_mirror.sh run_null.sh run_qemu.sh gpgvnoexpkeysig hooks/*/*.sh
+shellcheck --exclude=SC2016 coverage.sh make_mirror.sh run_null.sh run_qemu.sh gpgvnoexpkeysig mmdebstrap-autopkgtest-build-qemu hooks/*/*.sh
+
+shfmt --binary-next-line --case-indent --indent 2 --simplify -d coverage.sh make_mirror.sh run_null.sh run_qemu.sh mmdebstrap-autopkgtest-build-qemu gpgvnoexpkeysig
 
 mirrordir="./shared/cache/debian"
 
@@ -43,29 +60,15 @@ rm -f shared/cover_db.img
 
 if [ "$HAVE_QEMU" = "yes" ]; then
   # prepare image for cover_db
-	guestfish -N shared/cover_db.img=disk:64M -- mkfs vfat /dev/sda
+  fallocate -l 64M shared/cover_db.img
+  /usr/sbin/mkfs.vfat shared/cover_db.img
 
-	if [ ! -e "./shared/cache/debian-$DEFAULT_DIST.qcow" ]; then
-		echo "./shared/cache/debian-$DEFAULT_DIST.qcow does not exist" >&2
+  if [ ! -e "./shared/cache/debian-$DEFAULT_DIST.ext4" ]; then
+    echo "./shared/cache/debian-$DEFAULT_DIST.ext4 does not exist" >&2
     exit 1
   fi
 fi
 
-# check if all required debootstrap tarballs exist
-notfound=0
-for dist in oldstable stable testing unstable; do
-	for variant in minbase buildd -; do
-		if [ ! -e "shared/cache/debian-$dist-$variant.tar" ]; then
-			echo "shared/cache/debian-$dist-$variant.tar does not exist" >&2
-			notfound=1
-		fi
-	done
-done
-if [ "$notfound" -ne 0 ]; then
-	echo "not all required debootstrap tarballs are present" >&2
-	exit 1
-fi
-
 # choose the timestamp of the unstable Release file, so that we get
 # reproducible results for the same mirror timestamp
 SOURCE_DATE_EPOCH=$(date --date="$(grep-dctrl -s Date -n '' "$mirrordir/dists/$DEFAULT_DIST/Release")" +%s)
@@ -73,23 +76,19 @@ SOURCE_DATE_EPOCH=$(date --date="$(grep-dctrl -s Date -n '' "$mirrordir/dists/$D
 # for traditional sort order that uses native byte values
 export LC_ALL=C.UTF-8
 
-: "${HAVE_UNSHARE:=yes}"
 : "${HAVE_BINFMT:=yes}"
 
-# by default, use the mmdebstrap executable in the current directory together
-# with perl Devel::Cover but allow to overwrite this
-: "${CMD:=perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap}"
 mirror="http://127.0.0.1/debian"
 
-export HAVE_QEMU HAVE_UNSHARE HAVE_BINFMT RUN_MA_SAME_TESTS DEFAULT_DIST SOURCE_DATE_EPOCH CMD mirror
+export HAVE_QEMU HAVE_BINFMT RUN_MA_SAME_TESTS DEFAULT_DIST SOURCE_DATE_EPOCH CMD mirror
 
-./coverage.py
+./coverage.py "$@"
 
 if [ -e shared/cover_db.img ]; then
   # produce report inside the VM to make sure that the versions match or
   # otherwise we might get:
   # Can't read shared/cover_db/runs/1598213854.252.64287/cover.14 with Sereal: Sereal: Error: Bad Sereal header: Not a valid Sereal document. at offset 1 of input at srl_decoder.c line 600 at /usr/lib/x86_64-linux-gnu/perl5/5.30/Devel/Cover/DB/IO/Sereal.pm line 34, <$fh> chunk 1.
-	cat << END > shared/test.sh
+  cat <<END >shared/test.sh
 cover -nogcov -report html_basic cover_db >&2
 mkdir -p report
 for f in common.js coverage.html cover.css css.js mmdebstrap--branch.html mmdebstrap--condition.html mmdebstrap.html mmdebstrap--subroutine.html standardista-table-sorting.js; do
@@ -99,8 +98,6 @@ cover -delete cover_db >&2
 END
   if [ "$HAVE_QEMU" = "yes" ]; then
     ./run_qemu.sh
-	elif [ "$HAVE_UNSHARE" != "yes" ]; then
-		./run_null.sh SUDO
   else
     ./run_null.sh
   fi
@@ -110,4 +107,11 @@ END
   echo
 fi
 
-rm shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/tarfilter shared/proxysolver
+# check if the wiki has to be updated with pod2markdown output
+if [ "${DEBEMAIL-}" = "josch@debian.org" ]; then
+  bash -exc "diff -u <(curl --silent https://gitlab.mister-muffin.de/josch/mmdebstrap/wiki/raw/Home | dos2unix; echo) <(pod2markdown < mmdebstrap)" || :
+fi
+
+rm -f shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/tarfilter shared/proxysolver
+
+echo "$0 finished successfully" >&2

coverage.txt

@@ -1,10 +1,20 @@
+Test: debootstrap
+Dists: any
+Variants: minbase buildd -
+Needs-Root: true
+Needs-APT-Config: true
+
 Test: check-against-debootstrap-dist
 Dists: any
 Variants: minbase buildd -
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: as-debootstrap-unshare-wrapper
-Needs-QEMU: true
+Modes: unshare
+Needs-Root: true
+Variants: minbase -
+Needs-APT-Config: true
 
 Test: help
 
@@ -20,6 +30,7 @@ Needs-Root: true
 
 Test: dist-using-codename
 Dists: any
+Needs-APT-Config: true
 
 Test: fail-without-etc-subuid
 Needs-QEMU: true
@@ -29,12 +40,15 @@ Needs-QEMU: true
 
 Test: unshare-as-root-user-inside-chroot
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: root-mode-inside-chroot
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: root-mode-inside-unshare-chroot
-Needs-QEMU: true
+Modes: unshare
+Needs-APT-Config: true
 
 Test: root-without-cap-sys-admin
 Needs-Root: true
@@ -42,19 +56,27 @@ Needs-Root: true
 Test: mount-is-missing
 Needs-QEMU: true
 
-Test: check-for-bit-by-bit-identical-format-output
-Needs-QEMU: true
-Formats: tar squashfs ext2
+Test: mmdebstrap
+Needs-Root: true
+Modes: root
+Formats: tar squashfs ext2 ext4
 Variants: essential apt minbase buildd - standard
 Skip-If:
- variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
- variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
- fmt == "squashfs" and dist == "oldstable" # squashfs-tools-ng is not available
- fmt == "ext2" and dist == "oldstable" # genext2fs does not support SOURCE_DATE_EPOCH
+ variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
+ mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
+ variant == "standard" and dist in ["oldstable", "stable"] and hostarch in ["armel", "armhf", "mipsel"] # #1031276
+
+Test: check-for-bit-by-bit-identical-format-output
+Modes: unshare fakechroot
+Formats: tar squashfs ext2 ext4
+Variants: essential apt minbase buildd - standard
+Skip-If:
+ variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
+ mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
+ variant == "standard" and dist in ["oldstable", "stable"] and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
 Test: tarfilter-idshift
 Needs-QEMU: true
-Skip-If: dist == "oldstable" # python3 tarfile module does not preserve xattrs
 
 Test: progress-bars-on-fake-tty
 
@@ -74,19 +96,21 @@ Test: missing-device-nodes-outside-the-chroot
 Needs-QEMU: true
 
 Test: missing-dev-sys-proc-inside-the-chroot
-Needs-QEMU: true
+Modes: unshare
+Variants: custom
 
 Test: chroot-directory-not-accessible-by-apt-user
 Needs-Root: true
 
 Test: cwd-directory-not-accessible-by-unshared-user
-Needs-QEMU: true
+Needs-Root: true
+Modes: unshare
 
 Test: create-gzip-compressed-tarball
-Needs-QEMU: true
 
 Test: custom-tmpdir
-Needs-QEMU: true
+Needs-Root: true
+Modes: unshare
 
 Test: xz-compressed-tarball
 
@@ -109,6 +133,7 @@ Test: read-from-stdin-write-to-stdout
 Test: supply-components-manually
 Modes: root
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: stable-default-mirror
 Needs-QEMU: true
@@ -133,19 +158,23 @@ Needs-QEMU: true
 Test: mirror-is-deb
 
 Test: mirror-is-real-file
+Needs-APT-Config: true
 
 Test: deb822-1-2
 Modes: root
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: deb822-2-2
 Modes: root
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: automatic-mirror-from-suite
 Needs-QEMU: true
 
 Test: invalid-mirror
+Needs-APT-Config: true
 
 Test: fail-installing-to-root
 Modes: root
@@ -165,16 +194,18 @@ Skip-If:
  hostarch != "amd64"
  not run_ma_same_tests
 
-Test: include-libmagic-mgc-arm64
+Test: include-foreign-libmagic-mgc
 Needs-Root: true
+Needs-APT-Config: true
 Skip-If:
- hostarch != "amd64"
+ hostarch not in ["amd64", "arm64"]
  not run_ma_same_tests
 
-Test: include-libmagic-mgc-arm64-with-multiple-arch-options
+Test: include-foreign-libmagic-mgc-with-multiple-arch-options
 Needs-Root: true
+Needs-APT-Config: true
 Skip-If:
- hostarch != "amd64"
+ hostarch not in ["amd64", "arm64"]
  not run_ma_same_tests
 
 Test: aptopt
@@ -185,6 +216,7 @@ Needs-QEMU: true
 
 Test: keyring-overwrites
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: signed-by-without-host-keys
 Needs-QEMU: true
@@ -194,6 +226,7 @@ Needs-QEMU: true
 
 Test: signed-by-with-host-keys
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: dpkgopt
 Needs-Root: true
@@ -227,28 +260,34 @@ Needs-Root: true
 
 Test: special-hooks-using-helpers
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: special-hooks-using-helpers-and-env-vars
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: special-hooks-with-mode-mode
 Modes: root unshare fakechroot
-Needs-QEMU: true
 
 Test: debootstrap-no-op-options
 Needs-Root: true
 
 Test: verbose
-Needs-Root: true
+Variants: standard
+Skip-If:
+ variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
 
 Test: debug
-Needs-Root: true
+Variants: standard
+Skip-If:
+ variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
 
 Test: quiet
 Needs-Root: true
 
 Test: logfile
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: without-etc-resolv-conf-and-etc-hostname
 Needs-QEMU: true
@@ -270,68 +309,74 @@ Test: compare-output-with-pre-seeded-var-cache-apt-archives
 Needs-QEMU: true
 Variants: any
 Skip-If:
- variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
- variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
+ variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
 
 Test: create-directory-dry-run
+Modes: root
 
 Test: create-tarball-dry-run
 Variants: any
 Modes: any
 
 Test: unpack-doc-debian
-Needs-QEMU: true
-Modes: any
+Modes: root fakechroot
 Variants: extract
+Needs-APT-Config: true
 
 Test: install-doc-debian
 Modes: chrootless
 Variants: custom
+Needs-APT-Config: true
 
 Test: chrootless
 Variants: essential
 Modes: chrootless
 Needs-Root: true
 Skip-If:
- dist in ["oldstable", "stable"]
+ dist == "oldstable"
 
 Test: chrootless-fakeroot
 Variants: essential
 Modes: chrootless
-Needs-QEMU: true
 Skip-If:
- dist in ["oldstable", "stable"]
+ dist == "oldstable"
+ hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
 
 Test: chrootless-foreign
 Variants: essential
 Modes: chrootless
 Skip-If:
- dist in ["oldstable", "stable"]
- hostarch != "amd64"
+ dist == "oldstable"
+ hostarch not in ["amd64", "arm64"]
  not run_ma_same_tests
 Needs-QEMU: true
 
 Test: install-doc-debian-and-output-tarball
 Variants: custom
 Modes: chrootless
+Needs-APT-Config: true
 
 Test: install-doc-debian-and-test-hooks
 Variants: custom
 Modes: chrootless
+Needs-APT-Config: true
 
-Test: install-libmagic-mgc-on-arm64
+Test: install-libmagic-mgc-on-foreign
+Variants: custom
+Modes: chrootless
 Skip-If:
- hostarch != "amd64"
+ hostarch not in ["amd64", "arm64"]
  not have_binfmt
 
 Test: install-busybox-based-sub-essential-system
 Needs-Root: true
 
-Test: create-arm64-tarball
+Test: create-foreign-tarball
 Modes: root unshare fakechroot
 Skip-If:
- hostarch != "amd64"
+ hostarch not in ["amd64", "arm64"]
  mode == "fakechroot" and not run_ma_same_tests
+ mode == "fakechroot" and hostarch == "arm64" # usrmerge postinst under fakechroot wants to copy /lib/ld-linux-x86-64.so.2 (which does not exist) instead of /lib64/ld-linux-x86-64.so.2
  not have_binfmt
 
 Test: no-sbin-in-path
@@ -339,22 +384,54 @@ Modes: fakechroot
 
 Test: dev-ptmx
 Modes: root unshare
-Needs-QEMU: true
 
 Test: error-if-stdout-is-tty
 
 Test: variant-custom-timeout
 
 Test: include-deb-file
+Modes: root unshare fakechroot
+Needs-APT-Config: true
 
 Test: unshare-include-deb
 Modes: unshare
-Needs-QEMU: true
 
 Test: pivot_root
 Modes: root unshare
-Needs-QEMU: true
+Needs-APT-Config: true
 
 Test: jessie-or-older
-Needs-QEMU: true
+Needs-Root: true
+Modes: root unshare fakechroot
 Variants: essential apt minbase
+Skip-If: mode == "fakechroot" and hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
+
+Test: apt-patterns
+
+Test: apt-patterns-custom
+
+Test: empty-sources.list
+
+Test: merged-fakechroot-inside-unmerged-chroot
+Needs-Root: true
+Needs-APT-Config: true
+Skip-If:
+ hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
+ dist in ["testing", "unstable"]  # #1053671
+
+Test: auto-mode-as-normal-user
+Modes: auto
+
+Test: skip-output-dev
+Modes: root unshare
+
+Test: skip-output-mknod
+Modes: root unshare
+
+Test: skip-tar-in-mknod
+Modes: unshare
+
+Test: zombie-reaping
+Modes: unshare
+
+Test: empty-suite

gpgvnoexpkeysig

@@ -1,6 +1,7 @@
 #!/bin/sh
 #
-# This script is in the public domain
+# No copyright is claimed.  This code is in the public domain; do with
+# it what you wish.
 #
 # Author: Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
 #
@@ -40,7 +41,7 @@ find_gpgv_status_fd() {
 GPGSTATUSFD="$(find_gpgv_status_fd "$@")"
 
 case $GPGSTATUSFD in
-	''|*[!0-9]*)
+  '' | *[!0-9]*)
     echo "invalid --status-fd argument" >&2
     exit 1
     ;;

hooks/copy-host-apt-sources-and-preferences/customize00.pl

@@ -0,0 +1,44 @@
+#!/usr/bin/perl
+#
+# This script makes sure that all packages that are installed both locally as
+# well as inside the chroot have the same version.
+#
+# It is implemented in Perl because there are no associative arrays in POSIX
+# shell.
+
+use strict;
+use warnings;
+
+sub get_pkgs {
+    my $root = shift;
+    my %pkgs = ();
+    open(my $fh, '-|', 'dpkg-query', "--root=$root", '--showformat',
+        '${binary:Package}=${Version}\n', '--show')
+      // die "cannot exec dpkg-query";
+    while (my $line = <$fh>) {
+        my ($pkg, $ver) = split(/=/, $line, 2);
+        $pkgs{$pkg} = $ver;
+    }
+    close $fh;
+    if ($? != 0) { die "failed to run dpkg-query" }
+    return %pkgs;
+}
+
+my %pkgs_local  = get_pkgs('/');
+my %pkgs_chroot = get_pkgs($ARGV[0]);
+
+my @diff = ();
+foreach my $pkg (keys %pkgs_chroot) {
+    next unless exists $pkgs_local{$pkg};
+    if ($pkgs_local{$pkg} ne $pkgs_chroot{$pkg}) {
+        push @diff, $pkg;
+    }
+}
+
+if (scalar @diff > 0) {
+    print STDERR "E: packages from the host and the chroot differ:\n";
+    foreach my $pkg (@diff) {
+	print STDERR "E: $pkg $pkgs_local{$pkg} $pkgs_chroot{$pkg}\n";
+    }
+    exit 1;
+}

hooks/copy-host-apt-sources-and-preferences/setup00.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+#
+# This script makes sure that the apt sources.list and preferences from outside
+# the chroot also exist inside the chroot by *appending* them to any existing
+# files. If you do not want to keep the original content, add another setup
+# hook before this one which cleans up the files you don't want to keep.
+#
+# If instead of copying sources.list verbatim you want to mangle its contents,
+# consider using python-apt for that. An example can be found in the Debian
+# packaging of mmdebstrap in ./debian/tests/sourcesfilter
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
+
+if [ -n "${MMDEBSTRAP_SUITE:-}" ]; then
+	if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 1 ]; then
+		echo "W: using a non-empty suite name $MMDEBSTRAP_SUITE does not make sense with this hook and might select the wrong Essential:yes package set" >&2
+	fi
+fi
+
+rootdir="$1"
+
+SOURCELIST="/etc/apt/sources.list"
+eval "$(apt-config shell SOURCELIST Dir::Etc::SourceList/f)"
+SOURCEPARTS="/etc/apt/sources.d/"
+eval "$(apt-config shell SOURCEPARTS Dir::Etc::SourceParts/d)"
+PREFERENCES="/etc/apt/preferences"
+eval "$(apt-config shell PREFERENCES Dir::Etc::Preferences/f)"
+PREFERENCESPARTS="/etc/apt/preferences.d/"
+eval "$(apt-config shell PREFERENCESPARTS Dir::Etc::PreferencesParts/d)"
+
+for f in "$SOURCELIST" \
+	"$SOURCEPARTS"/*.list \
+	"$SOURCEPARTS"/*.sources \
+	"$PREFERENCES" \
+	"$PREFERENCESPARTS"/*; do
+	[ -e "$f" ] || continue
+	mkdir --parents "$(dirname "$rootdir/$f")"
+	if [ -e "$rootdir/$f" ]; then
+		if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 2 ]; then
+			echo "I: $f already exists in chroot, appending..." >&2
+		fi
+		# Add extra newline between old content and new content.
+		# This is required in case of deb822 files.
+		echo >> "$rootdir/$f"
+	fi
+	cat "$f" >> "$rootdir/$f"
+	if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+		echo "D: contents of $f inside the chroot:" >&2
+		cat "$rootdir/$f" >&2
+	fi
+done

hooks/eatmydata/customize.sh

@@ -17,10 +17,10 @@ libdir="/usr/lib/$(dpkg-architecture -a "$chrootarch" -q DEB_HOST_MULTIARCH)"
 
 # if eatmydata was actually installed properly, then we are not removing
 # anything here
-if ! chroot "$rootdir" dpkg-query --list eatmydata; then
+if ! chroot "$rootdir" dpkg-query --show eatmydata; then
 	rm "$rootdir/usr/bin/eatmydata"
 fi
-if ! chroot "$rootdir" dpkg-query --list libeatmydata1; then
+if ! chroot "$rootdir" dpkg-query --show libeatmydata1; then
 	rm "$rootdir$libdir"/libeatmydata.so*
 fi
 

hooks/file-mirror-automount/setup00.sh

@@ -14,6 +14,10 @@ env APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get indextargets --no-release-info -
 	| sort -u \
 	| while read -r path; do
 		mkdir -p "$rootdir/run/mmdebstrap"
+		if [ ! -d "/$path" ]; then
+			echo "W: /$path is not an existing directory" >&2
+			continue
+		fi
 		case $MMDEBSTRAP_MODE in
 			root|unshare)
 				echo "bind-mounting /$path into the chroot" >&2
@@ -22,8 +26,8 @@ env APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get indextargets --no-release-info -
 				;;
 			*)
 				echo "copying /$path into the chroot" >&2
-				mkdir -p "$rootdir/$(dirname "$path")"
-				cp -av "/$path" "$rootdir/$(dirname "$path")"
+				mkdir -p "$rootdir/$path"
+				"$MMDEBSTRAP_ARGV0" --hook-helper "$rootdir" "$MMDEBSTRAP_MODE" "$MMDEBSTRAP_HOOK" env "$MMDEBSTRAP_VERBOSITY" sync-in "/$path" "/$path" <&"$MMDEBSTRAP_HOOKSOCK" >&"$MMDEBSTRAP_HOOKSOCK"
 				;;
 		esac
 		printf '/%s\0' "$path" >> "$rootdir/run/mmdebstrap/file-mirror-automount"
@@ -47,6 +51,10 @@ for pkg in $MMDEBSTRAP_INCLUDE; do
 	fi
 	# make path absolute
 	pkg="$(realpath "$pkg")"
+	case "$pkg" in
+		/*) : ;;
+		*) echo "path for $pkg is not absolute" >&2; continue;;
+	esac
 	mkdir -p "$rootdir/run/mmdebstrap"
 	mkdir -p "$rootdir/$(dirname "$pkg")"
 	case $MMDEBSTRAP_MODE in
@@ -57,7 +65,7 @@ for pkg in $MMDEBSTRAP_INCLUDE; do
 			;;
 		*)
 			echo "copying $pkg into the chroot" >&2
-			cp -av "$pkg" "$rootdir/$pkg"
+			"$MMDEBSTRAP_ARGV0" --hook-helper "$rootdir" "$MMDEBSTRAP_MODE" "$MMDEBSTRAP_HOOK" env "$MMDEBSTRAP_VERBOSITY" upload "$pkg" "$pkg" <&"$MMDEBSTRAP_HOOKSOCK" >&"$MMDEBSTRAP_HOOKSOCK"
 			;;
 	esac
 	printf '/%s\0' "$pkg" >> "$rootdir/run/mmdebstrap/file-mirror-automount"

hooks/jessie-or-older/extract00.sh

@@ -8,41 +8,9 @@ fi
 
 TARGET="$1"
 
+# not needed since dpkg 1.17.11
 for f in available diversions cmethopt; do
 	if [ ! -e "$TARGET/var/lib/dpkg/$f" ]; then
 		touch "$TARGET/var/lib/dpkg/$f"
 	fi
 done
-
-if [ -z "${MMDEBSTRAP_ESSENTIAL+x}" ]; then
-	MMDEBSTRAP_ESSENTIAL=
-	for f in "$TARGET/var/cache/apt/archives/"*.deb; do
-		[ -f "$f" ] || continue
-		f="${f#"$TARGET"}"
-		MMDEBSTRAP_ESSENTIAL="$MMDEBSTRAP_ESSENTIAL $f"
-	done
-fi
-
-fname_base_passwd=
-fname_base_files=
-fname_dpkg=
-for pkg in $MMDEBSTRAP_ESSENTIAL; do
-	pkgname=$(dpkg-deb --show --showformat='${Package}' "$TARGET/$pkg")
-	# shellcheck disable=SC2034
-	case $pkgname in
-		base-passwd) fname_base_passwd=$pkg;;
-		base-files)  fname_base_files=$pkg;;
-		dpkg)        fname_dpkg=$pkg;;
-	esac
-done
-
-for var in base_passwd base_files dpkg; do
-	eval 'val=$fname_'"$var"
-	[ -z "$val" ] && continue
-	chroot "$TARGET" dpkg --install --force-depends "$val"
-done
-
-# shellcheck disable=SC2086
-chroot "$TARGET" dpkg --unpack --force-depends $MMDEBSTRAP_ESSENTIAL
-
-chroot "$TARGET" dpkg --configure --pending

hooks/jessie-or-older/extract01.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# needed until init 1.33 which pre-depends on systemd-sysv
+# starting with init 1.34, init is not Essential:yes anymore
+#
+# jessie has init 1.22
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
+
+TARGET="$1"
+
+if [ -z "${MMDEBSTRAP_ESSENTIAL+x}" ]; then
+	MMDEBSTRAP_ESSENTIAL=
+	for f in "$TARGET/var/cache/apt/archives/"*.deb; do
+		[ -f "$f" ] || continue
+		f="${f#"$TARGET"}"
+		MMDEBSTRAP_ESSENTIAL="$MMDEBSTRAP_ESSENTIAL $f"
+	done
+fi
+
+fname_base_passwd=
+fname_base_files=
+fname_dpkg=
+for pkg in $MMDEBSTRAP_ESSENTIAL; do
+	pkgname=$(dpkg-deb --show --showformat='${Package}' "$TARGET/$pkg")
+	# shellcheck disable=SC2034
+	case $pkgname in
+		base-passwd) fname_base_passwd=$pkg;;
+		base-files)  fname_base_files=$pkg;;
+		dpkg)        fname_dpkg=$pkg;;
+	esac
+done
+
+for var in base_passwd base_files dpkg; do
+	eval 'val=$fname_'"$var"
+	[ -z "$val" ] && continue
+	chroot "$TARGET" dpkg --install --force-depends "$val"
+done
+
+# shellcheck disable=SC2086
+chroot "$TARGET" dpkg --unpack --force-depends $MMDEBSTRAP_ESSENTIAL
+
+chroot "$TARGET" dpkg --configure --pending

hooks/maybe-jessie-or-older/extract00.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+set -eu
+
+# we need to check the version of dpkg
+# since at this point packages are just extracted but not installed, we cannot use dpkg-query
+# since we want to support chrootless, we cannot run dpkg --version inside the chroot
+# to avoid this hook depending on dpkg-dev being installed, we do not parse the extracted changelog with dpkg-parsechangelog
+# we also want to avoid parsing the changelog because /usr/share/doc might've been added to dpkg --path-exclude
+# instead, we just ask apt about the latest version of dpkg it knows of
+# this should only fail in situations where there are multiple versions of dpkg in different suites
+ver=$(env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions dpkg 2>/dev/null | sed -ne 's/^Version: \(.*\)$/\1/p' || printf '')
+if [ -z "$ver" ]; then
+	echo "no package called dpkg can be installed -- not running jessie-or-older extract00 hook" >&2
+	exit 0
+fi
+
+if dpkg --compare-versions "$ver" ge 1.17.11; then
+	echo "dpkg version $ver is >= 1.17.11 -- not running jessie-or-older extract00 hook" >&2
+	exit 0
+else
+	echo "dpkg version $ver is << 1.17.11 -- running jessie-or-older extract00 hook" >&2
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/jessie-or-older/extract00.sh" ] && [ -x "$p/jessie-or-older/extract01.sh" ]; then
+		"$p/jessie-or-older/extract00.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find jessie-or-older hook anywhere" >&2
+exit 1

hooks/maybe-jessie-or-older/extract01.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+set -eu
+
+# The jessie-or-older extract01 hook has to be run up to the point where the
+# Essential:yes field was removed from the init package (with
+# init-system-helpers 1.34). Since the essential packages have only been
+# extracted but not installed, we cannot use dpkg-query to find out its
+# version. Since /usr/share/doc might be missing due to dpkg --path-exclude, we
+# also cannot check whether /usr/share/doc/init/copyright exists. There also
+# was a time (before init-system-helpers 1.20) where there was no init package
+# at all where we also want to apply this hook. So we just ask apt about the
+# candidate version for init-system-helpers. This should only fail in
+# situations where there are multiple versions of init-system-helpers in
+# different suites.
+ver=$(env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions init-system-helpers 2>/dev/null | sed -ne 's/^Version: \(.*\)$/\1/p' || printf '')
+if [ -z "$ver" ]; then
+	# there is no package called init-system-helpers, so either:
+	#  - this is so old that init-system-helpers didn't exist yet
+	#  - we are in a future where init-system-helpers doesn't exist anymore
+	#  - something strange is going on
+	# we should only call the hook in the first case
+	ver=$(env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions base-files 2>/dev/null | sed -ne 's/^Version: \(.*\)$/\1/p' || printf '')
+	if [ -z "$ver" ]; then
+		echo "neither init-system-helpers nor base-files can be installed -- not running jessie-or-older extract01 hook" >&2
+		exit 0
+	fi
+
+	# Jessie is Debian 8
+	if dpkg --compare-versions "$ver" ge 8; then
+		echo "there is no init-system-helpers but base-files version $ver is >= 8 -- not running jessie-or-older extract01 hook" >&2
+		exit 0
+	else
+		echo "there is no init-system-helpers but base-files version $ver is << 8 -- running jessie-or-older extract01 hook" >&2
+	fi
+else
+	if dpkg --compare-versions "$ver" ge 1.34; then
+		echo "init-system-helpers version $ver is >= 1.34 -- not running jessie-or-older extract01 hook" >&2
+		exit 0
+	else
+		echo "init-system-helpers version $ver is << 1.34 -- running jessie-or-older extract01 hook" >&2
+	fi
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/jessie-or-older/extract00.sh" ] && [ -x "$p/jessie-or-older/extract01.sh" ]; then
+		"$p/jessie-or-older/extract01.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find jessie-or-older hook anywhere" >&2
+exit 1

hooks/maybe-merged-usr/essential00.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+set -eu
+
+ver=$(dpkg-query --root="$1" -f '${db:Status-Status} ${Source} ${Version}' --show usr-is-merged 2>/dev/null || printf '')
+case "$ver" in
+	'')
+		echo "no package called usr-is-merged is installed -- not running merged-usr essential hook" >&2
+		exit 0
+		;;
+	'installed mmdebstrap-dummy-usr-is-merged 1')
+		echo "dummy usr-is-merged package installed -- running merged-usr essential hook" >&2
+		;;
+	'installed usrmerge '*)
+		echo "usr-is-merged package from src:usrmerge installed -- not running merged-usr essential hook" >&2
+		exit 0
+		;;
+	'not-installed  ')
+		echo "usr-is-merged was not installed in a previous hook -- not running merged-usr essential hook" >&2
+		exit 0
+		;;
+	*)
+		echo "unexpected situation for package usr-is-merged: $ver" >&2
+		exit 1
+		;;
+esac
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/extract00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
+		"$p/merged-usr/essential00.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find merged-usr hook anywhere" >&2
+exit 1

hooks/maybe-merged-usr/extract00.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+set -eu
+
+env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get update --error-on=any
+
+if env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
+	# if apt-cache exited successfully, then usr-is-merged exists either as
+	# a real or virtual package
+	if env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged 2>/dev/null | grep -q "Package: usr-is-merged"; then
+		echo "usr-is-merged found -- running merged-usr extract hook" >&2
+	else
+		# The usr-is-merged must be virtual, so assume that nothing
+		# has to be done. This is the case with Debian Trixie or later
+		# or with Ubuntu Lunar or later
+		echo "usr-is-merged found but not real -- not running merged-usr extract hook" >&2
+		exit 0
+	fi
+else
+	# if the usr-is-merged package cannot be installed with apt, do nothing
+	echo "no package providing usr-is-merged found -- not running merged-usr extract hook" >&2
+	exit 0
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/extract00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
+		"$p/merged-usr/extract00.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find merged-usr hook anywhere" >&2
+exit 1

hooks/maybe-merged-usr/setup00.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+set -eu
+
+env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get update --error-on=any
+
+if env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
+	# if apt-cache exited successfully, then usr-is-merged exists either as
+	# a real or virtual package
+	if env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged 2>/dev/null | grep -q "Package: usr-is-merged"; then
+		echo "usr-is-merged found -- running merged-usr setup hook" >&2
+	else
+		# The usr-is-merged must be virtual, so assume that nothing
+		# has to be done. This is the case with Debian Trixie or later
+		# or with Ubuntu Lunar or later
+		echo "usr-is-merged found but not real -- not running merged-usr setup hook" >&2
+		exit 0
+	fi
+else
+	# if the usr-is-merged package cannot be installed with apt, do nothing
+	echo "no package providing usr-is-merged found -- not running merged-usr setup hook" >&2
+	exit 0
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/extract00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
+		"$p/merged-usr/setup00.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find merged-usr hook anywhere" >&2
+exit 1

hooks/merged-usr/essential00.sh

@@ -10,6 +10,7 @@ TARGET="$1"
 
 if [ "${MMDEBSTRAP_MODE:-}" = "chrootless" ]; then
 	APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install \
+		-oDPkg::Chroot-Directory= \
 		-oDPkg::Options::=--force-not-root \
 		-oDPkg::Options::=--force-script-chrootless \
 		-oDPkg::Options::=--root="$TARGET" \
@@ -20,7 +21,7 @@ if [ "${MMDEBSTRAP_MODE:-}" = "chrootless" ]; then
 	dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
 	dpkg --compare-versions "1" "lt" "$(dpkg-query --showformat '${Version}\n' --show usr-is-merged)"
 else
-	APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$TARGET" usr-is-merged
+	APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install usr-is-merged
 	chroot "$TARGET" dpkg-query --showformat '${db:Status-Status}\n' --show usr-is-merged | grep -q '^installed$'
 	chroot "$TARGET" dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
 	dpkg --compare-versions "1" "lt" "$(chroot "$TARGET" dpkg-query --showformat '${Version}\n' --show usr-is-merged)"

hooks/merged-usr/extract00.sh

@@ -0,0 +1,85 @@
+#!/bin/sh
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
+
+TARGET="$1"
+
+# can_usrmerge_symlink() and can_usrmerge_symlink() are
+# Copyright 2023 Helmut Grohne <helmut@subdivi.de>
+# and part of the debootstrap source in /usr/share/debootstrap/functions
+# https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/96
+# https://bugs.debian.org/104989
+can_usrmerge_symlink() {
+	# Absolute symlinks can be relocated without problems.
+	test "${2#/}" = "$2" || return 0
+	while :; do
+		if test "${2#/}" != "$2"; then
+			# Handle double-slashes.
+			set -- "$1" "${2#/}"
+		elif test "${2#./}" != "$2"; then
+			# Handle ./ inside a link target.
+			set -- "$1" "${2#./}"
+		elif test "$2" = ..; then
+			# A parent directory symlink is ok if it does not
+			# cross the top level directory.
+			test "${1%/*/*}" != "$1" -a -n "${1%/*/*}"
+			return $?
+		elif test "${2#../}" != "$2"; then
+			# Symbolic link crossing / cannot be moved safely.
+			# This is prohibited by Debian Policy 10.5.
+			test "${1%/*/*}" = "$1" -o -z "${1%/*/*}" && return 1
+			set -- "${1%/*}" "${2#../}"
+		else
+			# Consider the symlink ok if its target does not
+			# contain a parent directory. When we fail here,
+			# the link target is non-minimal and doesn't happen
+			# in the archive.
+			test "${2#*/../}" = "$2"
+			return $?
+		fi
+	done
+}
+
+merge_usr_entry() {
+	# shellcheck disable=SC3043
+	local entry canon
+	canon="$TARGET/usr/${1#"$TARGET/"}"
+	test -h "$canon" &&
+		error 1 USRMERGEFAIL "cannot move %s as its destination exists as a symlink" "${1#"$TARGET"}"
+	if ! test -e "$canon"; then
+		mv "$1" "$canon"
+		return 0
+	fi
+	test -d "$1" ||
+		error 1 USRMERGEFAIL "cannot move non-directory %s as its destination exists" "${1#"$TARGET"}"
+	test -d "$canon" ||
+		error 1 USRMERGEFAIL "cannot move directory %s as its destination is not a directory" "${1#"$TARGET"}"
+	for entry in "$1/"* "$1/."*; do
+		# Some shells return . and .. on dot globs.
+		test "${entry%/.}" != "${entry%/..}" && continue
+		if test -h "$entry" && ! can_usrmerge_symlink "${entry#"$TARGET"}" "$(readlink "$entry")"; then
+			error 1 USRMERGEFAIL "cannot move relative symlink crossing top-level directory" "${entry#"$TARGET"}"
+		fi
+		# Ignore glob match failures
+		if test "${entry%'/*'}" != "${entry%'/.*'}" && ! test -e "$entry"; then
+			continue
+		fi
+		merge_usr_entry "$entry"
+	done
+	rmdir "$1"
+}
+
+# This is list includes all possible multilib directories. It must be
+# updated when new multilib directories are being added. Hopefully,
+# all new architectures use multiarch instead, so we never get to
+# update this.
+for dir in bin lib lib32 lib64 libo32 libx32 sbin; do
+	test -h "$TARGET/$dir" && continue
+	test -e "$TARGET/$dir" || continue
+	merge_usr_entry "$TARGET/$dir"
+	ln -s "usr/$dir" "$TARGET/$dir"
+done

hooks/merged-usr/setup00.sh

@@ -47,40 +47,6 @@ fi
 
 TARGET="$1"
 
-ARCH=$(dpkg --print-architecture)
-eval "$(APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-config shell ARCH Apt::Architecture)"
-
-if [ -e /usr/share/debootstrap/functions ]; then
-	# shellcheck disable=SC1091
-	. /usr/share/debootstrap/functions
-	doing_variant () { [ "$1" != "buildd" ]; }
-	# shellcheck disable=SC2034
-	MERGED_USR="yes"
-	setup_merged_usr
-else
-	link_dir=""
-	case $ARCH in
-	    hurd-*) exit 0;;
-	    amd64) link_dir="lib32 lib64 libx32" ;;
-	    i386) link_dir="lib64 libx32" ;;
-	    mips|mipsel) link_dir="lib32 lib64" ;;
-	    mips64*|mipsn32*) link_dir="lib32 lib64 libo32" ;;
-	    powerpc) link_dir="lib64" ;;
-	    ppc64) link_dir="lib32 lib64" ;;
-	    ppc64el) link_dir="lib64" ;;
-	    s390x) link_dir="lib32" ;;
-	    sparc) link_dir="lib64" ;;
-	    sparc64) link_dir="lib32 lib64" ;;
-	    x32) link_dir="lib32 lib64 libx32" ;;
-	esac
-	link_dir="bin sbin lib $link_dir"
-
-	for dir in $link_dir; do
-		ln -s usr/"$dir" "$TARGET/$dir"
-		mkdir -p "$TARGET/usr/$dir"
-	done
-fi
-
 # now install an empty "usr-is-merged" package to avoid installing the
 # usrmerge package on this system even after init-system-helpers starts
 # depending on "usrmerge | usr-is-merged".

hooks/no-merged-usr/essential00.sh

@@ -1,15 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
-        set -x
-fi
-
-TARGET="$1"
-
-APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$TARGET" usr-is-merged
-
-chroot "$TARGET" dpkg-query --showformat '${db:Status-Status}\n' --show usr-is-merged | grep -q '^installed$'
-chroot "$TARGET" dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
-dpkg --compare-versions "1" "lt" "$(chroot "$TARGET" dpkg-query --showformat '${Version}\n' --show usr-is-merged)"

hooks/no-merged-usr/essential00.sh

@@ -0,0 +1 @@
+../merged-usr/essential00.sh

hooks/no-merged-usr/setup00.sh

@@ -18,6 +18,7 @@ fi
 TARGET="$1"
 
 echo "Warning: starting with Debian 12 (Bookworm), systems without merged-/usr are not supported anymore" >&2
+echo "Warning: starting with Debian 13 (Trixie), merged-/usr symlinks are shipped by packages in the essential-set making this hook ineffective" >&2
 
 echo "this system will not be supported in the future" > "$TARGET/etc/unsupported-skip-usrmerge-conversion"
 

ldconfig.fakechroot

@@ -106,10 +106,24 @@ def main():
     for d in get_libdirs(chroot, [chroot / "etc" / "ld.so.conf"]):
         make_relative(d)
 
+    rootarg = chroot
+    argv = sys.argv[1:]
+    for arg in sys.argv[1:]:
+        if arg == "-r":
+            rootarg = None
+        elif rootarg is None:
+            argpath = Path(arg)
+            if argpath.is_absolute():
+                rootarg = chroot / argpath.relative_to("/")
+            else:
+                rootarg = Path.cwd() / argpath
+    if rootarg is None:
+        rootarg = chroot
+
     # we add any additional arguments before "-r" such that any other "-r"
     # option will be overwritten by the one we set
     subprocess.check_call(
-        [chroot / "sbin" / "ldconfig"] + sys.argv[1:] + ["-r", chroot]
+        [chroot / "sbin" / "ldconfig"] + sys.argv[1:] + ["-r", rootarg]
     )
 
 

make_mirror.sh

@@ -11,7 +11,7 @@ set -eu
 
 deletecache() {
   dir="$1"
-	echo "running deletecache $dir">&2
+  echo "running deletecache $dir" >&2
   if [ ! -e "$dir" ]; then
     return
   fi
@@ -20,7 +20,10 @@ deletecache() {
     return 1
   fi
   # be very careful with removing the old directory
-	for dist in oldstable stable testing unstable; do
+  # experimental is pulled in with USE_HOST_APT_CONFIG=yes on debci
+  # when testing a package from experimental
+  for dist in oldstable stable testing unstable experimental; do
+    # deleting artifacts from test "debootstrap"
     for variant in minbase buildd -; do
       if [ -e "$dir/debian-$dist-$variant.tar" ]; then
         rm "$dir/debian-$dist-$variant.tar"
@@ -28,12 +31,24 @@ deletecache() {
         echo "does not exist: $dir/debian-$dist-$variant.tar" >&2
       fi
     done
+    # deleting artifacts from test "mmdebstrap"
+    for variant in essential apt minbase buildd - standard; do
+      for format in tar ext2 ext4 squashfs; do
+        if [ -e "$dir/mmdebstrap-$dist-$variant.$format" ]; then
+          # attempt to delete for all dists because DEFAULT_DIST might've been different the last time
+          rm "$dir/mmdebstrap-$dist-$variant.$format"
+        elif [ "$dist" = "$DEFAULT_DIST" ]; then
+          # only warn about non-existance when it's expected to exist
+          echo "does not exist: $dir/mmdebstrap-$dist-$variant.$format" >&2
+        fi
+      done
+    done
     if [ -e "$dir/debian/dists/$dist" ]; then
       rm --one-file-system --recursive "$dir/debian/dists/$dist"
     else
       echo "does not exist: $dir/debian/dists/$dist" >&2
     fi
-		case "$dist" in oldstable|stable)
+    case "$dist" in oldstable | stable)
       if [ -e "$dir/debian/dists/$dist-updates" ]; then
         rm --one-file-system --recursive "$dir/debian/dists/$dist-updates"
       else
@@ -41,15 +56,7 @@ deletecache() {
       fi
       ;;
     esac
-		case "$dist" in
-			oldstable)
-				if [ -e "$dir/debian-security/dists/$dist/updates" ]; then
-					rm --one-file-system --recursive "$dir/debian-security/dists/$dist/updates"
-				else
-					echo "does not exist: $dir/debian-security/dists/$dist/updates" >&2
-				fi
-				;;
-			stable)
+    case "$dist" in oldstable | stable)
       if [ -e "$dir/debian-security/dists/$dist-security" ]; then
         rm --one-file-system --recursive "$dir/debian-security/dists/$dist-security"
       else
@@ -58,14 +65,21 @@ deletecache() {
       ;;
     esac
   done
-	for f in "$dir/debian-"*.qcow; do
+  for f in "$dir/debian-"*.ext4; do
+    if [ -e "$f" ]; then
       rm --one-file-system "$f"
-	done
-	if [ -e "$dir/debian/pool/main" ]; then
-		rm --one-file-system --recursive "$dir/debian/pool/main"
-	else
-		echo "does not exist: $dir/debian/pool/main" >&2
     fi
+  done
+  # on i386 and amd64, the intel-microcode and amd64-microcode packages
+  # from non-free-firwame get pulled in because they are
+  # priority:standard with USE_HOST_APT_CONFIG=yes
+  for c in main non-free-firmware; do
+    if [ -e "$dir/debian/pool/$c" ]; then
+      rm --one-file-system --recursive "$dir/debian/pool/$c"
+    else
+      echo "does not exist: $dir/debian/pool/$c" >&2
+    fi
+  done
   if [ -e "$dir/debian-security/pool/updates/main" ]; then
     rm --one-file-system --recursive "$dir/debian-security/pool/updates/main"
   else
@@ -94,72 +108,6 @@ cleanup_newcachedir() {
   deletecache "$newcachedir"
 }
 
-get_oldaptnames() {
-	if [ ! -e "$1/$2" ]; then
-		return
-	fi
-	xz -dc "$1/$2" \
-		| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename '' \
-		| paste -sd "    \n" \
-		| while read -r name ver arch fname; do
-			if [ ! -e "$1/$fname" ]; then
-				continue
-			fi
-			# apt stores deb files with the colon encoded as %3a while
-			# mirrors do not contain the epoch at all #645895
-			case "$ver" in *:*) ver="${ver%%:*}%3a${ver#*:}";; esac
-			aptname="$rootdir/var/cache/apt/archives/${name}_${ver}_${arch}.deb"
-			# we have to cp and not mv because other
-			# distributions might still need this file
-			# we have to cp and not symlink because apt
-			# doesn't recognize symlinks
-			cp --link "$1/$fname" "$aptname"
-			echo "$aptname"
-		done
-}
-
-get_newaptnames() {
-	if [ ! -e "$1/$2" ]; then
-		return
-	fi
-	# skip empty files by trying to uncompress the first byte of the payload
-	if [ "$(xz -dc "$1/$2" | head -c1 | wc -c)" -eq 0 ]; then
-		return
-	fi
-	xz -dc "$1/$2" \
-		| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename,SHA256 '' \
-		| paste -sd "     \n" \
-		| while read -r name ver arch fname hash; do
-			# sanity check for the hash because sometimes the
-			# archive switches the hash algorithm
-			if [ "${#hash}" -ne 64 ]; then
-				echo "expected hash length of 64 but got ${#hash} for: $hash" >&2
-				exit 1
-			fi
-			dir="${fname%/*}"
-			# apt stores deb files with the colon encoded as %3a while
-			# mirrors do not contain the epoch at all #645895
-			case "$ver" in *:*) ver="${ver%%:*}%3a${ver#*:}";; esac
-			aptname="$rootdir/var/cache/apt/archives/${name}_${ver}_${arch}.deb"
-			if [ -e "$aptname" ]; then
-				# make sure that we found the right file by checking its hash
-				echo "$hash  $aptname" | sha256sum --check >&2
-				mkdir -p "$1/$dir"
-				# since we move hardlinks around, the same hardlink might've been
-				# moved already into the same place by another distribution.
-				# mv(1) refuses to copy A to B if both are hardlinks of each other.
-				if [ -e "$aptname" ] && [ -e "$1/$fname" ] && [ "$(stat -c "%d %i" "$aptname")" = "$(stat -c "%d %i" "$1/$fname")" ]; then
-					# both files are already the same so we just need to
-					# delete the source
-					rm "$aptname"
-				else
-					mv "$aptname" "$1/$fname"
-				fi
-				echo "$aptname"
-			fi
-		done
-}
-
 cleanupapt() {
   echo "running cleanupapt" >&2
   if [ ! -e "$rootdir" ]; then
@@ -173,10 +121,11 @@ cleanupapt() {
     "$rootdir/var/lib/dpkg/status" \
     "$rootdir/var/lib/dpkg/lock-frontend" \
     "$rootdir/var/lib/dpkg/lock" \
+    "$rootdir/var/lib/apt/lists/lock" \
     "$rootdir/etc/apt/apt.conf" \
+    "$rootdir/etc/apt/sources.list.d/"* \
+    "$rootdir/etc/apt/preferences.d/"* \
     "$rootdir/etc/apt/sources.list" \
-		"$rootdir/oldaptnames" \
-		"$rootdir/newaptnames" \
     "$rootdir/var/cache/apt/archives/lock"; do
     if [ ! -e "$f" ]; then
       echo "does not exist: $f" >&2
@@ -205,16 +154,16 @@ update_cache() (
   # we only set this trap here and overwrite the previous trap, because
   # the update_cache function is run as part of a pipe and thus in its
   # own process which will EXIT after it finished
-	trap "cleanupapt" EXIT INT TERM
+  trap 'kill "$PROXYPID" || :;cleanupapt' EXIT INT TERM
 
   for p in /etc/apt/apt.conf.d /etc/apt/sources.list.d /etc/apt/preferences.d /var/cache/apt/archives /var/lib/apt/lists/partial /var/lib/dpkg; do
     mkdir -p "$rootdir/$p"
   done
 
   # read sources.list content from stdin
-	cat > "$rootdir/etc/apt/sources.list"
+  cat >"$rootdir/etc/apt/sources.list"
 
-	cat << END > "$rootdir/etc/apt/apt.conf"
+  cat <<END >"$rootdir/etc/apt/apt.conf"
 Apt::Architecture "$nativearch";
 Apt::Architectures "$nativearch";
 Dir::Etc "$rootdir/etc/apt";
@@ -225,34 +174,49 @@ Apt::Get::Download-Only true;
 Acquire::Languages "none";
 Dir::Etc::Trusted "/etc/apt/trusted.gpg";
 Dir::Etc::TrustedParts "/etc/apt/trusted.gpg.d";
-Acquire::http::Dl-Limit "1000";
-Acquire::https::Dl-Limit "1000";
-Acquire::Retries "5";
+Acquire::http::Proxy "http://127.0.0.1:8080/";
 END
 
-	: > "$rootdir/var/lib/dpkg/status"
+  : >"$rootdir/var/lib/dpkg/status"
 
-	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update
+  if [ "$dist" = "$DEFAULT_DIST" ] && [ "$nativearch" = "$HOSTARCH" ] && [ "$USE_HOST_APT_CONFIG" = "yes" ]; then
+    # we append sources and settings instead of overwriting after
+    # an empty line
+    for f in /etc/apt/sources.list /etc/apt/sources.list.d/*; do
+      [ -e "$f" ] || continue
+      [ -e "$rootdir/$f" ] && echo >>"$rootdir/$f"
+      # Filter out file:// repositories as they are added
+      # to each mmdebstrap call verbatim by
+      # debian/tests/copy_host_apt_config
+      # Also filter out all mirrors that are not of suite
+      # $DEFAULT_DIST, except experimental if the suite
+      # is unstable. This prevents packages from
+      # unstable entering a testing mirror.
+      if [ "$dist" = unstable ]; then
+        grep -v ' file://' "$f" \
+          | grep -E " (unstable|experimental) " \
+            >>"$rootdir/$f" || :
+      else
+        grep -v ' file://' "$f" \
+          | grep " $DEFAULT_DIST " \
+            >>"$rootdir/$f" || :
+      fi
+    done
+    for f in /etc/apt/preferences.d/*; do
+      [ -e "$f" ] || continue
+      [ -e "$rootdir/$f" ] && echo >>"$rootdir/$f"
+      cat "$f" >>"$rootdir/$f"
+    done
+  fi
 
-	# before downloading packages and before replacing the old Packages
-	# file, copy all old *.deb packages from the mirror to
-	# /var/cache/apt/archives so that apt will not re-download *.deb
-	# packages that we already have
-	{
-		get_oldaptnames "$oldmirrordir" "dists/$dist/main/binary-$nativearch/Packages.xz"
-		case "$dist" in oldstable|stable)
-			get_oldaptnames "$oldmirrordir" "dists/$dist-updates/main/binary-$nativearch/Packages.xz"
-			;;
-		esac
-		case "$dist" in
-			oldstable)
-				get_oldaptnames "$oldcachedir/debian-security" "dists/$dist/updates/main/binary-$nativearch/Packages.xz"
-				;;
-			stable)
-				get_oldaptnames "$oldcachedir/debian-security" "dists/$dist-security/main/binary-$nativearch/Packages.xz"
-				;;
-		esac
-	} | sort -u > "$rootdir/oldaptnames"
+  echo "creating mirror for $dist" >&2
+  for f in /etc/apt/sources.list /etc/apt/sources.list.d/* /etc/apt/preferences.d/*; do
+    [ -e "$rootdir/$f" ] || continue
+    echo "contents of $f:" >&2
+    cat "$rootdir/$f" >&2
+  done
+
+  APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update --error-on=any
 
   pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \
     --format '$(FILENAME)' 'Created-By: Packages' "Architecture: $nativearch" \
@@ -262,85 +226,24 @@ END
       --or --field=Priority important --or --field=Priority standard \
       \))
 
-	pkgs="$pkgs build-essential busybox gpg eatmydata"
+  pkgs="$pkgs build-essential busybox gpg eatmydata fakechroot fakeroot"
 
   # we need usr-is-merged to simulate debootstrap behaviour for all dists
   # starting from Debian 12 (Bullseye)
   case "$dist" in
-		oldstable|stable) : ;;
+    oldstable) : ;;
     *) pkgs="$pkgs usr-is-merged usrmerge" ;;
   esac
 
   # shellcheck disable=SC2086
-	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs
-
-	# to be able to also test gpg verification, we need to create a mirror
-	mkdir -p "$newmirrordir/dists/$dist/main/binary-$nativearch/"
-	curl --location "$mirror/dists/$dist/Release" > "$newmirrordir/dists/$dist/Release"
-	curl --location "$mirror/dists/$dist/Release.gpg" > "$newmirrordir/dists/$dist/Release.gpg"
-	curl --location "$mirror/dists/$dist/main/binary-$nativearch/Packages.xz" > "$newmirrordir/dists/$dist/main/binary-$nativearch/Packages.xz"
-	codename=$(awk '/^Codename: / { print $2; }' < "$newmirrordir/dists/$dist/Release")
-	[ -L "$newmirrordir/dists/$codename" ] || ln -s "$dist" "$newmirrordir/dists/$codename"
-	case "$dist" in oldstable|stable)
-		mkdir -p "$newmirrordir/dists/$dist-updates/main/binary-$nativearch/"
-		curl --location "$mirror/dists/$dist-updates/Release" > "$newmirrordir/dists/$dist-updates/Release"
-		curl --location "$mirror/dists/$dist-updates/Release.gpg" > "$newmirrordir/dists/$dist-updates/Release.gpg"
-		curl --location "$mirror/dists/$dist-updates/main/binary-$nativearch/Packages.xz" > "$newmirrordir/dists/$dist-updates/main/binary-$nativearch/Packages.xz"
-		[ -L "$newmirrordir/dists/$codename-updates" ] || ln -s "$dist-updates" "$newmirrordir/dists/$codename-updates"
-		;;
-	esac
-	case "$dist" in
-		oldstable)
-			mkdir -p "$newcachedir/debian-security/dists/$dist/updates/main/binary-$nativearch/"
-			curl --location "$security_mirror/dists/$dist/updates/Release" > "$newcachedir/debian-security/dists/$dist/updates/Release"
-			curl --location "$security_mirror/dists/$dist/updates/Release.gpg" > "$newcachedir/debian-security/dists/$dist/updates/Release.gpg"
-			curl --location "$security_mirror/dists/$dist/updates/main/binary-$nativearch/Packages.xz" > "$newcachedir/debian-security/dists/$dist/updates/main/binary-$nativearch/Packages.xz"
-			;;
-		stable)
-			mkdir -p "$newcachedir/debian-security/dists/$dist-security/main/binary-$nativearch/"
-			curl --location "$security_mirror/dists/$dist-security/Release" > "$newcachedir/debian-security/dists/$dist-security/Release"
-			curl --location "$security_mirror/dists/$dist-security/Release.gpg" > "$newcachedir/debian-security/dists/$dist-security/Release.gpg"
-			curl --location "$security_mirror/dists/$dist-security/main/binary-$nativearch/Packages.xz" > "$newcachedir/debian-security/dists/$dist-security/main/binary-$nativearch/Packages.xz"
-			[ -L "$newcachedir/debian-security/dists/$codename-security" ] || ln -s "$dist-security" "$newcachedir/debian-security/dists/$codename-security"
-			;;
-	esac
-
-	# the deb files downloaded by apt must be moved to their right locations in the
-	# pool directory
-	#
-	# Instead of parsing the Packages file, we could also attempt to move the deb
-	# files ourselves to the appropriate pool directories. But that approach
-	# requires re-creating the heuristic by which the directory is chosen, requires
-	# stripping the epoch from the filename and will break once mirrors change.
-	# This way, it doesn't matter where the mirror ends up storing the package.
-	{
-		get_newaptnames "$newmirrordir" "dists/$dist/main/binary-$nativearch/Packages.xz";
-		case "$dist" in oldstable|stable)
-			get_newaptnames "$newmirrordir" "dists/$dist-updates/main/binary-$nativearch/Packages.xz"
-			;;
-		esac
-		case "$dist" in
-			oldstable)
-				get_newaptnames "$newcachedir/debian-security" "dists/$dist/updates/main/binary-$nativearch/Packages.xz"
-				;;
-			stable)
-				get_newaptnames "$newcachedir/debian-security" "dists/$dist-security/main/binary-$nativearch/Packages.xz"
-				;;
-		esac
-	} | sort -u > "$rootdir/newaptnames"
+  APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs \
+    || APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install \
+      -oDebug::pkgProblemResolver=true -oDebug::pkgDepCache::Marker=1 \
+      -oDebug::pkgDepCache::AutoInstall=1 \
+      $pkgs
 
   rm "$rootdir/var/cache/apt/archives/lock"
   rmdir "$rootdir/var/cache/apt/archives/partial"
-	# remove all packages that were in the old Packages file but not in the
-	# new one anymore
-	comm -23 "$rootdir/oldaptnames" "$rootdir/newaptnames" | xargs --delimiter="\n" --no-run-if-empty rm
-	# now the apt cache should be empty
-	if [ -n "$(ls -1qA "$rootdir/var/cache/apt/archives/")" ]; then
-		echo "$rootdir/var/cache/apt/archives not empty:"
-		ls -la "$rootdir/var/cache/apt/archives/"
-		exit 1
-	fi
-
   APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --option Dir::Etc::SourceList=/dev/null update
   APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get clean
 
@@ -351,6 +254,17 @@ END
   trap "-" EXIT INT TERM
 )
 
+check_proxy_running() {
+  if timeout 1 bash -c 'exec 3<>/dev/tcp/127.0.0.1/8080 && printf "GET http://deb.debian.org/debian/dists/'"$DEFAULT_DIST"'/InRelease HTTP/1.1\nHost: deb.debian.org\n\n" >&3 && grep "Suite: '"$DEFAULT_DIST"'" <&3 >/dev/null' 2>/dev/null; then
+    return 0
+  elif timeout 1 env http_proxy="http://127.0.0.1:8080/" wget --quiet -O - "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" | grep "Suite: $DEFAULT_DIST" >/dev/null; then
+    return 0
+  elif timeout 1 curl --proxy "http://127.0.0.1:8080/" --silent "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" | grep "Suite: $DEFAULT_DIST" >/dev/null; then
+    return 0
+  fi
+  return 1
+}
+
 if [ -e "./shared/cache.A" ] && [ -e "./shared/cache.B" ]; then
   echo "both ./shared/cache.A and ./shared/cache.B exist" >&2
   echo "was a former run of the script aborted?" >&2
@@ -358,16 +272,22 @@ if [ -e "./shared/cache.A" ] && [ -e "./shared/cache.B" ]; then
     echo "cache symlink points to $(readlink ./shared/cache)" >&2
     case "$(readlink ./shared/cache)" in
       cache.A)
-				echo "maybe rm -r ./shared/cache.B" >&2
+        echo "removing ./shared/cache.B" >&2
+        rm -r ./shared/cache.B
         ;;
       cache.B)
-				echo "maybe rm -r ./shared/cache.A" >&2
+        echo "removing ./shared/cache.A" >&2
+        rm -r ./shared/cache.A
         ;;
       *)
         echo "unexpected" >&2
-		esac
-	fi
         exit 1
+        ;;
+    esac
+  else
+    echo "./shared/cache doesn't exist" >&2
+    exit 1
+  fi
 fi
 
 if [ -e "./shared/cache.A" ]; then
@@ -389,54 +309,82 @@ security_mirror="http://security.debian.org/debian-security"
 components=main
 
 : "${DEFAULT_DIST:=unstable}"
+: "${ONLY_DEFAULT_DIST:=no}"
+: "${ONLY_HOSTARCH:=no}"
 : "${HAVE_QEMU:=yes}"
 : "${RUN_MA_SAME_TESTS:=yes}"
 # by default, use the mmdebstrap executable in the current directory
 : "${CMD:=./mmdebstrap}"
+: "${USE_HOST_APT_CONFIG:=no}"
+: "${FORCE_UPDATE:=no}"
 
-if [ -e "$oldmirrordir/dists/$DEFAULT_DIST/Release" ]; then
-	http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/Release" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/Release")
+if [ "$FORCE_UPDATE" != "yes" ] && [ -e "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
+  http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/InRelease")
   case "$http_code" in
     200) ;; # need update
-		304) echo up-to-date; exit 0;;
-		*) echo "unexpected status: $http_code"; exit 1;;
+    304)
+      echo up-to-date
+      exit 0
+      ;;
+    *)
+      echo "unexpected status: $http_code"
+      exit 1
+      ;;
   esac
 fi
 
-trap "cleanup_newcachedir" EXIT INT TERM
+./caching_proxy.py "$oldcachedir" "$newcachedir" &
+PROXYPID=$!
+trap 'kill "$PROXYPID" || :' EXIT INT TERM
+
+for i in $(seq 10); do
+  check_proxy_running && break
+  sleep 1
+done
+if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
+  echo "failed to start proxy" >&2
+  kill $PROXYPID
+  exit 1
+fi
+
+trap 'kill "$PROXYPID" || :;cleanup_newcachedir' EXIT INT TERM
 
 mkdir -p "$newcachedir"
 touch "$newcachedir/mmdebstrapcache"
 
 HOSTARCH=$(dpkg --print-architecture)
+arches="$HOSTARCH"
 if [ "$HOSTARCH" = amd64 ]; then
-	arches="amd64 arm64 i386"
-else
-	arches="$HOSTARCH"
+  arches="$arches arm64 i386"
+elif [ "$HOSTARCH" = arm64 ]; then
+  arches="$arches amd64 armhf"
 fi
 
-for nativearch in $arches; do
-	for dist in oldstable stable testing unstable; do
+# we need the split_inline_sig() function
+# shellcheck disable=SC1091
+. /usr/share/debootstrap/functions
+
+for dist in oldstable stable testing unstable; do
+  for nativearch in $arches; do
     # non-host architectures are only downloaded for $DEFAULT_DIST
     if [ "$nativearch" != "$HOSTARCH" ] && [ "$DEFAULT_DIST" != "$dist" ]; then
       continue
     fi
+    # if ONLY_DEFAULT_DIST is set, only download DEFAULT_DIST
+    if [ "$ONLY_DEFAULT_DIST" = "yes" ] && [ "$DEFAULT_DIST" != "$dist" ]; then
+      continue
+    fi
+    if [ "$ONLY_HOSTARCH" = "yes" ] && [ "$nativearch" != "$HOSTARCH" ]; then
+      continue
+    fi
     # we need a first pass without updates and security patches
     # because otherwise, old package versions needed by
     # debootstrap will not get included
     echo "deb [arch=$nativearch] $mirror $dist $components" | update_cache "$dist" "$nativearch"
     # we need to include the base mirror again or otherwise
     # packages like build-essential will be missing
-		case "$dist" in
-			oldstable)
-				cat << END | update_cache "$dist" "$nativearch"
-deb [arch=$nativearch] $mirror $dist $components
-deb [arch=$nativearch] $mirror $dist-updates main
-deb [arch=$nativearch] $security_mirror $dist/updates main
-END
-				;;
-			stable)
-				cat << END | update_cache "$dist" "$nativearch"
+    case "$dist" in oldstable | stable)
+      cat <<END | update_cache "$dist" "$nativearch"
 deb [arch=$nativearch] $mirror $dist $components
 deb [arch=$nativearch] $mirror $dist-updates main
 deb [arch=$nativearch] $security_mirror $dist-security main
@@ -444,8 +392,20 @@ END
       ;;
     esac
   done
+  codename=$(awk '/^Codename: / { print $2; }' <"$newmirrordir/dists/$dist/InRelease")
+  ln -s "$dist" "$newmirrordir/dists/$codename"
+
+  # split the InRelease file into Release and Release.gpg not because apt
+  # or debootstrap need it that way but because grep-dctrl does
+  split_inline_sig \
+    "$newmirrordir/dists/$dist/InRelease" \
+    "$newmirrordir/dists/$dist/Release" \
+    "$newmirrordir/dists/$dist/Release.gpg"
+  touch --reference="$newmirrordir/dists/$dist/InRelease" "$newmirrordir/dists/$dist/Release" "$newmirrordir/dists/$dist/Release.gpg"
 done
 
+kill $PROXYPID
+
 # Create some symlinks so that we can trick apt into accepting multiple apt
 # lines that point to the same repository but look different. This is to
 # avoid the warning:
@@ -463,10 +423,7 @@ cleanuptmpdir() {
   if [ ! -e "$tmpdir" ]; then
     return
   fi
-	for f in "$tmpdir/worker.sh" \
-		"$tmpdir/mini-httpd" "$tmpdir/hosts" \
-		"$tmpdir/debian-chroot.tar" \
-		"$tmpdir/mmdebstrap.service"; do
+  for f in "$tmpdir/worker.sh" "$tmpdir/mmdebstrap.service"; do
     if [ ! -e "$f" ]; then
       echo "does not exist: $f" >&2
       continue
@@ -480,61 +437,47 @@ SOURCE_DATE_EPOCH="$(date --date="$(grep-dctrl -s Date -n '' "$newmirrordir/dist
 export SOURCE_DATE_EPOCH
 
 if [ "$HAVE_QEMU" = "yes" ]; then
-	case "$HOSTARCH" in
-		amd64|i386|arm64)
-			# okay
-			;;
-		*)
-			echo "qemu support is only available on amd64, i386 and arm64" >&2
-			echo "because grub is only available on those arches" >&2
+  # we use the caching proxy again when building the qemu image
+  #  - we can re-use the packages that were already downloaded earlier
+  #  - we make sure that the qemu image uses the same Release file even
+  #    if a mirror push happened between now and earlier
+  #  - we avoid polluting the mirror with the additional packages by
+  #    using --readonly
+  ./caching_proxy.py --readonly "$oldcachedir" "$newcachedir" &
+  PROXYPID=$!
+
+  for i in $(seq 10); do
+    check_proxy_running && break
+    sleep 1
+  done
+  if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
+    echo "failed to start proxy" >&2
+    kill $PROXYPID
     exit 1
-			;;
-	esac
-
-	# We must not use any --dpkgopt here because any dpkg options still
-	# leak into the chroot with chrootless mode.
-	# We do not use our own package cache here because
-	#   - it doesn't (and shouldn't) contain the extra packages
-	#   - it doesn't matter if the base system is from a different mirror timestamp
-	# procps is needed for /sbin/sysctl
-	tmpdir="$(mktemp -d)"
-	trap "cleanuptmpdir; cleanup_newcachedir" EXIT INT TERM
-
-	pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,grub-efi
-	if [ "$DEFAULT_DIST" != "oldstable" ]; then
-		pkgs="$pkgs,squashfs-tools-ng,genext2fs"
   fi
+
+  tmpdir="$(mktemp -d)"
+  trap 'kill "$PROXYPID" || :;cleanuptmpdir; cleanup_newcachedir' EXIT INT TERM
+
+  pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-binfmt,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,squashfs-tools-ng,genext2fs,linux-image-generic,passwd,e2fsprogs,uuid-runtime
   if [ ! -e ./mmdebstrap ]; then
     pkgs="$pkgs,mmdebstrap"
   fi
+  arches=$HOSTARCH
+  if [ "$RUN_MA_SAME_TESTS" = "yes" ]; then
     case "$HOSTARCH" in
-		amd64|arm64)
-			pkgs="$pkgs,linux-image-$HOSTARCH"
-			;;
-		i386)
-			pkgs="$pkgs,linux-image-686"
-			;;
-		ppc64el)
-			pkgs="$pkgs,linux-image-powerpc64le"
-			;;
-		*)
-			echo "no kernel image for $HOSTARCH" >&2
-			exit 1
-			;;
-	esac
-	if [ "$HOSTARCH" = amd64 ] && [ "$RUN_MA_SAME_TESTS" = "yes" ]; then
+      amd64)
         arches=amd64,arm64
         pkgs="$pkgs,libfakechroot:arm64,libfakeroot:arm64"
-	else
-		arches=$HOSTARCH
+        ;;
+      arm64)
+        arches=arm64,amd64
+        pkgs="$pkgs,libfakechroot:amd64,libfakeroot:amd64"
+        ;;
+    esac
   fi
-	$CMD --variant=apt --architectures="$arches" --include="$pkgs" \
-		--aptopt='Acquire::http::Dl-Limit "1000"' \
-		--aptopt='Acquire::https::Dl-Limit "1000"' \
-		--aptopt='Acquire::Retries "5"' \
-		"$DEFAULT_DIST" - "$mirror" > "$tmpdir/debian-chroot.tar"
 
-	cat << END > "$tmpdir/mmdebstrap.service"
+  cat <<END >"$tmpdir/mmdebstrap.service"
 [Unit]
 Description=mmdebstrap worker script
 
@@ -553,13 +496,15 @@ END
   # filesystem that doesn't support ownership information at all and a umask that
   # gives read/write access to everybody.
   # https://github.com/pjcj/Devel--Cover/issues/223
-	cat << 'END' > "$tmpdir/worker.sh"
+  cat <<'END' >"$tmpdir/worker.sh"
 #!/bin/sh
 echo 'root:root' | chpasswd
 mount -t 9p -o trans=virtio,access=any,msize=128k mmdebstrap /mnt
 # need to restart mini-httpd because we mounted different content into www-root
 systemctl restart mini-httpd
 
+ip link set enp0s1 down || :
+
 handler () {
 	while IFS= read -r line || [ -n "$line" ]; do
 		printf "%s %s: %s\n" "$(date -u -d "0 $(date +%s.%3N) seconds - $2 seconds" +"%T.%3N")" "$1" "$line"
@@ -582,126 +527,43 @@ handler () {
 	    } 2>&1;
 	  } | { read xs; exit $xs; };
 	} 3>&1 || ret=$?
+	echo $ret > /mnt/exitstatus.txt
 	if [ -e cover_db.img ]; then
 		df -h cover_db
 		umount cover_db
 	fi
-	echo $ret
-) > /mnt/result.txt 2>&1
+) > /mnt/output.txt 2>&1
 umount /mnt
 systemctl poweroff
 END
   chmod +x "$tmpdir/worker.sh"
-	# initially we serve from the new cache so that debootstrap can grab
-	# the new package repository and not the old
-	cat << END > "$tmpdir/mini-httpd"
-START=1
-DAEMON_OPTS="-h 127.0.0.1 -p 80 -u nobody -dd /mnt/$newcache -i /var/run/mini-httpd.pid -T UTF-8"
-END
-	cat << 'END' > "$tmpdir/hosts"
-127.0.0.1 localhost
-END
-	#libguestfs-test-tool
-	#export LIBGUESTFS_DEBUG=1 LIBGUESTFS_TRACE=1
-	#
-	# In case the rootfs was prepared in fakechroot mode, ldconfig has to
-	# run to populate /etc/ld.so.cache or otherwise fakechroot tests will
-	# fail to run.
-	#
-	# The disk size is sufficient in most cases. Sometimes, gcc will do
-	# an upload with unstripped executables to make tracking down ICEs much
-	# easier (see #872672, #894014). During times with unstripped gcc, the
-	# buildd variant will not be 400MB but 1.3GB large and needs a 10G
-	# disk.
   if [ -z ${DISK_SIZE+x} ]; then
     DISK_SIZE=10G
   fi
-	case "$HOSTARCH" in
-		amd64) GRUB_TARGET=x86_64-efi;;
-		i386) GRUB_TARGET=i386-efi;;
-		arm64) GRUB_TARGET=arm64-efi;;
-	esac
-	case "$HOSTARCH" in
-		arm64) SERIAL="loglevel=3 console=tty0 console=ttyAMA0,115200n8" ;;
-		*) SERIAL="loglevel=3 console=tty0 console=ttyS0,115200n8" ;;
-	esac
-	guestfish -- \
-		disk-create "$newcachedir/debian-$DEFAULT_DIST.qcow" qcow2 "$DISK_SIZE" : \
-		add-drive "$newcachedir/debian-$DEFAULT_DIST.qcow" format:qcow2 : \
-		launch : \
-		part-init /dev/sda gpt : \
-		part-add /dev/sda primary 8192 262144 : \
-		part-add /dev/sda primary 262145 -34 : \
-		part-set-gpt-type /dev/sda 1 C12A7328-F81F-11D2-BA4B-00A0C93EC93B  : \
-		mkfs ext2 /dev/sda2 : \
-		mount /dev/sda2 / : \
-		tar-in "$tmpdir/debian-chroot.tar" / xattrs:true : \
-		mkdir-p /boot/efi : \
-		mkfs vfat /dev/sda1 : \
-		mount /dev/sda1 /boot/efi : \
-		command /sbin/ldconfig : \
-		mkdir-p /etc/systemd/system/multi-user.target.wants : \
-		ln-s ../mmdebstrap.service /etc/systemd/system/multi-user.target.wants/mmdebstrap.service : \
-		copy-in "$tmpdir/mmdebstrap.service" /etc/systemd/system/ : \
-		copy-in "$tmpdir/worker.sh" / : \
-		copy-in "$tmpdir/mini-httpd" /etc/default : \
-		copy-in "$tmpdir/hosts" /etc/ : \
-		touch /mmdebstrap-testenv : \
-		command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda2) / ext4 errors=remount-ro 0 1 > /etc/fstab'" : \
-		command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda1) /boot/efi vfat errors=remount-ro 0 2 >> /etc/fstab'" : \
-		command "sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=/GRUB_CMDLINE_LINUX_DEFAULT=\"biosdevname=0 net.ifnames=0 consoleblank=0 rw $SERIAL\"/' /etc/default/grub" : \
-		command "update-initramfs -u" : \
-		command "grub-mkconfig -o /boot/grub/grub.cfg" : \
-		command "grub-install /dev/sda --target=$GRUB_TARGET --no-nvram --force-extra-removable --no-floppy --modules=part_gpt --grub-mkdevicemap=/boot/grub/device.map" : \
-		sync : \
-		umount /boot/efi : \
-		umount / : \
-		shutdown
+  # set PATH to pick up the correct mmdebstrap variant
+  env PATH="$(dirname "$(realpath --canonicalize-existing "$CMD")"):$PATH" \
+    debvm-create --skip=usrmerge,systemdnetwork \
+    --size="$DISK_SIZE" --release="$DEFAULT_DIST" \
+    --output="$newcachedir/debian-$DEFAULT_DIST.ext4" -- \
+    --architectures="$arches" --include="$pkgs" \
+    --setup-hook='echo "Acquire::http::Proxy \"http://127.0.0.1:8080/\";" > "$1/etc/apt/apt.conf.d/00proxy"' \
+    --hook-dir=/usr/share/mmdebstrap/hooks/maybe-merged-usr \
+    --customize-hook='rm "$1/etc/apt/apt.conf.d/00proxy"' \
+    --customize-hook='mkdir -p "$1/etc/systemd/system/multi-user.target.wants"' \
+    --customize-hook='ln -s ../mmdebstrap.service "$1/etc/systemd/system/multi-user.target.wants/mmdebstrap.service"' \
+    --customize-hook='touch "$1/mmdebstrap-testenv"' \
+    --customize-hook='copy-in "'"$tmpdir"'/mmdebstrap.service" /etc/systemd/system/' \
+    --customize-hook='copy-in "'"$tmpdir"'/worker.sh" /' \
+    --customize-hook='echo 127.0.0.1 localhost > "$1/etc/hosts"' \
+    --customize-hook='printf "START=1\nDAEMON_OPTS=\"-h 127.0.0.1 -p 80 -u nobody -dd /mnt/cache -i /var/run/mini-httpd.pid -T UTF-8\"\n" > "$1/etc/default/mini-httpd"' \
+    --customize-hook='touch "$1/etc/systemd/system/tmp.mount"' \
+    "$mirror"
+
+  kill $PROXYPID
   cleanuptmpdir
   trap "cleanup_newcachedir" EXIT INT TERM
 fi
 
-mirror="http://127.0.0.1/debian"
-for dist in oldstable stable testing unstable; do
-	for variant in minbase buildd -; do
-		echo "running debootstrap --variant=$variant $dist \${TEMPDIR} $mirror"
-		cat << END > shared/test.sh
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-export SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
-echo "SOURCE_DATE_EPOCH=\$SOURCE_DATE_EPOCH"
-tmpdir="\$(mktemp -d)"
-chmod 755 "\$tmpdir"
-case "$dist" in
-	oldstable|stable)
-		debootstrap --no-merged-usr --variant=$variant $dist "\$tmpdir" $mirror
-		;;
-	*)
-		debootstrap --merged-usr --variant=$variant $dist "\$tmpdir" $mirror
-		;;
-esac
-tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system --xattrs -C "\$tmpdir" -c . > "$newcache/debian-$dist-$variant.tar"
-rm -r "\$tmpdir"
-END
-		if [ "$HAVE_QEMU" = "yes" ]; then
-			cachedir=$newcachedir ./run_qemu.sh
-		else
-			./run_null.sh SUDO
-		fi
-	done
-done
-
-if [ "$HAVE_QEMU" = "yes" ]; then
-	# now replace the minihttpd config with one that serves the new repository
-	guestfish -a "$newcachedir/debian-$DEFAULT_DIST.qcow" -i <<EOF
-upload -<<END /etc/default/mini-httpd
-START=1
-DAEMON_OPTS="-h 127.0.0.1 -p 80 -u nobody -dd /mnt/cache -i /var/run/mini-httpd.pid -T UTF-8"
-END
-EOF
-fi
-
 # delete possibly leftover symlink
 if [ -e ./shared/cache.tmp ]; then
   rm ./shared/cache.tmp
@@ -713,3 +575,5 @@ mv --no-target-directory ./shared/cache.tmp ./shared/cache
 deletecache "$oldcachedir"
 
 trap - EXIT INT TERM
+
+echo "$0 finished successfully" >&2

mmdebstrap-autopkgtest-build-qemu

@@ -1,255 +1,460 @@
 #!/bin/sh
+# Copyright 2023 Johannes Schauer Marin Rodrigues <josch@debian.org>
+# Copyright 2023 Helmut Grohne <helmut@subdivi.de>
+# SPDX-License-Identifier: MIT
+
+# We generally use single quotes to avoid variable expansion:
+# shellcheck disable=SC2016
+
+# Replacement for autopkgtest-build-qemu and vmdb2 for all architectures
+# supporting EFI booting (amd64, arm64, armhf, i386, riscv64).
+# For use as replacement for autopkgtest-build-qemu and vmdb2 on ppc64el which
+# neither supports extlinux nor efi booting there is an unmaintained script
+# which uses grub instead to boot:
 #
-# © 2022 Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to
-# deal in the Software without restriction, including without limitation the
-# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-# sell copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# The software is provided "as is", without warranty of any kind, express or
-# implied, including but not limited to the warranties of merchantability,
-# fitness for a particular purpose and noninfringement. In no event shall the
-# authors or copyright holders be liable for any claim, damages or other
-# liability, whether in an action of contract, tort or otherwise, arising
-# from, out of or in connection with the software or the use or other dealings
-# in the software.
+#     https://gitlab.mister-muffin.de/josch/mmdebstrap/src/commit/
+#     e523741610a4ed8579642bfc755956f64c847ef3/mmdebstrap-autopkgtest-build-qemu
+
+: <<'POD2MAN'
+=head1 NAME
+
+mmdebstrap-autopkgtest-build-qemu - autopkgtest-build-qemu without vmdb2 but mmdebstrap and EFI boot
+
+=head1 SYNOPSIS
+
+B<mmdebstrap-autopkgtest-build-qemu> [I<OPTIONS>] B<--boot>=B<efi> I<RELEASE> I<IMAGE>
+
+=head1 DESCRIPTION
+
+B<mmdebstrap-autopkgtest-build-qemu> is a mostly compatible drop-in replacement
+for L<autopkgtest-build-qemu(1)> with two main differences: Firstly, it uses
+L<mmdebstrap(1)> instead of L<vmdb2(1)> and thus is able to create QEMU disk
+images without requiring superuser privileges and with bit-by-bit reproducible
+output. Secondly, it uses L<systemd-boot(7)> and thus only supports booting via
+EFI. For architectures for which L<autopkgtest-virt-qemu(1)> does not default
+to EFI booting you must pass B<--boot=efi> when invoking the autopkgtest virt
+backend.
+
+=head1 POSITIONAL PARAMETERS
+
+=over 8
+
+=item I<RELEASE>
+
+The release to download from the I<MIRROR>.  This parameter is required.
+
+=item I<IMAGE>
+
+The file to write, in raw format. This parameter is required.
+
+=back
+
+=head1 OPTIONS
+
+=over 8
+
+=item B<--mirror>=I<MIRROR>
+
+Specify  which  distribution  to install.  It defaults to
+http://deb.debian.org/debian (i.e. Debian), but you can pass a mirror of any
+Debian derivative.
+
+=item B<--architecture>=I<ARCHITECTURE>
+
+Set the architecture for the virtual machine image, specified as a L<dpkg(1)>
+architecture.  If  omitted, the host architecture is assumed.
+
+B<--arch>=I<ARCH> is an alias for this option.
+
+=item B<--script>=I<SCRIPT>
+
+Specifies a user script that will be called with the root filesystem of the
+image as its first parameter. This script can them make any necesssary
+modifications to the root filesystem.
+
+The script must be a POSIX shell script, and should not depend on bash-specific
+features. This  script will be executed inside a L<chroot(1)> call in the
+virtual machine root filesystem.
+
+=item B<--size>=I<SIZE>
+
+Specifies the image size for the virtual machine, defaulting to 25G.
+
+=item B<--apt-proxy>=I<PROXY>
+
+Specify  an  apt proxy to use in the virtual machine.  By default, if you have
+an apt proxy configured on the host, the virtual machine will automatically use
+this, otherwise there is no default.
+
+=item B<--boot>=B<efi>, B<--efi>
+
+Select the way the generated image will expect to be booted. Unless you
+explicitly select --boot=efi, operation will fail.
+
+=item B<--keyring>=I<KEYRING>
+
+Passes an additional B<--keyring> parameter to B<mmdebstrap>.
+
+=back
+
+=head1 EXAMPLES
+
+Make sure, that F</path/to/debian-unstable.img> is a path that the unshared
+user has access to. This can be done by ensuring world-execute permissions on
+all path components or by creating the image in a world-readable directory like
+/tmp before copying it into its final location.
+
+    $ mmdebstrap-autopkgtest-build-qemu --boot=efi --arch=amd64 unstable /path/to/debian-unstable.img
+    [...]
+    $ autopkgtest mypackage -- qemu --boot=efi --dpkg-architecture=amd64 /path/to/debian-unstable.img
+
+Make sure to add B<--boot=efi> to both the B<mmdebstrap-autopkgtest-build-qemu>
+as well as the B<autopkgtest-virt-qemu> invocation.
+
+Create bit-by-bit reproducible images from a given snapshot.d.o timestamp.
+
+    SOURCE_DATE_EPOCH=1612543740 mmdebstrap-autopkgtest-build-qemu --boot=efi \
+        --mirror=http://snapshot.debian.org/archive/debian/20210205T164900Z/ \
+        unstable /path/to/debian-unstable.img
+
+=head1 SEE ALSO
+
+L<autopkgtest-build-qemu(1)>, L<autopkgtest-virt-qemu(1)>, L<mmdebstrap(1)>, L<autopkgtest(1)>
+
+=cut
+POD2MAN
 
 set -eu
 
-# This script creates debian-$RELEASE.qcow2 in the current directory which can
-# then be used by the autopkgtest qemu backend.
-#
-# Thanks to Francesco Poli for providing ideas and testing this.
-#
-# Thanks to Lars Wirzenius of vmdb2 where the grub and efi magic comes from.
-#
-# Only the native architecture is supported because guestfish doesn't support
-# foreign architectures.
-
+die() {
+  echo "$*" 1>&2
+  exit 1
+}
 usage() {
-	echo "Usage: $0 [--size=SIZE] [--boot=BOOT] RELEASE IMAGE" >&2
-	echo >&2
-	echo "RELEASE is a Debian release like unstable" >&2
-	echo "IMAGE will be stored in qcow2 format" >&2
-	echo "SIZE is 25G by default" >&2
-	echo "BOOT is either auto (the default), bios, efi or ieee1275" >&2
+  die "usage: $0 [--architecture=|--apt-proxy=|--keyring=|--mirror=|--script=|--size=] --boot=efi <RELEASE> <IMAGE>"
+}
+usage_error() {
+  echo "error: $*" 1>&2
+  usage
 }
 
-nativearch="$(dpkg --print-architecture)"
+BOOT=auto
+ARCHITECTURE=$(dpkg --print-architecture)
+IMAGE=
+MIRROR=
+KEYRING=
+RELEASE=
+SIZE=25G
+SCRIPT=
 
-SIZE="25G" # default from autopkgtest-build-qemu
-BOOT="auto"
-if [ "$#" -lt 2 ]; then
-	echo "Error: Insufficient number of arguments" >&2
-	usage
-	exit 1
-elif [ "$#" -eq 2 ]; then
-	RELEASE=$1
-	IMAGE=$2
-else
-	# parse options
-	OPTS=$(getopt -n "$0" -o h --long size:,boot:,architecture:,help -- "$@")
-	if [ "$?" -ne 0 ]; then
-		echo "Error: Cannot parse arguments" >&2
-		usage
-		exit 1
+# consumed by setup-testbed
+export AUTOPKGTEST_BUILD_QEMU=1
+
+opt_boot() {
+  BOOT="$1"
+}
+opt_architecture() {
+  ARCHITECTURE="$1"
+}
+opt_arch() {
+  ARCHITECTURE="$1"
+}
+opt_apt_proxy() {
+  # consumed by setup-testbed
+  export AUTOPKGTEST_APT_PROXY="$1"
+  # consumed by mmdebstrap
+  if test "$1" = DIRECT; then
+    unset http_proxy
+  else
+    export http_proxy="$1"
   fi
-	eval set -- "$OPTS"
-	while true; do
+}
+opt_keyring() {
+  KEYRING="$1"
+}
+opt_mirror() {
+  # consumed by setup-testbed
+  export MIRROR="$1"
+}
+opt_script() {
+  test -f "$1" || die "passed script '$1' does not refer to a file"
+  SCRIPT="$1"
+}
+opt_size() {
+  SIZE="$1"
+}
+
+positional=1
+positional_1() {
+  # consumed by setup-testbed
+  export RELEASE="$1"
+}
+positional_2() {
+  IMAGE="$1"
+}
+positional_3() { opt_mirror "$@"; }
+positional_4() { opt_architecture "$@"; }
+positional_5() { opt_script "$@"; }
+positional_6() { opt_size "$@"; }
+positional_7() {
+  die "too many positional options"
+}
+
+while test "$#" -gt 0; do
   case "$1" in
-			--size) SIZE="$2"; shift 2; continue;;
-			--boot) BOOT="$2"; shift 2; continue;;
-			--help) usage; exit 1;;
-			--architecture)
-				echo "Error: cannot (yet) create foreign architecture images" >&2
-				exit 1
+    --architecture=* | --arch=* | --boot=* | --keyring=* | --mirror=* | --script=* | --size=*)
+      optname="${1%%=*}"
+      "opt_${optname#--}" "${1#*=}"
+      ;;
+    --apt-proxy=*)
+      opt_apt_proxy "${1#*=}"
+      ;;
+    --architecture | --arch | --boot | --keyring | --mirror | --script | --size)
+      test "$#" -ge 2 || usage_error "missing argument for $1"
+      "opt_${1#--}" "$2"
+      shift
+      ;;
+    --apt-proxy)
+      test "$#" -ge 2 || usage_error "missing argument for $1"
+      opt_apt_proxy "$2"
+      shift
+      ;;
+    --efi)
+      opt_boot efi
+      ;;
+    --*)
+      usage_error "unrecognized argument $1"
       ;;
-			--) shift; break;;
     *)
-				echo "Error: unknown option $1" >&2
-				usage
-				exit 1
+      "positional_$positional" "$1"
+      positional=$((positional + 1))
       ;;
   esac
-	done
-	RELEASE=$1
-	IMAGE=$2
-fi
+  shift
+done
 
-# By default with --boot=auto (the default), bios boot is chosen for
-# amd64 and i386. Compare /usr/share/autopkgtest/lib/autopkgtest_qemu.py
-# But in practice, amd64 and i386 also support efi boot. But then
-# autopkgtest-virt-qemu has to be run with --boot=efi
-case "$BOOT" in
-	auto)
-		case "$nativearch" in
-			amd64|i386) BOOT=bios;;
-			armhf|arm64) BOOT=efi;;
-			ppc64el) BOOT=ieee1275;;
-		esac
-		;;
-	bios)
-		case "$nativearch" in amd64|i386);;
-			*)
-				echo "bios booting only possible on amd64 and i386" >&2
-				exit 1
-				;;
-		esac
-		;;
-	efi)
-		case "$nativearch" in amd64|i386|armhf|arm64);;
-			*)
-				echo "efi booting only possible on amd64, i386, armhf and arm64" >&2
-				exit 1
-				;;
-		esac
-		;;
-	ieee1275)
-		if [ "$nativearch" != "ppc64el" ]; then
-			echo "ieee1275 booting only possible on ppc64el" >&2
-			exit 1
-		fi
-		;;
-	*)
-		echo "invalid value for --boot" >&2;;
-esac
+test -z "$RELEASE" -o -z "$IMAGE" && usage_error "missing positional arguments"
+test "$BOOT" = efi \
+  || die "this tool does not support boot modes other than efi"
 
-case "$nativearch" in
+case "$ARCHITECTURE" in
   amd64)
-		[ $BOOT = bios ] || [ $BOOT = efi ]
-		if [ $BOOT = bios ]; then
-			include="linux-image-amd64 grub-pc"
-			grub_target="i386-pc"
-		elif [ $BOOT = efi ]; then
-			include="linux-image-amd64 grub-efi"
-			grub_target="x86_64-efi"
-		fi
+    EFIIMG=bootx64.efi
+    QEMUARCH=x86_64
+    VMFPKG=ovmf
+    LINUXIMAGE=linux-image-amd64
     ;;
   arm64)
-		[ $BOOT = efi ]
-		include="linux-image-arm64 grub-efi"
-		grub_target="arm64-efi"
+    EFIIMG=bootaa64.efi
+    QEMUARCH=aarch64
+    VMFPKG=qemu-efi-aarch64
+    LINUXIMAGE=linux-image-arm64
     ;;
   armhf)
-		[ $BOOT = efi ]
-		include="linux-image-armmp-lpae grub-efi"
-		grub_target="arm-efi"
+    EFIIMG=bootarm.efi
+    QEMUARCH=arm
+    VMFPKG=qemu-efi-arm
+    LINUXIMAGE=linux-image-armmp
     ;;
   i386)
-		[ $BOOT = bios ] || [ $BOOT = efi ]
-		if [ $BOOT = bios ]; then
-			include="linux-image-686-pae grub-pc"
-			grub_target="i386-pc"
-		elif [ $BOOT = efi ]; then
-			include="linux-image-686-pae grub-efi"
-			grub_target="i386-efi"
-		fi
+    EFIIMG=bootia32.efi
+    QEMUARCH=i386
+    VMFPKG=ovmf-ia32
+    LINUXIMAGE=linux-image-686-pae
     ;;
-	ppc64el)
-		[ $BOOT = ieee1275 ]
-		include="linux-image-powerpc64le grub-ieee1275"
-		grub_target="powerpc-ieee1275"
+  riscv64)
+    EFIIMG=bootriscv64.efi
+    QEMUARCH=riscv64
+    VMFPKG=
+    LINUXIMAGE=linux-image-riscv64
     ;;
   *)
-		echo "architecture $nativearch not yet supported" >&2
-		exit 1
+    die "unsupported architecture: $ARCHITECTURE"
     ;;
 esac
 
-case "$nativearch" in
-	arm64|armhf) serial="loglevel=3 console=tty0 console=ttyAMA0,115200n8" ;;
-	ppc64el) serial="loglevel=3 console=tty0 console=hvc0,115200n8" ;;
-	*) serial="loglevel=3 console=tty0 console=ttyS0,115200n8" ;;
-esac
-
-if ! command -v guestfish >/dev/null; then
-	echo "Error: requires guestfish being installed" >&2
-	exit 1
+if test "$(dpkg-query -f '${db:Status-Status}' -W binutils-multiarch)" = installed; then
+  GNU_PREFIX=
+  BINUTILS=
+else
+  GNU_ARCHITECTURE="$(dpkg-architecture "-a$ARCHITECTURE" -qDEB_HOST_GNU_TYPE)"
+  GNU_PREFIX="$GNU_ARCHITECTURE-"
+  GNU_SUFFIX="-$(echo "$GNU_ARCHITECTURE" | tr _ -)"
+  BINUTILS=", binutils$GNU_SUFFIX | binutils-multiarch"
 fi
 
-if [ ! -e /usr/share/autopkgtest/setup-commands/setup-testbed ]; then
-	echo "Error: requires autopkgtest being installed" >&2
-	exit 1
-fi
-
-run_mmdebstrap() {
-	mmdebstrap --variant=important --include="$include" \
-	--customize-hook='chroot "$1" passwd --delete root' \
-	--customize-hook='chroot "$1" useradd --home-dir /home/user --create-home user' \
-	--customize-hook='chroot "$1" passwd --delete user' \
-	--customize-hook='echo host > "$1/etc/hostname"' \
-	--customize-hook='echo "127.0.0.1 localhost host" > "$1/etc/hosts"' \
-	--customize-hook='env AUTOPKGTEST_BUILD_QEMU=1 /usr/share/autopkgtest/setup-commands/setup-testbed "$1"' \
-	"$RELEASE" -
-}
-
-guestfish_bios() {
-	guestfish -- \
-	disk-create "$IMAGE" qcow2 "$SIZE" : \
-	add-drive "$IMAGE" format:qcow2 : \
-	launch : \
-	part-disk /dev/sda mbr : \
-	part-set-bootable /dev/sda 1 true : \
-	mkfs ext4 /dev/sda1 : mount /dev/sda1 / : \
-	tar-in - / xattrs:true : \
-	command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda1) / ext4 errors=remount-ro 0 1 > /etc/fstab'" : \
-	command "update-initramfs -u" : \
-	command "grub-mkconfig -o /boot/grub/grub.cfg" : \
-	command "grub-install /dev/sda --target=$grub_target --no-nvram --force-extra-removable --no-floppy --modules=part_gpt --grub-mkdevicemap=/boot/grub/device.map" : \
-	sync : umount / : shutdown
-}
-
-guestfish_efi() {
-	guestfish -- \
-	disk-create "$IMAGE" qcow2 "$SIZE" : \
-	add-drive "$IMAGE" format:qcow2 : \
-	launch : \
-	part-init /dev/sda gpt : \
-	part-add /dev/sda primary 8192 262144 : \
-	part-add /dev/sda primary 262145 -34 : \
-	part-set-gpt-type /dev/sda 1 C12A7328-F81F-11D2-BA4B-00A0C93EC93B  : \
-	mkfs ext4 /dev/sda2 : mount /dev/sda2 / : \
-	tar-in - / xattrs:true : \
-	mkdir-p /boot/efi : \
-	mkfs vfat /dev/sda1 : mount /dev/sda1 /boot/efi : \
-	command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda2) / ext4 errors=remount-ro 0 1 > /etc/fstab'" : \
-	command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda1) /boot/efi vfat errors=remount-ro 0 2 >> /etc/fstab'" : \
-	command "sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=/GRUB_CMDLINE_LINUX_DEFAULT=\"biosdevname=0 net.ifnames=0 consoleblank=0 rw $serial\"/' /etc/default/grub" : \
-	command "update-initramfs -u" : \
-	command "grub-mkconfig -o /boot/grub/grub.cfg" : \
-	command "grub-install /dev/sda --target=$grub_target --no-nvram --force-extra-removable --no-floppy --modules=part_gpt --grub-mkdevicemap=/boot/grub/device.map" : \
-	sync : umount /boot/efi : umount / : shutdown
-}
-
-guestfish_ieee1275() {
-	guestfish -- \
-	disk-create "$IMAGE" qcow2 "$SIZE" : \
-	add-drive "$IMAGE" format:qcow2 : \
-	launch : \
-	part-init /dev/sda gpt : \
-	part-add /dev/sda primary 8192 20480 : \
-	part-add /dev/sda primary 20481 -34 : \
-	part-set-gpt-type /dev/sda 1 9E1A2D38-C612-4316-AA26-8B49521E5A8B : \
-	mkfs ext4 /dev/sda2 : mount /dev/sda2 / : \
-	tar-in - / xattrs:true : \
-	command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda2) / ext4 errors=remount-ro 0 1 > /etc/fstab'" : \
-	command "sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=/GRUB_CMDLINE_LINUX_DEFAULT=\"biosdevname=0 net.ifnames=0 consoleblank=0 rw $serial\"/' /etc/default/grub" : \
-	command "update-initramfs -u" : \
-	command "grub-mkconfig -o /boot/grub/grub.cfg" : \
-	command "grub-install /dev/sda --target=$grub_target --no-nvram --force-extra-removable --no-floppy --modules=part_gpt --grub-mkdevicemap=/boot/grub/device.map" : \
-	sync : umount / : shutdown
-}
-
-case "$BOOT" in
-	bios) run_mmdebstrap | guestfish_bios;;
-	efi) run_mmdebstrap | guestfish_efi;;
-	ieee1275) run_mmdebstrap | guestfish_ieee1275;;
+arches=" $(dpkg --print-architecture) $(dpkg --print-foreign-architectures | tr '\n' ' ') "
+case $arches in
+  *" $ARCHITECTURE "*) : ;; # nothing to do
+  *) die "enable $ARCHITECTURE by running: sudo dpkg --add-architecture $ARCHITECTURE && sudo apt update" ;;
 esac
 
-echo "Success! The image is stored as $IMAGE" >&2
+test "$(dpkg-query -f '${db:Status-Status}' -W "dpkg-dev")" = installed \
+  || die "please install dpkg-dev"
+
+dpkg-checkbuilddeps -d "autopkgtest, dosfstools, e2fsprogs, fdisk, mount, mtools, passwd, uidmap, libarchive13, systemd-boot-efi:$ARCHITECTURE $BINUTILS" /dev/null \
+  || die "please install the required packages listed above"
+
+BOOTSTUB="/usr/lib/systemd/boot/efi/linux${EFIIMG#boot}.stub"
+
+WORKDIR=
+
+cleanup() {
+  test -n "$WORKDIR" && rm -Rf "$WORKDIR"
+}
+
+trap cleanup EXIT INT TERM QUIT
+
+WORKDIR=$(mktemp -d)
+
+FAT_OFFSET_SECTORS=$((1024 * 2))
+FAT_SIZE_SECTORS=$((1024 * 254))
+
+# The image is raw and not in qcow2 format because:
+#  - faster run-time as the "qemu-image convert" step is not needed
+#  - image can be used independent of qemu tooling
+#  - modifying the image just with "mount" instead of requiring qemu-nbd
+#  - sparse images make the file just as small as with qcow2
+#  - trim support is more difficult on qcow2
+#  - snapshots and overlays work just as well with raw images
+#  - users who prefer qcow2 get to choose to run it themselves with their own
+#    custom options like compression
+
+set -- \
+  --mode=unshare \
+  --format=tar \
+  --variant=important \
+  --architecture="$ARCHITECTURE"
+
+case $MIRROR in http://snapshot.debian.org/archive/* | https://snapshot.debian.org/archive/*)
+  set -- "$@" --aptopt='Acquire::Check-Valid-Until "false"'
+  ;;
+esac
+
+EXT_FEATURES=
+if test "$RELEASE" = jessie; then
+  set -- "$@" --keyring=/usr/share/keyrings/debian-archive-removed-keys.gpg
+  set -- "$@" --aptopt='Apt::Key::gpgvcommand "/usr/libexec/mmdebstrap/gpgvnoexpkeysig"'
+  set -- "$@" --hook-dir=/usr/share/mmdebstrap/hooks/jessie-or-older
+  EXT_FEATURES="^metadata_csum,^metadata_csum_seed,^orphan_file"
+fi
+
+set -- "$@" \
+  "--include=init,$LINUXIMAGE,python3" \
+  '--customize-hook=echo host >"$1/etc/hostname"' \
+  '--customize-hook=echo 127.0.0.1 localhost host >"$1/etc/hosts"' \
+  '--customize-hook=passwd --root "$1" --delete root' \
+  '--customize-hook=useradd --root "$1" --home-dir /home/user --create-home user' \
+  '--customize-hook=passwd --root "$1" --delete user' \
+  '--customize-hook=/usr/share/autopkgtest/setup-commands/setup-testbed'
+
+if test -n "$SCRIPT"; then
+  set -- "$@" \
+    "--customize-hook=upload '$SCRIPT' /userscript" \
+    "--chrooted-customize-hook=sh /userscript" \
+    '--customize-hook=rm -f "$1/userscript"'
+fi
+
+set -- "$@" \
+  "--customize-hook=download vmlinuz '$WORKDIR/kernel'" \
+  "--customize-hook=download initrd.img '$WORKDIR/initrd'" \
+  "$RELEASE" \
+  -
+
+test -n "$MIRROR" && set -- "$@" "$MIRROR"
+test -n "$KEYRING" && set -- "$@" "--keyring=$KEYRING"
+
+echo "+ mmdebstrap $*" >&2
+# https://github.com/koalaman/shellcheck/issues/2555
+# shellcheck disable=SC3040
+set -o pipefail
+mmdebstrap "$@" | {
+  set -- -t ext4 -L autopkgtestvm -d -
+  if test -n "$EXT_FEATURES"; then
+    set -- "$@" -O "$EXT_FEATURES"
+  fi
+  EXTOPTS="offset=$(((FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS) * 512))"
+  if test -n "${SOURCE_DATE_EPOCH-}"; then
+    uuid="$(uuidgen --sha1 --namespace="$(uuidgen --sha1 --namespace='@dns' --name mister-muffin.de)" --name "$SOURCE_DATE_EPOCH")"
+    set -- "$@" -U "$uuid"
+    EXTOPTS="$EXTOPTS,hash_seed=$uuid"
+  fi
+  set -- "$@" -E "$EXTOPTS" "$IMAGE" "$SIZE"
+  echo "+ mke2fs $*" >&2
+  /sbin/mke2fs "$@"
+}
+
+echo "root=LABEL=autopkgtestvm rw console=ttyS0" >"$WORKDIR/cmdline"
+
+align_size() {
+  echo "$((($1) + ($2) - 1 - (($1) + ($2) - 1) % ($2)))"
+}
+
+alignment=$("${GNU_PREFIX}objdump" -p "$BOOTSTUB" | sed 's/^SectionAlignment\s\+\([0-9]\)/0x/;t;d')
+test -z "$alignment" && die "failed to discover the alignment of the efi stub"
+echo "determined efi vma alignment as $alignment"
+test "$RELEASE" = jessie -a "$((alignment))" -lt "$((1024 * 1024))" && {
+  echo "increasing efi vma alignment for jessie"
+  alignment=$((1024 * 1024))
+}
+lastoffset=0
+# shellcheck disable=SC2034  # unused variables serve documentation
+lastoffset="$("${GNU_PREFIX}objdump" -h "$BOOTSTUB" \
+  | while read -r idx name size vma lma fileoff algn behind; do
+    test -z "$behind" -a "${algn#"2**"}" != "$algn" || continue
+    offset=$((0x$vma + 0x$size))
+    test "$offset" -gt "$lastoffset" || continue
+    lastoffset="$offset"
+    echo "$lastoffset"
+  done | tail -n1)"
+lastoffset=$(align_size "$lastoffset" "$alignment")
+echo "determined minimum efi vma offset as $lastoffset"
+
+cmdline_size="$(stat -Lc%s "$WORKDIR/cmdline")"
+cmdline_size="$(align_size "$cmdline_size" "$alignment")"
+linux_size="$(stat -Lc%s "$WORKDIR/kernel")"
+linux_size="$(align_size "$linux_size" "$alignment")"
+cmdline_offset="$lastoffset"
+linux_offset=$((cmdline_offset + cmdline_size))
+initrd_offset=$((linux_offset + linux_size))
+
+SOURCE_DATE_EPOCH=0 \
+  "${GNU_PREFIX}objcopy" \
+  --enable-deterministic-archives \
+  --add-section .cmdline="$WORKDIR/cmdline" \
+  --change-section-vma .cmdline="$(printf 0x%x "$cmdline_offset")" \
+  --add-section .linux="$WORKDIR/kernel" \
+  --change-section-vma .linux="$(printf 0x%x "$linux_offset")" \
+  --add-section .initrd="$WORKDIR/initrd" \
+  --change-section-vma .initrd="$(printf 0x%x "$initrd_offset")" \
+  "$BOOTSTUB" "$WORKDIR/efiimg"
+
+rm -f "$WORKDIR/kernel" "$WORKDIR/initrd"
+
+truncate -s "$((FAT_SIZE_SECTORS * 512))" "$WORKDIR/fat"
+/sbin/mkfs.fat -F 32 --invariant "$WORKDIR/fat"
+mmd -i "$WORKDIR/fat" EFI EFI/BOOT
+mcopy -i "$WORKDIR/fat" "$WORKDIR/efiimg" "::EFI/BOOT/$EFIIMG"
+
+rm -f "$WORKDIR/efiimg"
+
+truncate --size="+$((34 * 512))" "$IMAGE"
+/sbin/sfdisk "$IMAGE" <<EOF
+label: gpt
+unit: sectors
+
+start=$FAT_OFFSET_SECTORS, size=$FAT_SIZE_SECTORS, type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B
+start=$((FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS)), type=0FC63DAF-8483-4772-8E79-3D69D8477DE4
+EOF
+
+dd if="$WORKDIR/fat" of="$IMAGE" conv=notrunc,sparse bs=512 "seek=$FAT_OFFSET_SECTORS" status=none
+
+if test "$(dpkg --print-architecture)" != "$ARCHITECTURE" && test "$(dpkg-query -f '${db:Status-Status}' -W "qemu-system-$QEMUARCH")" != installed; then
+  echo "I: you might need to install a package providing qemu-system-$QEMUARCH to use this image with autopkgtest-virt-qemu" >&2
+fi
+if test -n "$VMFPKG" && test "$(dpkg-query -f '${db:Status-Status}' -W "$VMFPKG")" != installed; then
+  echo "I: you might need to install $VMFPKG to use this image with autopkgtest-virt-qemu" >&2
+fi
+
+echo "I: SUCCESS! Your new image can be found here: $IMAGE" >&2
+echo "I: Don't forget to pass --boot=efi when running autopkgtest-virt-qemu with this image" >&2

run_null.sh

@@ -17,14 +17,33 @@ while [ "$#" -gt 0 ]; do
   shift
 done
 
-# subshell so that we can cd without effecting the rest
-(
-	set +e
-	cd ./shared;
-	$SUDO sh -x ./test.sh;
-	echo $?;
-) 2>&1 | tee shared/result.txt | head --lines=-1
-if [ "$(tail --lines=1 shared/result.txt)" -ne 0 ]; then
+# - Run command with fds 3 and 4 closed so that whatever test.sh does it
+#   cannot interfere with these.
+# - Both stdin and stderr of test.sh are written to stdout
+# - Write exit status of test.sh to fd 3
+# - Write stdout to shared/output.txt as well as to fd 4
+# - Redirect fd 3 to stdout
+# - Read fd 3 and let the group exit with that value
+# - Redirect fd 4 to stdout
+ret=0
+{
+  {
+    {
+      {
+        ret=0
+        (
+          exec 3>&- 4>&-
+          env --chdir=./shared $SUDO sh -x ./test.sh 2>&1
+        ) || ret=$?
+        echo $ret >&3
+      } | tee shared/output.txt >&4
+    } 3>&1
+  } | {
+    read -r xs
+    exit "$xs"
+  }
+} 4>&1 || ret=$?
+if [ "$ret" -ne 0 ]; then
   echo "test.sh failed"
   exit 1
 fi

run_qemu.sh

@@ -4,17 +4,15 @@ set -eu
 
 : "${DEFAULT_DIST:=unstable}"
 : "${cachedir:=./shared/cache}"
+: "${MMDEBSTRAP_TESTS_DEBUG:=no}"
 tmpdir="$(mktemp -d)"
 
 cleanup() {
   rv=$?
-	rm -f "$tmpdir/debian-$DEFAULT_DIST-overlay.qcow"
   rm -f "$tmpdir/log"
   [ -e "$tmpdir" ] && rmdir "$tmpdir"
-	if [ -e shared/result.txt ]; then
-		head --lines=-1 shared/result.txt
-		res="$(tail --lines=1 shared/result.txt)"
-		rm shared/result.txt
+  if [ -e shared/output.txt ]; then
+    res="$(cat shared/exitstatus.txt)"
     if [ "$res" != "0" ]; then
       # this might possibly overwrite another non-zero rv
       rv=1
@@ -25,48 +23,49 @@ cleanup() {
 
 trap cleanup INT TERM EXIT
 
-ARCH=$(dpkg --print-architecture)
-case $ARCH in
-	i386)
-		MACHINE="accel=kvm:tcg"
-		CODE="/usr/share/OVMF/OVMF32_CODE_4M.secboot.fd"
-		QEMUARCH="i386"
-		;;
-	amd64)
-		MACHINE="accel=kvm:tcg"
-		CODE="/usr/share/OVMF/OVMF_CODE.fd"
-		QEMUARCH="x86_64"
-		;;
-	arm64)
-		MACHINE="type=virt,gic-version=host,accel=kvm"
-		CODE="/usr/share/AAVMF/AAVMF_CODE.fd"
-		QEMUARCH="aarch64"
-		;;
-	*) echo "qemu kvm not supported on $ARCH" >&2;;
-esac
+echo 1 >shared/exitstatus.txt
+if [ -e shared/output.txt ]; then
+  rm shared/output.txt
+fi
+touch shared/output.txt
+setpriv --pdeathsig TERM tail -f shared/output.txt &
 
-# the path to debian-$DEFAULT_DIST.qcow must be absolute or otherwise qemu will
-# look for the path relative to debian-$DEFAULT_DIST-overlay.qcow
-qemu-img create -f qcow2 -b "$(realpath "$cachedir")/debian-$DEFAULT_DIST.qcow" -F qcow2 "$tmpdir/debian-$DEFAULT_DIST-overlay.qcow"
-# to connect to serial use:
-#   minicom -D 'unix#/tmp/ttyS0'
-#
-# or this (quit with ctrl+q):
-#   socat stdin,raw,echo=0,escape=0x11 unix-connect:/tmp/ttyS0
-ret=0
-timeout --foreground 40m qemu-system-"$QEMUARCH" \
-	-cpu host \
-	-no-user-config \
-	-M "$MACHINE" -m 4G -nographic \
-	-object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \
+set -- timeout --foreground 40m \
+  debvm-run --image="$(realpath "$cachedir")/debian-$DEFAULT_DIST.ext4" \
+  --
+cpuname=$(lscpu | awk '/Model name:/ {print $3}' | tr '\n' '+')
+ncpu=$(lscpu | awk '/Core\(s\) per socket:/ {print $4}' | tr '\n' '+')
+if [ "$cpuname" = "Cortex-A53+Cortex-A73+" ] && [ "$ncpu" = "2+4+" ]; then
+  # crude detection of the big.LITTLE heterogeneous setup of cores on the
+  # amlogic a311d bananapi
+  #
+  # https://lists.nongnu.org/archive/html/qemu-devel/2020-10/msg08494.html
+  # https://gitlab.com/qemu-project/qemu/-/issues/239
+  # https://segments.zhan.science/posts/kvm_on_pinehone_pro/#trouble-with-heterogeneous-architecture
+  set -- taskset --cpu-list 2,3,4,5 "$@" -smp 4
+fi
+
+set -- "$@" -nic none -m 4G -snapshot
+
+if [ "$MMDEBSTRAP_TESTS_DEBUG" = "no" ]; then
+  # to connect to serial use:
+  #   minicom -D 'unix#/tmp/ttyS0'
+  # or this (quit with ctrl+q):
+  #   socat stdin,raw,echo=0,escape=0x11 unix-connect:/tmp/ttyS0
+  set -- "$@" \
     -monitor unix:/tmp/monitor,server,nowait \
     -serial unix:/tmp/ttyS0,server,nowait \
-	-serial unix:/tmp/ttyS1,server,nowait \
-	-net nic,model=virtio -net user \
-	-drive if=pflash,format=raw,unit=0,read-only=on,file="$CODE" \
-	-virtfs local,id=mmdebstrap,path="$(pwd)/shared",security_model=none,mount_tag=mmdebstrap \
-	-drive file="$tmpdir/debian-$DEFAULT_DIST-overlay.qcow",cache=unsafe,index=0,if=virtio \
-	>"$tmpdir/log" 2>&1 || ret=$?
+    -serial unix:/tmp/ttyS1,server,nowait
+fi
+
+set -- "$@" -virtfs local,id=mmdebstrap,path="$(pwd)/shared",security_model=none,mount_tag=mmdebstrap
+
+ret=0
+if [ "$MMDEBSTRAP_TESTS_DEBUG" = "no" ]; then
+  "$@" >"$tmpdir/log" 2>&1 || ret=$?
+else
+  "$@" 2>&1 | tee "$tmpdir/log" || ret=$?
+fi
 if [ "$ret" -ne 0 ]; then
   cat "$tmpdir/log"
   exit $ret

tarfilter

@@ -43,6 +43,29 @@ class PaxFilterAction(argparse.Action):
         setattr(namespace, "paxfilter", items)
 
 
+class TypeFilterAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = getattr(namespace, "typefilter", [])
+        match values:
+            case "REGTYPE" | "0":
+                items.append(tarfile.REGTYPE)
+            case "LNKTYPE" | "1":
+                items.append(tarfile.LNKTYPE)
+            case "SYMTYPE" | "2":
+                items.append(tarfile.SYMTYPE)
+            case "CHRTYPE" | "3":
+                items.append(tarfile.CHRTYPE)
+            case "BLKTYPE" | "4":
+                items.append(tarfile.BLKTYPE)
+            case "DIRTYPE" | "5":
+                items.append(tarfile.DIRTYPE)
+            case "FIFOTYPE" | "6":
+                items.append(tarfile.FIFOTYPE)
+            case _:
+                raise ValueError("invalid type: %s" % values)
+        setattr(namespace, "typefilter", items)
+
+
 class TransformAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         items = getattr(namespace, "trans", [])
@@ -89,10 +112,11 @@ dpkg(1) for information on how these two options work in detail. To reuse the
 exact same semantics as used by dpkg, paths must be given as /path and not as
 ./path even though they might be stored as such in the tarball.
 
-Secondly, filter out unwanted pax extended headers. This is useful in cases
-where a tool only accepts certain xattr prefixes. For example tar2sqfs only
-supports SCHILY.xattr.user.*, SCHILY.xattr.trusted.* and
-SCHILY.xattr.security.* but not SCHILY.xattr.system.posix_acl_default.*.
+Secondly, filter out unwanted pax extended headers using --pax-exclude and
+--pax-include. This is useful in cases where a tool only accepts certain xattr
+prefixes. For example tar2sqfs only supports SCHILY.xattr.user.*,
+SCHILY.xattr.trusted.* and SCHILY.xattr.security.* but not
+SCHILY.xattr.system.posix_acl_default.*.
 
 Both types of options use Unix shell-style wildcards:
 
@@ -101,10 +125,16 @@ Both types of options use Unix shell-style wildcards:
    [seq] matches any character in seq
   [!seq] matches any character not in seq
 
-Thirdly, transform the path of tar members using a sed expression just as with
+Thirdly, filter out files matching a specific tar archive member type using
+--type-exclude. Valid type names are REGTYPE (regular file), LNKTYPE
+(hardlink), SYMTYPE (symlink), CHRTYPE (character special), BLKTYPE (block
+special), DIRTYPE (directory), FIFOTYPE (fifo) or their tar format flag value
+(0-6, respectively).
+
+Fourthly, transform the path of tar members using a sed expression just as with
 GNU tar --transform.
 
-Fourthly, strip leading directory components off of tar members. Just as with
+Fifthly, strip leading directory components off of tar members. Just as with
 GNU tar --strip-components, tar members that have less or equal components in
 their path are not passed through.
 
@@ -140,6 +170,15 @@ Lastly, shift user id and group id of each entry by the value given by the
         help="Re-include a pax header after a previous exclusion. "
         "This option can be specified multiple times.",
     )
+    parser.add_argument(
+        "--type-exclude",
+        metavar="type",
+        action=TypeFilterAction,
+        help="Exclude certain member types by their type. Choose types either "
+        "by their name (REGTYPE, LNKTYPE, SYMTYPE, CHRTYPE, BLKTYPE, DIRTYPE, "
+        "FIFOTYPE) or by their tar format flag values (0-6, respectively). "
+        "This option can be specified multiple times.",
+    )
     parser.add_argument(
         "--transform",
         "--xform",
@@ -164,6 +203,7 @@ Lastly, shift user id and group id of each entry by the value given by the
     if (
         not hasattr(args, "pathfilter")
         and not hasattr(args, "paxfilter")
+        and not hasattr(args, "typefilter")
         and not hasattr(args, "strip_components")
     ):
         from shutil import copyfileobj
@@ -178,19 +218,22 @@ Lastly, shift user id and group id of each entry by the value given by the
         skip = False
         if not hasattr(args, "pathfilter"):
             return False
-        for (t, r) in args.pathfilter:
-            if r.match(member.name[1:]) is not None:
+        # normalize path and make it absolute by stripping off all leading
+        # dots and slashes and then prepending a slash
+        name = "/" + member.name.lstrip("./")
+        for t, r in args.pathfilter:
+            if r.match(name) is not None:
                 if t == "path_include":
                     skip = False
                 else:
                     skip = True
         if skip and (member.isdir() or member.issym()):
-            for (t, r) in args.pathfilter:
+            for t, r in args.pathfilter:
                 if t != "path_include":
                     continue
                 prefix = prefix_prog.sub(r"\1", r.pattern)
                 prefix = prefix.rstrip("/")
-                if member.name[1:].startswith(prefix):
+                if name.startswith(prefix):
                     return False
         return skip
 
@@ -198,7 +241,7 @@ Lastly, shift user id and group id of each entry by the value given by the
         if not hasattr(args, "paxfilter"):
             return False
         skip = False
-        for (t, r) in args.paxfilter:
+        for t, r in args.paxfilter:
             if r.match(header) is None:
                 continue
             if t == "pax_include":
@@ -207,14 +250,24 @@ Lastly, shift user id and group id of each entry by the value given by the
                 skip = True
         return skip
 
-    # starting with Python 3.8, the default format became PAX_FORMAT, so this
-    # is only for compatibility with older versions of Python 3
+    def type_filter_should_skip(member):
+        if not hasattr(args, "typefilter"):
+            return False
+        for t in args.typefilter:
+            if member.type == t:
+                return True
+        return False
+
+    # starting with Python 3.8, the default format became PAX_FORMAT but we
+    # are still explicit here in case of future changes.
     with tarfile.open(fileobj=sys.stdin.buffer, mode="r|*") as in_tar, tarfile.open(
         fileobj=sys.stdout.buffer, mode="w|", format=tarfile.PAX_FORMAT
     ) as out_tar:
         for member in in_tar:
             if path_filter_should_skip(member):
                 continue
+            if type_filter_should_skip(member):
+                continue
             if args.strip_components:
                 comps = member.name.split("/")
                 # just as with GNU tar, archive members with less or equal

tests/apt-patterns

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+{{ CMD }} --mode={{ MODE }} --variant=essential \
+  --include '?or(?exact-name(dummy-does-not-exist),?exact-name(apt))' \
+  {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/apt-patterns-custom

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+{{ CMD }} --mode={{ MODE }} --variant=custom \
+  --include '?narrow(?archive(^{{ DIST }}$),?essential)' \
+  --include apt \
+  {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+{
+  tar -tf /tmp/debian-chroot.tar
+  echo ./var/lib/apt/extended_states
+} | sort | diff -u tar1.txt -

tests/aptopt

@@ -2,7 +2,7 @@
 set -eu
 export LC_ALL=C.UTF-8
 trap "rm -rf /tmp/debian-chroot; rm -f /tmp/config" EXIT INT TERM
-echo 'Acquire::Languages "none";' > /tmp/config
+echo 'Acquire::Languages "none";' >/tmp/config
 {{ CMD }} --mode=root --variant=apt --aptopt='Acquire::Check-Valid-Until "false"' --aptopt=/tmp/config {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 printf 'Acquire::Check-Valid-Until "false";\nAcquire::Languages "none";\n' | cmp /tmp/debian-chroot/etc/apt/apt.conf.d/99mmdebstrap -
 rm /tmp/debian-chroot/etc/apt/apt.conf.d/99mmdebstrap

tests/arm64-without-qemu-support

@@ -5,7 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
   echo "this test modifies the system and should only be run inside a container" >&2
   exit 1
 fi
-apt-get remove --yes qemu-user-static binfmt-support qemu-user
+apt-get remove --yes qemu-user-binfmt binfmt-support qemu-user
 # the following is not necessary anymore since systemd-binfmt
 # successfully disables support upon removal of qemu-user with
 # the upload of src:systemd 251.2-4: https://bugs.debian.org/1012163

tests/as-debootstrap-unshare-wrapper

@@ -2,19 +2,51 @@
 set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-if [ ! -e /mmdebstrap-testenv ]; then
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+    if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
+    fi
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
+fi
+
+# debootstrap uses apt-config to figure out whether the system running it has
+# any proxies configured and then runs the binary to set the http_proxy
+# environment variable. This will fail if debootstrap is run in a linux user
+# namespace because auto-apt-proxy will see /tmp/.auto-apt-proxy-0 as being
+# owned by the user "nobody" and group "nogroup" and fail with:
+#   insecure cache dir /tmp/.auto-apt-proxy-0. Must be owned by UID 0 and have permissions 700
+# We cannot overwrite a configuration item using the APT_CONFIG environment
+# variable, so instead we use it to set the Dir configuration option
+# to /dev/null to force all apt settings to their defaults.
+# There is currently no better way to disable this behavior. See also:
+# https://bugs.debian.org/1031105
+# https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/90
+AUTOPROXY=
+eval "$(apt-config shell AUTOPROXY Acquire::http::Proxy-Auto-Detect)"
+if [ -n "$AUTOPROXY" ] && [ -x "$AUTOPROXY" ] && [ -e /tmp/.auto-apt-proxy-0 ]; then
+  TMP_APT_CONFIG=$(mktemp)
+  echo 'Dir "/dev/null";' >"$TMP_APT_CONFIG"
+  chmod 644 "$TMP_APT_CONFIG"
+fi
+
+$prefix {{ CMD }} --variant=custom --mode={{ MODE }} \
+  --setup-hook='env '"${AUTOPROXY:+APT_CONFIG='$TMP_APT_CONFIG'}"' debootstrap --variant={{ VARIANT }} unstable "$1" {{ MIRROR }}' \
+  - /tmp/debian-mm.tar {{ MIRROR }}
+if [ -n "$AUTOPROXY" ] && [ -x "$AUTOPROXY" ] && [ -e /tmp/.auto-apt-proxy-0 ]; then
+  rm "$TMP_APT_CONFIG"
 fi
-sysctl -w kernel.unprivileged_userns_clone=1
-adduser --gecos user --disabled-password user
-runuser -u user -- {{ CMD }} --variant=custom --mode=unshare --setup-hook='env container=lxc debootstrap unstable "$1" {{ MIRROR }}' - /tmp/debian-mm.tar {{ MIRROR }}
 
 mkdir /tmp/debian-mm
 tar --xattrs --xattrs-include='*' -C /tmp/debian-mm -xf /tmp/debian-mm.tar
 
 mkdir /tmp/debian-debootstrap
-tar --xattrs --xattrs-include='*' -C /tmp/debian-debootstrap -xf "cache/debian-unstable--.tar"
+tar --xattrs --xattrs-include='*' -C /tmp/debian-debootstrap -xf "cache/debian-unstable-{{ VARIANT }}.tar"
 
 # diff cannot compare device nodes, so we use tar to do that for us and then
 # delete the directory
@@ -39,30 +71,42 @@ find /tmp/debian-debootstrap/run/ -mindepth 1 -maxdepth 1 ! -name lock -print0 |
 
 # debootstrap doesn't clean apt
 rm /tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_main_binary-{{ HOSTARCH }}_Packages \
+  /tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_InRelease \
   /tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_Release \
   /tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_Release.gpg
 
-rm /tmp/debian-debootstrap/etc/machine-id /tmp/debian-mm/etc/machine-id
+if [ -e /tmp/debian-debootstrap/etc/machine-id ]; then
+  rm /tmp/debian-debootstrap/etc/machine-id /tmp/debian-mm/etc/machine-id
+fi
 rm /tmp/debian-mm/var/cache/apt/archives/lock
 rm /tmp/debian-mm/var/lib/apt/lists/lock
 rm /tmp/debian-mm/var/lib/dpkg/arch
 
-# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
 # also needed for users that are created by systemd-sysusers before systemd 252
 # https://github.com/systemd/systemd/pull/24534
 for f in shadow shadow-; do
+  if [ ! -e /tmp/debian-debootstrap/etc/$f ]; then
+    continue
+  fi
   if ! cmp /tmp/debian-debootstrap/etc/$f /tmp/debian-mm/etc/$f >&2; then
     echo patching /etc/$f >&2
-		awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-mm/etc/$f > /tmp/debian-mm/etc/$f.bak
-		cat /tmp/debian-mm/etc/$f.bak > /tmp/debian-mm/etc/$f
+    awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' </tmp/debian-mm/etc/$f >/tmp/debian-mm/etc/$f.bak
+    cat /tmp/debian-mm/etc/$f.bak >/tmp/debian-mm/etc/$f
     rm /tmp/debian-mm/etc/$f.bak
   else
     echo no difference for /etc/$f >&2
   fi
 done
 
+# isc-dhcp-client postinst doesn't create this file in debootstrap run with
+# unshared wrapper. The responsible postinst snippet was automatically added
+# by dh_apparmor since isc-dhcp-client 4.4.3-P1-1.1
+if [ -e /tmp/debian-debootstrap/etc/apparmor.d/local/sbin.dhclient ] && [ ! -s /tmp/debian-debootstrap/etc/apparmor.d/local/sbin.dhclient ]; then
+  echo /sbin/setcap >/tmp/debian-debootstrap/etc/apparmor.d/local/sbin.dhclient
+fi
+
 # check if the file content differs
-diff --no-dereference --recursive /tmp/debian-debootstrap /tmp/debian-mm >&2
+diff --unified --no-dereference --recursive /tmp/debian-debootstrap /tmp/debian-mm >&2
 
 # check permissions, ownership, symlink targets, modification times using tar
 # mtimes of directories created by mmdebstrap will differ, thus we equalize them first
@@ -76,8 +120,8 @@ for d in ./var/lib/apt/lists/partial ./var/cache/apt/archives/partial; do
 done
 tar -C /tmp/debian-debootstrap --numeric-owner --xattrs --xattrs-include='*' --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/root1.tar .
 tar -C /tmp/debian-mm --numeric-owner --xattrs --xattrs-include='*' --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/root2.tar .
-tar --full-time --verbose -tf /tmp/root1.tar > /tmp/root1.tar.list
-tar --full-time --verbose -tf /tmp/root2.tar > /tmp/root2.tar.list
+tar --full-time --verbose -tf /tmp/root1.tar >/tmp/root1.tar.list
+tar --full-time --verbose -tf /tmp/root2.tar >/tmp/root2.tar.list
 # despite SOURCE_DATE_EPOCH and --clamp-mtime, the timestamps in the tarball
 # will slightly differ from each other in the sub-second precision (last
 # decimals) so the tarballs will not be identical, so we use diff to compare

tests/ascii-armored-keys

@@ -5,14 +5,17 @@ if [ ! -e /mmdebstrap-testenv ]; then
   echo "this test modifies the system and should only be run inside a container" >&2
   exit 1
 fi
+for f in /etc/apt/trusted.gpg.d/*.gpg /etc/apt/trusted.gpg.d/*.asc; do
+  [ -e "$f" ] || continue
+  rm "$f"
+done
+rmdir /etc/apt/trusted.gpg.d
+mkdir /etc/apt/trusted.gpg.d
 for f in /usr/share/keyrings/*.gpg; do
   name=$(basename "$f" .gpg)
-	gpg --enarmor < "/usr/share/keyrings/$name.gpg" \
-		| sed 's/ PGP ARMORED FILE/ PGP PUBLIC KEY BLOCK/;/^Comment: /d' \
-		> "/etc/apt/trusted.gpg.d/$name.asc"
+  gpg --no-default-keyring --keyring="/usr/share/keyrings/$name.gpg" --armor --output="/etc/apt/trusted.gpg.d/$name.asc" --export
+  rm "/usr/share/keyrings/$name.gpg"
 done
-rm /etc/apt/trusted.gpg.d/*.gpg
-rm /usr/share/keyrings/*.gpg
 {{ CMD }} --mode=root --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
 rm -r /tmp/debian-chroot.tar

tests/aspcud-apt-solver

@@ -3,9 +3,12 @@ set -eu
 export LC_ALL=C.UTF-8
 trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 {{ CMD }} --mode={{ MODE }} --variant=custom \
-    --include "$(tr '\n' ',' < pkglist.txt)" \
+  --include "$(tr '\n' ',' <pkglist.txt)" \
   --aptopt='APT::Solver "aspcud"' \
   {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-tar -tf /tmp/debian-chroot.tar | sort \
+{
+  tar -tf /tmp/debian-chroot.tar
+  echo ./var/lib/apt/extended_states
+} | sort \
   | grep -v '^./etc/apt/apt.conf.d/99mmdebstrap$' \
   | diff -u tar1.txt -

tests/auto-mode-as-normal-user

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+
+trap "rm -f /tmp/debian-chroot.tar.gz" EXIT INT TERM
+
+[ {{ MODE }} = "auto" ]
+
+prefix=
+if [ "$(id -u)" -eq 0 ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+    if [ ! -e /mmdebstrap-testenv ]; then
+      echo "this test modifies the system and should only be run inside a container" >&2
+      exit 1
+    fi
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
+fi
+
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar.gz {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar.gz | sort | diff -u tar1.txt -

tests/auto-mode-without-unshare-capabilities

@@ -5,8 +5,10 @@ if [ ! -e /mmdebstrap-testenv ]; then
   echo "this test modifies the system and should only be run inside a container" >&2
   exit 1
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=0
+useradd --home-dir /home/user --create-home user
+if [ -e /proc/sys/kernel/unprivileged_userns_clone ] && [ "$(sysctl -n kernel.unprivileged_userns_clone)" = "1" ]; then
+  sysctl -w kernel.unprivileged_userns_clone=0
+fi
 runuser -u user -- {{ CMD }} --mode=auto --variant=apt {{ DIST }} /tmp/debian-chroot.tar.gz {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar.gz | sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar.gz

tests/automatic-mirror-from-suite

@@ -5,7 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
   echo "this test modifies the system and should only be run inside a container" >&2
   exit 1
 fi
-cat << HOSTS >> /etc/hosts
+cat <<HOSTS >>/etc/hosts
 127.0.0.1 deb.debian.org
 127.0.0.1 security.debian.org
 HOSTS

tests/check-against-debootstrap-dist

@@ -11,8 +11,9 @@ echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
 # https://bugs.debian.org/969631
 # we cannot use useradd because passwd is not Essential:yes
 {{ CMD }} --variant={{ VARIANT }} --mode={{ MODE }} \
-	--essential-hook='case {{ DIST }} in oldstable|stable) if [ {{ VARIANT }} = - ]; then echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd; fi;; esac' \
-	"$(case {{ DIST }} in oldstable|stable) echo --merged-usr ;; *) echo --hook-dir=./hooks/merged-usr ;; esac)" \
+  --essential-hook='[ {{ DIST }} = oldstable ] && [ {{ VARIANT }} = - ] && echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd || :' \
+  "$(if [ {{ DIST }} = oldstable ]; then echo --merged-usr; else echo --hook-dir=./hooks/merged-usr; fi)" \
+  "$(case {{ DIST }} in oldstable) echo --include=e2fsprogs,mount,tzdata,gcc-9-base ;; stable) echo --include=e2fsprogs,mount,tzdata ;; *) echo --include=base-files ;; esac)" \
   {{ DIST }} /tmp/debian-{{ DIST }}-mm.tar {{ MIRROR }}
 
 mkdir /tmp/debian-{{ DIST }}-mm
@@ -69,6 +70,8 @@ rm /tmp/debian-{{ DIST }}-debootstrap/var/cache/debconf/templates.dat-old \
   /tmp/debian-{{ DIST }}-mm/var/cache/debconf/templates.dat-old
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status-old \
   /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status-old
+rm -f /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/diversions-old \
+  /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/diversions-old
 # remove dpkg files
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/available
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/cmethopt
@@ -76,12 +79,12 @@ rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/cmethopt
 rm /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/arch
 # since we installed packages directly from the .deb files, Priorities differ
 # thus we first check for equality and then remove the files
-chroot /tmp/debian-{{ DIST }}-debootstrap dpkg --list > /tmp/dpkg1
-chroot /tmp/debian-{{ DIST }}-mm dpkg --list > /tmp/dpkg2
+chroot /tmp/debian-{{ DIST }}-debootstrap dpkg --list >/tmp/dpkg1
+chroot /tmp/debian-{{ DIST }}-mm dpkg --list >/tmp/dpkg2
 diff -u /tmp/dpkg1 /tmp/dpkg2 >&2
 rm /tmp/dpkg1 /tmp/dpkg2
-grep -v '^Priority: ' /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status > /tmp/status1
-grep -v '^Priority: ' /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status > /tmp/status2
+grep -v '^Priority: ' /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status >/tmp/status1
+grep -v '^Priority: ' /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status >/tmp/status2
 diff -u /tmp/status1 /tmp/status2 >&2
 rm /tmp/status1 /tmp/status2
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status
@@ -96,6 +99,7 @@ fi
 find /tmp/debian-{{ DIST }}-debootstrap/run/ -mindepth 1 -maxdepth 1 ! -name lock -print0 | xargs --no-run-if-empty -0 rm -r
 # debootstrap doesn't clean apt
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_{{ DIST }}_main_binary-{{ HOSTARCH }}_Packages \
+  /tmp/debian-{{ DIST }}-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_{{ DIST }}_InRelease \
   /tmp/debian-{{ DIST }}-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_{{ DIST }}_Release \
   /tmp/debian-{{ DIST }}-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_{{ DIST }}_Release.gpg
 
@@ -105,11 +109,9 @@ if [ "{{ VARIANT }}" = "-" ]; then
   rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/systemd/catalog/database
   rm /tmp/debian-{{ DIST }}-mm/var/lib/systemd/catalog/database
 
+  case {{ DIST }} in oldstable | stable)
     cap=$(chroot /tmp/debian-{{ DIST }}-debootstrap /sbin/getcap /bin/ping)
     expected="/bin/ping cap_net_raw=ep"
-	if [ "{{ DIST }}" = oldstable ]; then
-		expected="/bin/ping = cap_net_raw+ep"
-	fi
     if [ "$cap" != "$expected" ]; then
       echo "expected bin/ping to have capabilities $expected" >&2
       echo "but debootstrap produced: $cap" >&2
@@ -121,7 +123,10 @@ if [ "{{ VARIANT }}" = "-" ]; then
       echo "but mmdebstrap produced: $cap" >&2
       exit 1
     fi
+    ;;
+  esac
 fi
+
 rm /tmp/debian-{{ DIST }}-mm/var/cache/apt/archives/lock
 rm /tmp/debian-{{ DIST }}-mm/var/lib/apt/extended_states
 rm /tmp/debian-{{ DIST }}-mm/var/lib/apt/lists/lock
@@ -147,56 +152,82 @@ done
 # Because of unreproducible uids (#969631) we created the _apt user ourselves
 # and because passwd is not Essential:yes we didn't use useradd. But newer
 # versions of adduser and shadow will create a different /etc/shadow
-if [ "{{ VARIANT }}" = "-" ]; then
-case {{ DIST }} in oldstable|stable)
-for f in shadow shadow-; do
+if [ "{{ VARIANT }}" = "-" ] && [ "{{ DIST}}" = oldstable ]; then
+  for f in shadow shadow-; do
     if grep -q '^_apt:!:' /tmp/debian-{{ DIST }}-debootstrap/etc/$f; then
       sed -i 's/^_apt:\*:\([^:]\+\):0:99999:7:::$/_apt:!:\1::::::/' /tmp/debian-{{ DIST }}-mm/etc/$f
     fi
-done;;
-esac
+  done
 fi
 
 for log in faillog lastlog; do
-	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/var/log/$log /tmp/debian-{{ DIST }}-mm/var/log/$log >&2;then
+  f1="/tmp/debian-{{ DIST }}-debootstrap/var/log/$log"
+  f2="/tmp/debian-{{ DIST }}-mm/var/log/$log"
+  # skip cmp if file is absent in both chroots
+  if [ ! -e "$f1" ] && [ ! -e "$f2" ]; then
+    continue
+  fi
+  if ! cmp "$f1" "$f2" >&2; then
     # if the files differ, make sure they are all zeroes
-		cmp -n "$(stat -c %s "/tmp/debian-{{ DIST }}-debootstrap/var/log/$log")" "/tmp/debian-{{ DIST }}-debootstrap/var/log/$log" /dev/zero >&2
-		cmp -n "$(stat -c %s "/tmp/debian-{{ DIST }}-mm/var/log/$log")" "/tmp/debian-{{ DIST }}-mm/var/log/$log" /dev/zero >&2
+    cmp -n "$(stat -c %s "$f1")" "$f1" /dev/zero >&2
+    cmp -n "$(stat -c %s "$f2")" "$f2" /dev/zero >&2
     # then delete them
-		rm /tmp/debian-{{ DIST }}-debootstrap/var/log/$log /tmp/debian-{{ DIST }}-mm/var/log/$log
+    rm "$f1" "$f2"
   fi
 done
 
-# the order in which systemd and cron get installed differ and thus the order
-# of lines in /etc/group and /etc/gshadow differs
 if [ "{{ VARIANT }}" = "-" ]; then
-	case {{ DIST }} in testing|unstable)
+  # the order in which systemd and cron get installed differ and thus the order
+  # of lines in /etc/group and /etc/gshadow differs
   for f in group group- gshadow gshadow-; do
-			cmp /tmp/debian-{{ DIST }}-mm/etc/$f /tmp/debian-{{ DIST }}-debootstrap/etc/$f 2>/dev/null && exit 1
     for d in mm debootstrap; do
-				sort /tmp/debian-{{ DIST }}-$d/etc/$f > /tmp/debian-{{ DIST }}-$d/etc/$f.bak
+      sort /tmp/debian-{{ DIST }}-$d/etc/$f >/tmp/debian-{{ DIST }}-$d/etc/$f.bak
       mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
     done
   done
-		;;
-	esac
+  # the order in which systemd and passwd get installed differ and thus
+  # the order of lines in /etc/shadow and /etc/shadow- differs
+  for f in shadow shadow-; do
+    for d in mm debootstrap; do
+      sort /tmp/debian-{{ DIST }}-$d/etc/$f >/tmp/debian-{{ DIST }}-$d/etc/$f.bak
+      mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
+    done
+  done
+  # and since the order was different, ignore the *- files
+  for f in shadow- passwd-; do
+    for d in mm debootstrap; do
+      rm /tmp/debian-{{ DIST }}-$d/etc/$f
+    done
+  done
 fi
 
-# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
-case {{ DIST }} in oldstable|stable)
-for f in shadow shadow-; do
-	if [ ! -e /tmp/debian-{{ DIST }}-mm/etc/$f ]; then
-		continue
-	fi
-	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/etc/$f /tmp/debian-{{ DIST }}-mm/etc/$f >&2; then
-		echo patching /etc/$f on {{ DIST }} {{ VARIANT }} >&2
-		awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-{{ DIST }}-mm/etc/$f > /tmp/debian-{{ DIST }}-mm/etc/$f.bak
-		cat /tmp/debian-{{ DIST }}-mm/etc/$f.bak > /tmp/debian-{{ DIST }}-mm/etc/$f
-		rm /tmp/debian-{{ DIST }}-mm/etc/$f.bak
-	else
-		echo no difference for /etc/$f on {{ DIST }} {{ VARIANT }} >&2
-	fi
-done;;
+# since debootstrap 1.0.133 there is no tzdata in the buildd variant and thus
+# debootstrap creates its own /etc/localtime
+if [ "{{ VARIANT }}" = "buildd" ] && [ "{{ DIST }}" != "stable" ] && [ "{{ DIST }}" != "oldstable" ]; then
+  [ "$(readlink /tmp/debian-{{ DIST }}-debootstrap/etc/localtime)" = /usr/share/zoneinfo/UTC ]
+  rm /tmp/debian-{{ DIST }}-debootstrap/etc/localtime
+fi
+
+# starting with systemd 255 upstream dropped splitusr support and depending on
+# the installation order, symlink targets are prefixed with /usr or not
+# See #1060000 and #1054137
+case {{ DIST }} in testing | unstable)
+  for f in multi-user.target.wants/e2scrub_reap.service timers.target.wants/apt-daily-upgrade.timer timers.target.wants/apt-daily.timer timers.target.wants/e2scrub_all.timer; do
+    for d in mm debootstrap; do
+      [ -L "/tmp/debian-{{ DIST }}-$d/etc/systemd/system/$f" ] || continue
+      oldlink="$(readlink "/tmp/debian-{{ DIST }}-$d/etc/systemd/system/$f")"
+      case $oldlink in
+        /usr/*) : ;;
+        /*) oldlink="/usr$oldlink" ;;
+        *)
+          echo unexpected >&2
+          exit 1
+          ;;
+      esac
+      ln -sf "$oldlink" "/tmp/debian-{{ DIST }}-$d/etc/systemd/system/$f"
+    done
+  done
+  ;;
 esac
 
 # check if the file content differs
@@ -207,13 +238,14 @@ diff --unified --no-dereference --recursive /tmp/debian-{{ DIST }}-debootstrap /
 find /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm -type d -print0 | xargs -0 touch --date="@{{ SOURCE_DATE_EPOCH }}"
 # debootstrap never ran apt -- fixing permissions
 for d in ./var/lib/apt/lists/partial ./var/cache/apt/archives/partial; do
-	chroot /tmp/debian-{{ DIST }}-debootstrap chmod 0700 $d
-	chroot /tmp/debian-{{ DIST }}-debootstrap chown "$(id -u _apt):root" $d
+  unmergedPATH="$PATH$(if [ "{{ DIST }}" = oldstable ]; then echo :/bin:/sbin; fi)"
+  PATH="$unmergedPATH" chroot /tmp/debian-{{ DIST }}-debootstrap chmod 0700 $d
+  PATH="$unmergedPATH" chroot /tmp/debian-{{ DIST }}-debootstrap chown "$(id -u _apt):root" $d
 done
 tar -C /tmp/debian-{{ DIST }}-debootstrap --numeric-owner --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/root1.tar .
 tar -C /tmp/debian-{{ DIST }}-mm --numeric-owner --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/root2.tar .
-tar --full-time --verbose -tf /tmp/root1.tar > /tmp/root1.tar.list
-tar --full-time --verbose -tf /tmp/root2.tar > /tmp/root2.tar.list
+tar --full-time --verbose -tf /tmp/root1.tar >/tmp/root1.tar.list
+tar --full-time --verbose -tf /tmp/root2.tar >/tmp/root2.tar.list
 diff -u /tmp/root1.tar.list /tmp/root2.tar.list >&2
 rm /tmp/root1.tar /tmp/root2.tar /tmp/root1.tar.list /tmp/root2.tar.list
 

tests/check-for-bit-by-bit-identical-format-output

@@ -1,37 +1,28 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+
+trap "rm -f /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }}" EXIT INT TERM
+
+case {{ MODE }} in unshare | fakechroot) : ;; *) exit 1 ;; esac
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+    if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
+    fi
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
-export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-{{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-root.{{ FORMAT }} {{ MIRROR }}
-if [ "{{ FORMAT }}" = tar ]; then
-	printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 /tmp/debian-chroot-root.tar -
-elif [ "{{ FORMAT }}" = squashfs ]; then
-	printf 'hsqs' | cmp --bytes=4 /tmp/debian-chroot-root.squashfs -
-elif [ "{{ FORMAT }}" = ext2 ]; then
-	printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 /tmp/debian-chroot-root.ext2 -
-else
-	echo "unknown format: {{ FORMAT }}" >&2
-	exit 1
-fi
-runuser -u user -- {{ CMD }} --mode=unshare --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-unshare.{{ FORMAT }} {{ MIRROR }}
-cmp /tmp/debian-chroot-root.{{ FORMAT }} /tmp/debian-chroot-unshare.{{ FORMAT }}
-rm /tmp/debian-chroot-unshare.{{ FORMAT }}
-case {{ VARIANT }} in essential|apt|minbase|buildd)
-	# variants important and standard differ because permissions drwxr-sr-x
-	# and extended attributes of ./var/log/journal/ cannot be preserved
-	# in fakechroot mode
-	runuser -u user -- {{ CMD }} --mode=fakechroot --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-fakechroot.{{ FORMAT }} {{ MIRROR }}
-	cmp /tmp/debian-chroot-root.{{ FORMAT }} /tmp/debian-chroot-fakechroot.{{ FORMAT }}
-	rm /tmp/debian-chroot-fakechroot.{{ FORMAT }}
-	;;
-esac
+
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} {{ MIRROR }}
+cmp ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} \
+  || diffoscope ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }}
+
 # we cannot test chrootless mode here, because mmdebstrap relies on the
 # usrmerge package to set up merged-/usr and that doesn't work in chrootless
 # mode
-rm /tmp/debian-chroot-root.{{ FORMAT }}

tests/chrootless

@@ -2,19 +2,15 @@
 set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-if dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libpam-runtime)" le 1.5.2-5; then
-	# https://bugs.debian.org/1022952
-	exit 0
-fi
 trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
 # we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
 # DPKG_ROOT
-for INCLUDE in '' 'systemd-sysv'; do
+for INCLUDE in '' 'apt' 'apt,build-essential' 'systemd-sysv'; do
   for MODE in root chrootless; do
     {{ CMD }} --mode=$MODE --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
-			${INCLUDE:+--include="$INCLUDE"} \
+      ${INCLUDE:+--include="$INCLUDE"} --skip=check/chrootless \
       {{ DIST }} "/tmp/$MODE.tar" {{ MIRROR }}
   done
-	cmp /tmp/root.tar /tmp/chrootless.tar
+  cmp /tmp/root.tar /tmp/chrootless.tar || diffoscope /tmp/root.tar /tmp/chrootless.tar
   rm /tmp/chrootless.tar /tmp/root.tar
 done

tests/chrootless-fakeroot

@@ -3,27 +3,41 @@ set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
+[ {{ MODE }} = chrootless ]
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
     if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
     fi
-	adduser --gecos user --disabled-password user
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+
+MMTARFILTER=
+[ -x /usr/bin/mmtarfilter ] && MMTARFILTER=/usr/bin/mmtarfilter
+[ -x ./tarfilter ] && MMTARFILTER=./tarfilter
+
 # we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
 # DPKG_ROOT
 # permissions drwxr-sr-x and extended attributes of ./var/log/journal/ cannot
 # be preserved under fakeroot
-for INCLUDE in '' 'systemd-sysv'; do
+# this applies to 'z' lines in files in /usr/lib/tmpfiles.d/
+for INCLUDE in '' 'apt' 'apt,build-essential' 'systemd-sysv'; do
   {{ CMD }} --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
-		--customize-hook='if [ -d "$1"/var/log/journal ]; then rmdir "$1"/var/log/journal; mkdir --mode=2755 "$1"/var/log/journal; chroot "$1" chown root:systemd-journal /var/log/journal; fi' \
     ${INCLUDE:+--include="$INCLUDE"} \
-		{{ DIST }} /tmp/root.tar {{ MIRROR }}
-	$prefix fakeroot {{ CMD }} --mode=chrootless --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
+    {{ DIST }} - {{ MIRROR }} \
+    | "$MMTARFILTER" --path-exclude="/var/log/journal" --path-exclude="/etc/credstore*" \
+      >/tmp/root.tar
+  $prefix fakeroot {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
     ${INCLUDE:+--include="$INCLUDE"} \
-		{{ DIST }} /tmp/chrootless.tar {{ MIRROR }}
-	cmp /tmp/root.tar /tmp/chrootless.tar
+    {{ DIST }} - {{ MIRROR }} \
+    | "$MMTARFILTER" --path-exclude="/var/log/journal" --path-exclude="/etc/credstore*" \
+      >/tmp/chrootless.tar
+  cmp /tmp/root.tar /tmp/chrootless.tar || diffoscope /tmp/root.tar /tmp/chrootless.tar
   rm /tmp/chrootless.tar /tmp/root.tar
 done

tests/chrootless-foreign

@@ -8,22 +8,43 @@ if [ ! -e /mmdebstrap-testenv ]; then
   exit 1
 fi
 
+deb2qemu() {
+  case "$1" in
+    amd64) echo x86_64 ;;
+    arm64) echo aarch64 ;;
+    armel | armhf) echo arm ;;
+    ppc64el) echo ppc64le ;;
+    *) echo "$1" ;;
+  esac
+}
+if [ "$(dpkg --print-architecture)" = "arm64" ]; then
+  arch=amd64
+else
+  arch=arm64
+fi
+
 [ "$(id -u)" -eq 0 ]
-[ -e /proc/sys/fs/binfmt_misc/qemu-aarch64 ]
+[ -e "/proc/sys/fs/binfmt_misc/qemu-$(deb2qemu "$arch")" ]
 
 # we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
 # DPKG_ROOT
-for INCLUDE in '' 'systemd-sysv'; do
-	echo 1 > /proc/sys/fs/binfmt_misc/qemu-aarch64
-	arch-test arm64
-	{{ CMD }} --mode=root --architecture=arm64 --variant={{ VARIANT }} \
+#
+# dpkg is unable to install architecture arch:all packages with a
+# dependency on an arch:any package (perl-modules-5.34 in this case)
+# inside foreign architecture chrootless chroots, because dpkg will use
+# its own architecture as the native architecture, see #825385 and #1020533
+# So we are not testing the installation of apt,build-essential here.
+for INCLUDE in '' 'apt' 'systemd-sysv'; do
+  echo 1 >"/proc/sys/fs/binfmt_misc/qemu-$(deb2qemu "$arch")"
+  arch-test "$arch"
+  {{ CMD }} --mode=root --architecture="$arch" --variant={{ VARIANT }} \
     --hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
     {{ DIST }} "/tmp/root.tar" {{ MIRROR }}
-	echo 0 > /proc/sys/fs/binfmt_misc/qemu-aarch64
-	arch-test arm64 && exit 1
-	{{ CMD }} --mode=chrootless --architecture=arm64 --variant={{ VARIANT }} \
+  echo 0 >"/proc/sys/fs/binfmt_misc/qemu-$(deb2qemu "$arch")"
+  arch-test "$arch" && exit 1
+  {{ CMD }} --mode=chrootless --architecture="$arch" --variant={{ VARIANT }} \
     --hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
-		{{ DIST }} "/tmp/chrootless.tar" {{ MIRROR }}
+    --skip=check/chrootless {{ DIST }} "/tmp/chrootless.tar" {{ MIRROR }}
   # when creating a foreign architecture chroot, the tarballs are not
   # bit-by-bit identical but contain a few remaining differences:
   #
@@ -31,16 +52,15 @@ for INCLUDE in '' 'systemd-sysv'; do
   #  * /var/lib/dpkg/triggers -- #990712
   #  * /var/cache/debconf/*.dat-old -- needs investigation
   for tar in root chrootless; do
-		<"/tmp/$tar.tar" \
-		./tarfilter \
+    ./tarfilter <"/tmp/$tar.tar" \
       --path-exclude=/var/cache/debconf/config.dat-old \
       --path-exclude=/var/cache/debconf/templates.dat-old \
       --path-exclude=/etc/ld.so.cache \
       --path-exclude=/var/lib/dpkg/triggers/File \
       --path-exclude=/var/lib/dpkg/triggers/ldconfig \
-			> "/tmp/$tar.tar.tmp"
+      >"/tmp/$tar.tar.tmp"
     mv "/tmp/$tar.tar.tmp" "/tmp/$tar.tar"
   done
-	cmp /tmp/root.tar /tmp/chrootless.tar
+  cmp /tmp/root.tar /tmp/chrootless.tar || diffoscope /tmp/root.tar /tmp/chrootless.tar
   rm /tmp/chrootless.tar /tmp/root.tar
 done

tests/compare-output-with-pre-seeded-var-cache-apt-archives

@@ -21,7 +21,7 @@ fi
   --setup-hook='mkdir -p "$1"/var/cache/apt/archives/partial' \
   --setup-hook='touch "$1"/var/cache/apt/archives/lock' \
   --setup-hook='chmod 0640 "$1"/var/cache/apt/archives/lock' \
-	{{ DIST }} - {{ MIRROR }} > /tmp/orig.tar
+  {{ DIST }} - {{ MIRROR }} >/tmp/orig.tar
 # somehow, when trying to create a tarball from the 9p mount, tar throws the
 # following error: tar: ./doc-debian_6.4_all.deb: File shrank by 132942 bytes; padding with zeros
 # to reproduce, try: tar --directory /mnt/cache/debian/pool/main/d/doc-debian/ --create --file - . | tar --directory /tmp/ --extract --file -
@@ -32,7 +32,7 @@ cp /mnt/cache/debian/pool/main/b/busybox/busybox_*"_{{ HOSTARCH }}.deb" /mnt/cac
 {{ CMD }} $include --mode={{ MODE }} --variant={{ VARIANT }} \
   --setup-hook='mkdir -p "$1"/var/cache/apt/archives/partial' \
   --setup-hook='sync-in "'"$tmpdir"'" /var/cache/apt/archives/partial' \
-	{{ DIST }} - {{ MIRROR }} > /tmp/test1.tar
+  {{ DIST }} - {{ MIRROR }} >/tmp/test1.tar
 cmp /tmp/orig.tar /tmp/test1.tar
 {{ CMD }} $include --mode={{ MODE }} --variant={{ VARIANT }} \
   --customize-hook='touch "$1"/var/cache/apt/archives/partial' \
@@ -40,5 +40,5 @@ cmp /tmp/orig.tar /tmp/test1.tar
   --setup-hook='sync-in "'"$tmpdir"'" /var/cache/apt/archives/' \
   --setup-hook='chmod 0755 "$1"/var/cache/apt/archives/' \
   --customize-hook='find "'"$tmpdir"'" -type f -exec md5sum "{}" \; | sed "s|"'"$tmpdir"'"|$1/var/cache/apt/archives|" | md5sum --check' \
-	{{ DIST }} - {{ MIRROR }} > /tmp/test2.tar
+  {{ DIST }} - {{ MIRROR }} >/tmp/test2.tar
 cmp /tmp/orig.tar /tmp/test2.tar

tests/create-arm64-tarball

@@ -1,68 +0,0 @@
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	if [ ! -e /mmdebstrap-testenv ]; then
-		echo "this test modifies the system and should only be run inside a container" >&2
-		exit 1
-	fi
-	adduser --gecos user --disabled-password user
-fi
-if [ "{{ MODE }}" = unshare ]; then
-	if [ ! -e /mmdebstrap-testenv ]; then
-		echo "this test modifies the system and should only be run inside a container" >&2
-		exit 1
-	fi
-	sysctl -w kernel.unprivileged_userns_clone=1
-fi
-prefix=
-[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
-[ "{{ MODE }}" = "fakechroot" ] && prefix="$prefix fakechroot fakeroot"
-$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --architectures=arm64 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-# we ignore differences between architectures by ignoring some files
-# and renaming others
-{ tar -tf /tmp/debian-chroot.tar \
-	| grep -v '^\./usr/lib/ld-linux-aarch64\.so\.1$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/ld-linux-aarch64\.so\.1$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/int-ll64\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/types\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/unistd\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/sigcontext\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/sve_context\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/types\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-extra\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-id\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-prregset\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/gnu/stubs-lp64\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/linux/types\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/sys/procfs\.ph$' \
-	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/sys/user\.ph$' \
-	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.arm64\.gz$' \
-	| sed 's/aarch64-linux-gnu/x86_64-linux-gnu/' \
-	| sed 's/arm64/amd64/';
-} | sort > tar2.txt
-{ < tar1.txt \
-	grep -v '^\./usr/bin/i386$' \
-	| grep -v '^\./usr/bin/x86_64$' \
-	| grep -v '^\./lib32$' \
-	| grep -v '^\./lib64$' \
-	| grep -v '^\./libx32$' \
-	| grep -v '^\./usr/lib32/$' \
-	| grep -v '^\./usr/libx32/$' \
-	| grep -v '^\./usr/lib64/$' \
-	| grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/ld-linux-x86-64\.so\.2$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/libmvec\.so\.1$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_32\.ph$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_64\.ph$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_x32\.ph$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_32\.ph$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_64\.ph$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_x32\.ph$' \
-	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/gnu/stubs-64\.ph$' \
-	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.amd64\.gz$' \
-	| grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
-	| grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$';
-} | sort | diff -u - tar2.txt >&2
-rm /tmp/debian-chroot.tar

tests/create-directory

@@ -1,7 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
 {{ CMD }} --mode=root --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-chroot /tmp/debian-chroot dpkg-query --showformat '${binary:Package}\n' --show > pkglist.txt
-tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort > tar1.txt
-rm -r /tmp/debian-chroot
+chroot /tmp/debian-chroot dpkg-query --showformat '${binary:Package}\n' --show >pkglist.txt
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort >tar1.txt

tests/create-foreign-tarball

@@ -0,0 +1,78 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+    if [ ! -e /mmdebstrap-testenv ]; then
+      echo "this test modifies the system and should only be run inside a container" >&2
+      exit 1
+    fi
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
+fi
+
+case "$(dpkg --print-architecture)" in
+  arm64)
+    native_arch=arm64
+    native_gnu=aarch64-linux-gnu
+    foreign_arch=amd64
+    foreign_gnu=x86_64-linux-gnu
+    ;;
+  amd64)
+    native_arch=amd64
+    native_gnu=x86_64-linux-gnu
+    foreign_arch=arm64
+    foreign_gnu=aarch64-linux-gnu
+    ;;
+  *)
+    echo "unsupported native architecture" >&2
+    exit 1
+    ;;
+esac
+
+[ "{{ MODE }}" = "fakechroot" ] && prefix="$prefix fakechroot fakeroot"
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --architectures="$foreign_arch" \
+  {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+# we ignore differences between architectures by ignoring some files
+# and renaming others
+{
+  tar -tf /tmp/debian-chroot.tar \
+    | grep -v '^\./usr/bin/i386$' \
+    | grep -v '^\./usr/bin/x86_64$' \
+    | grep -v '^\./lib64$' \
+    | grep -v '^\./usr/lib64/$' \
+    | grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
+    | grep -v '^\./usr/lib/ld-linux-aarch64\.so\.1$' \
+    | grep -v "^\\./usr/lib/$foreign_gnu/ld-linux-aarch64\\.so\\.1$" \
+    | grep -v "^\\./usr/lib/$foreign_gnu/ld-linux-x86-64\\.so\\.2$" \
+    | grep -v "^\\./usr/lib/$foreign_gnu/perl/5\\.[0-9][.0-9]\\+/.*\\.ph$" \
+    | grep -v "^\\./usr/lib/$foreign_gnu/libmvec\\.so\\.1$" \
+    | grep -v "^\\./usr/share/doc/[^/]\\+/changelog\\(\\.Debian\\)\\?\\.$foreign_arch\\.gz$" \
+    | grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
+    | grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$' \
+    | sed "s/$foreign_gnu/$native_gnu/" \
+    | sed "s/$foreign_arch/$native_arch/"
+} | sort >/tmp/tar2.txt
+{
+  grep <tar1.txt -v '^\./usr/bin/i386$' \
+    | grep -v '^\./usr/bin/x86_64$' \
+    | grep -v '^\./lib32$' \
+    | grep -v '^\./lib64$' \
+    | grep -v '^\./libx32$' \
+    | grep -v '^\./usr/lib32/$' \
+    | grep -v '^\./usr/libx32/$' \
+    | grep -v '^\./usr/lib64/$' \
+    | grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
+    | grep -v '^\./usr/lib/ld-linux-aarch64\.so\.1$' \
+    | grep -v "^\\./usr/lib/$native_gnu/ld-linux-x86-64\\.so\\.2$" \
+    | grep -v "^\\./usr/lib/$native_gnu/ld-linux-aarch64\\.so\\.1$" \
+    | grep -v "^\\./usr/lib/$native_gnu/libmvec\\.so\\.1$" \
+    | grep -v "^\\./usr/lib/$native_gnu/perl/5\\.[0-9][.0-9]\\+/.*\\.ph$" \
+    | grep -v "^\\./usr/share/doc/[^/]\\+/changelog\\(\\.Debian\\)\\?\\.$native_arch\\.gz$" \
+    | grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
+    | grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$'
+} | sort | diff -u - /tmp/tar2.txt >&2
+rm /tmp/debian-chroot.tar /tmp/tar2.txt

tests/create-gzip-compressed-tarball

@@ -1,13 +1,20 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+    if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
+    fi
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
-runuser -u user -- {{ CMD }} --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot.tar.gz {{ MIRROR }}
+
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar.gz {{ MIRROR }}
 printf '\037\213\010' | cmp --bytes=3 /tmp/debian-chroot.tar.gz -
 tar -tf /tmp/debian-chroot.tar.gz | sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar.gz

tests/create-tarball-dry-run

@@ -8,24 +8,16 @@ export LC_ALL=C.UTF-8
 prefix=
 include=,
 if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != root ] && [ "{{ MODE }}" != auto ]; then
-	# this must be qemu
-	if ! id -u user >/dev/null 2>&1; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
     if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
     fi
-		adduser --gecos user --disabled-password user
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
   fi
-	if [ "{{ MODE }}" = unshare ]; then
-		if [ ! -e /mmdebstrap-testenv ]; then
-			echo "this test modifies the system and should only be run inside a container" >&2
-			exit 1
-		fi
-		sysctl -w kernel.unprivileged_userns_clone=1
-	fi
-	prefix="runuser -u user --"
+  prefix="runuser -u ${SUDO_USER:-user} --"
   if [ "{{ VARIANT }}" = extract ] || [ "{{ VARIANT }}" = custom ]; then
-		include="$(tr '\n' ',' < pkglist.txt)"
+    include="$(tr '\n' ',' <pkglist.txt)"
   fi
 fi
 $prefix {{ CMD }} --mode={{ MODE }} --include="$include" --dry-run --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}

tests/custom-tmpdir

@@ -1,26 +1,33 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
+
+[ "$(id -u)" -eq 0 ]
+[ {{ MODE }} = "unshare" ]
+
+if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+  if [ ! -e /mmdebstrap-testenv ]; then
     echo "this test modifies the system and should only be run inside a container" >&2
     exit 1
+  fi
+  useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 fi
+prefix="runuser -u ${SUDO_USER:-user} --"
+
 # https://www.etalabs.net/sh_tricks.html
-quote () { printf %s\\n "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" ; }
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
-homedir=$(runuser -u user -- sh -c 'cd && pwd')
+quote() { printf %s\\n "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+homedir=$($prefix sh -c 'cd && pwd')
 # apt:test/integration/test-apt-key
-TMPDIR_ADD="This is fü\$\$ing cràzy, \$(apt -v)\$!"
-runuser -u user -- mkdir "$homedir/$TMPDIR_ADD"
+TMPDIR_ADD='This is fü$$ing cràzy, $(apt -v)$!'
+$prefix mkdir "$homedir/$TMPDIR_ADD"
 # make sure the unshared user can traverse into the TMPDIR
 chmod 711 "$homedir"
 # set permissions and sticky bit like the real /tmp
 chmod 1777 "$homedir/$TMPDIR_ADD"
-runuser -u user -- env TMPDIR="$homedir/$TMPDIR_ADD" {{ CMD }} --mode=unshare --variant=apt \
+$prefix env TMPDIR="$homedir/$TMPDIR_ADD" {{ CMD }} --mode={{ MODE }} --variant=apt \
   --setup-hook='case "$1" in '"$(quote "$homedir/$TMPDIR_ADD/mmdebstrap.")"'??????????) exit 0;; *) echo "$1"; exit 1;; esac' \
   {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
 # use rmdir as a quick check that nothing is remaining in TMPDIR
-runuser -u user -- rmdir "$homedir/$TMPDIR_ADD"
+$prefix rmdir "$homedir/$TMPDIR_ADD"
 rm /tmp/debian-chroot.tar

tests/customize-hook

@@ -2,7 +2,7 @@
 set -eu
 export LC_ALL=C.UTF-8
 trap "rm -rf /tmp/debian-chroot; rm -f /tmp/customize.sh" EXIT INT TERM
-cat << 'SCRIPT' > /tmp/customize.sh
+cat <<'SCRIPT' >/tmp/customize.sh
 #!/bin/sh
 chroot "$1" whoami > "$1/output2"
 chroot "$1" pwd >> "$1/output2"

tests/cwd-directory-not-accessible-by-unshared-user

@@ -1,22 +1,30 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
+
+[ "$(id -u)" -eq 0 ]
+[ {{ MODE }} = "unshare" ]
+
+if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+  if [ ! -e /mmdebstrap-testenv ]; then
     echo "this test modifies the system and should only be run inside a container" >&2
     exit 1
+  fi
+  useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+prefix="runuser -u ${SUDO_USER:-user} --"
+
 mkdir /tmp/debian-chroot
 chmod 700 /tmp/debian-chroot
-chown user:user /tmp/debian-chroot
+chown "${SUDO_USER:-user}:${SUDO_USER:-user}" /tmp/debian-chroot
+set -- env --chdir=/tmp/debian-chroot
 if [ "{{ CMD }}" = "./mmdebstrap" ]; then
-	set -- "$(realpath --canonicalize-existing ./mmdebstrap)"
+  set -- "$@" "$(realpath --canonicalize-existing ./mmdebstrap)"
 elif [ "{{ CMD }}" = "perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap" ]; then
-	set -- perl -MDevel::Cover=-silent,-nogcov "$(realpath --canonicalize-existing ./mmdebstrap)"
+  set -- "$@" perl -MDevel::Cover=-silent,-nogcov "$(realpath --canonicalize-existing ./mmdebstrap)"
 else
-	set -- {{ CMD }}
+  set -- "$@" {{ CMD }}
 fi
-env --chdir=/tmp/debian-chroot runuser -u user -- "$@" --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+$prefix "$@" --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar

tests/deb822-1-2

@@ -2,13 +2,13 @@
 set -eu
 export LC_ALL=C.UTF-8
 trap "rm -rf /tmp/debian-chroot; rm -f /tmp/sources.list /tmp/deb822.sources" EXIT INT TERM
-cat << SOURCES > /tmp/deb822.sources
+cat <<SOURCES >/tmp/deb822.sources
 Types: deb
 URIs: {{ MIRROR }}1
 Suites: {{ DIST }}
 Components: main
 SOURCES
-echo "deb {{ MIRROR }}2 {{ DIST }} main" > /tmp/sources.list
+echo "deb {{ MIRROR }}2 {{ DIST }} main" >/tmp/sources.list
 echo "deb {{ MIRROR }}3 {{ DIST }} main" \
   | {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} \
     /tmp/debian-chroot \
@@ -19,13 +19,13 @@ echo "deb {{ MIRROR }}3 {{ DIST }} main" \
     {{ MIRROR }}6 \
     /tmp/sources.list
 test ! -e /tmp/debian-chroot/etc/apt/sources.list
-cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0000deb822.sources -
+cat <<SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0000deb822.sources -
 Types: deb
 URIs: {{ MIRROR }}1
 Suites: {{ DIST }}
 Components: main
 SOURCES
-cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0001main.list -
+cat <<SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0001main.list -
 deb {{ MIRROR }}4 {{ DIST }} main
 
 deb {{ MIRROR }}3 {{ DIST }} main
@@ -40,6 +40,6 @@ tar -C /tmp/debian-chroot --one-file-system -c . \
     tar -t \
       | grep -v "^./etc/apt/sources.list.d/0000deb822.sources$" \
       | grep -v "^./etc/apt/sources.list.d/0001main.list$" \
-			| grep -v "^./etc/apt/sources.list.d/0002sources.list";
-		printf "./etc/apt/sources.list\n";
+      | grep -v "^./etc/apt/sources.list.d/0002sources.list"
+    printf "./etc/apt/sources.list\n"
   } | sort | diff -u tar1.txt -

tests/deb822-2-2

@@ -2,14 +2,14 @@
 set -eu
 export LC_ALL=C.UTF-8
 trap "rm -rf /tmp/debian-chroot; rm -f /tmp/sources /tmp/deb822" EXIT INT TERM
-cat << SOURCES > /tmp/deb822
+cat <<SOURCES >/tmp/deb822
 Types: deb
 URIs: {{ MIRROR }}1
 Suites: {{ DIST }}
 Components: main
 SOURCES
-echo "deb {{ MIRROR }}2 {{ DIST }} main" > /tmp/sources
-cat << SOURCES | {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} \
+echo "deb {{ MIRROR }}2 {{ DIST }} main" >/tmp/sources
+cat <<SOURCES | {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} \
   /tmp/debian-chroot \
   /tmp/deb822 \
   - \
@@ -21,13 +21,13 @@ Components: main
 SOURCES
 test ! -e /tmp/debian-chroot/etc/apt/sources.list
 ls -lha /tmp/debian-chroot/etc/apt/sources.list.d/
-cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0000deb822.sources -
+cat <<SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0000deb822.sources -
 Types: deb
 URIs: {{ MIRROR }}1
 Suites: {{ DIST }}
 Components: main
 SOURCES
-cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0001main.sources -
+cat <<SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0001main.sources -
 Types: deb
 URIs: {{ MIRROR }}3
 Suites: {{ DIST }}
@@ -39,6 +39,6 @@ tar -C /tmp/debian-chroot --one-file-system -c . \
     tar -t \
       | grep -v "^./etc/apt/sources.list.d/0000deb822.sources$" \
       | grep -v "^./etc/apt/sources.list.d/0001main.sources$" \
-			| grep -v "^./etc/apt/sources.list.d/0002sources.list$";
-		printf "./etc/apt/sources.list\n";
+      | grep -v "^./etc/apt/sources.list.d/0002sources.list$"
+    printf "./etc/apt/sources.list\n"
   } | sort | diff -u tar1.txt -

tests/debootstrap

@@ -0,0 +1,16 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+
+tmpdir="$(mktemp -d)"
+chmod 755 "$tmpdir"
+ret=0
+debootstrap "$([ "{{ DIST }}" = oldstable ] && echo --no-merged-usr || echo --merged-usr)" --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }} || ret=$?
+if [ "$ret" -ne 0 ]; then
+  echo "E: debootstrap failed, dumping $tmpdir/debootstrap/debootstrap.log"
+  cat "$tmpdir/debootstrap/debootstrap.log"
+  exit 1
+fi
+tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system --xattrs -C "$tmpdir" -c . >"./cache/debian-{{ DIST }}-{{ VARIANT }}.tar"
+rm -r "$tmpdir"

tests/debug

@@ -1,6 +1,15 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-{{ CMD }} --mode=root --variant=apt --debug {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
-rm -r /tmp/debian-chroot
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+
+# we use variant standard in verbose mode to see the maximum number of packages
+# that was chosen in case of USE_HOST_APT_CONFIG=yes
+case {{ VARIANT }} in standard) : ;; *) exit 1 ;; esac
+
+{{ CMD }} --variant={{ VARIANT }} --debug {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+
+cmp ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar /tmp/debian-chroot.tar \
+  || diffoscope ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar /tmp/debian-chroot.tar

tests/dev-ptmx

@@ -7,18 +7,20 @@ if [ {{ MODE }} != unshare ] && [ {{ MODE }} != root ]; then
   exit 1
 fi
 
-if [ ! -e /mmdebstrap-testenv ]; then
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+    if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
+    fi
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	adduser --gecos user --disabled-password user
-fi
-prefix=
-[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
 
 # this mimics what apt does in apt-pkg/deb/dpkgpm.cc/pkgDPkgPM::StartPtyMagic()
-cat > /tmp/test.c << 'END'
+cat >/tmp/test.c <<'END'
 #define _GNU_SOURCE
 
 #include <stdlib.h>
@@ -116,19 +118,20 @@ END
 
 # use script to create a fake tty
 # run all tests as root and as a normal user (the latter requires ptmxmode=666)
-script -qfc "$prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
-	--include=gcc,libc6-dev,python3,adduser \
-	--customize-hook='chroot \"\$1\" adduser --gecos user --disabled-password user' \
+script -qfec "$prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
+	--include=gcc,libc6-dev,python3,passwd \
+	--customize-hook='chroot \"\$1\" useradd --home-dir /home/user --create-home user' \
 	--customize-hook='chroot \"\$1\" python3 -c \"import pty; print(pty.openpty())\"' \
 	--customize-hook='chroot \"\$1\" runuser -u user -- python3 -c \"import pty; print(pty.openpty())\"' \
 	--customize-hook='chroot \"\$1\" script -c \"echo foobar\"' \
 	--customize-hook='chroot \"\$1\" runuser -u user -- env --chdir=/home/user script -c \"echo foobar\"' \
-	--customize-hook='chroot \"\$1\" apt-get install --yes doc-debian 2>&1 | tee /tmp/log' \
+	--customize-hook='chroot \"\$1\" apt-get install --yes doc-debian 2>&1 | tee \"\$1\"/tmp/log' \
 	--customize-hook=\"copy-in /tmp/test.c /tmp\" \
 	--customize-hook='chroot \"\$1\" gcc /tmp/test.c -o /tmp/test' \
 	--customize-hook='chroot \"\$1\" /tmp/test' \
 	--customize-hook='chroot \"\$1\" runuser -u user -- /tmp/test' \
 	--customize-hook='rm \"\$1\"/tmp/test \"\$1\"/tmp/test.c' \
+	--customize-hook=\"copy-out /tmp/log /tmp\" \
 	{{ DIST }} /dev/null {{ MIRROR }}" /dev/null
 
 fail=0

tests/dist-using-codename

@@ -4,8 +4,10 @@
 
 set -eu
 export LC_ALL=C.UTF-8
-trap "rm -f Release; rm -rf /tmp/debian-chroot" EXIT INT TERM
-/usr/lib/apt/apt-helper download-file "{{ MIRROR }}/dists/{{ DIST }}/Release" Release
-codename=$(awk '/^Codename: / { print $2; }' Release)
-{{ CMD }} --mode={{ MODE }} --variant=apt "$codename" /tmp/debian-chroot {{ MIRROR }}
-echo "deb {{ MIRROR }} $codename main" | diff -u - /tmp/debian-chroot/etc/apt/sources.list
+trap "rm -f InRelease; rm -rf /tmp/debian-chroot.tar /tmp/expected" EXIT INT TERM
+/usr/lib/apt/apt-helper download-file "{{ MIRROR }}/dists/{{ DIST }}/InRelease" InRelease
+codename=$(awk '/^Codename: / { print $2; }' InRelease)
+{{ CMD }} --mode={{ MODE }} --variant=apt "$codename" /tmp/debian-chroot.tar {{ MIRROR }}
+echo "deb {{ MIRROR }} $codename main" >/tmp/expected
+tar --to-stdout --extract --file /tmp/debian-chroot.tar ./etc/apt/sources.list \
+  | diff -u /tmp/expected -

tests/dpkgopt

@@ -2,9 +2,13 @@
 set -eu
 export LC_ALL=C.UTF-8
 trap "rm -rf /tmp/debian-chroot; rm -f /tmp/config" EXIT INT TERM
-echo no-pager > /tmp/config
+echo no-pager >/tmp/config
 {{ CMD }} --mode=root --variant=apt --dpkgopt="path-exclude=/usr/share/doc/*" --dpkgopt=/tmp/config --dpkgopt="path-include=/usr/share/doc/dpkg/copyright" {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 printf 'path-exclude=/usr/share/doc/*\nno-pager\npath-include=/usr/share/doc/dpkg/copyright\n' | cmp /tmp/debian-chroot/etc/dpkg/dpkg.cfg.d/99mmdebstrap -
 rm /tmp/debian-chroot/etc/dpkg/dpkg.cfg.d/99mmdebstrap
-tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort > tar2.txt
-{ grep -v '^./usr/share/doc/.' tar1.txt; echo ./usr/share/doc/dpkg/; echo ./usr/share/doc/dpkg/copyright; } | sort | diff -u - tar2.txt
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort >tar2.txt
+{
+  grep -v '^./usr/share/doc/.' tar1.txt
+  echo ./usr/share/doc/dpkg/
+  echo ./usr/share/doc/dpkg/copyright
+} | sort | diff -u - tar2.txt

tests/eatmydata-via-hook-dir

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-cat << SCRIPT > /tmp/checkeatmydata.sh
+cat <<SCRIPT >/tmp/checkeatmydata.sh
 #!/bin/sh
 set -exu
 cat << EOF | diff - "\$1"/usr/bin/dpkg
@@ -12,18 +12,24 @@ EOF
 SCRIPT
 chmod +x /tmp/checkeatmydata.sh
 # first four bytes: magic
-elfheader="\\177ELF"
+elfheader='\177ELF'
 # fifth byte: bits
 case "$(dpkg-architecture -qDEB_HOST_ARCH_BITS)" in
-	32) elfheader="$elfheader\\001";;
-	64) elfheader="$elfheader\\002";;
-	*) echo "bits not supported"; exit 1;;
+  32) elfheader="$elfheader\\001" ;;
+  64) elfheader="$elfheader\\002" ;;
+  *)
+    echo "bits not supported"
+    exit 1
+    ;;
 esac
 # sixth byte: endian
 case "$(dpkg-architecture -qDEB_HOST_ARCH_ENDIAN)" in
-	little) elfheader="$elfheader\\001";;
-	big) elfheader="$elfheader\\002";;
-	*) echo "endian not supported"; exit 1;;
+  little) elfheader="$elfheader\\001" ;;
+  big) elfheader="$elfheader\\002" ;;
+  *)
+    echo "endian not supported"
+    exit 1
+    ;;
 esac
 # seventh and eigth byte: elf version (1) and abi (unset)
 elfheader="$elfheader\\001\\000"
@@ -34,6 +40,9 @@ elfheader="$elfheader\\001\\000"
   --hook-dir=./hooks/eatmydata \
   --customize-hook='printf "'"$elfheader"'" | cmp --bytes=8 - "$1"/usr/bin/dpkg' \
   {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+tar -C /tmp/debian-chroot --one-file-system -c . \
+  | tar -t \
+  | sort \
+  | diff -u tar1.txt -
 rm /tmp/checkeatmydata.sh
 rm -r /tmp/debian-chroot

tests/empty-sources.list

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+printf '' | {{ CMD }} --mode={{ MODE }} --variant=apt \
+  --setup-hook='test -e "$1"/etc/apt/sources.list || echo "deb {{ MIRROR }} {{ DIST }} main" > "$1"/etc/apt/sources.list' \
+  {{ DIST }} /tmp/debian-chroot.tar -
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/empty-suite

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+
+{{ CMD }} --variant={{ VARIANT }} --verbose \
+  --setup-hook='echo deb {{ MIRROR }} {{ DIST }} main >> "$1"/etc/apt/sources.list' \
+  '' /tmp/debian-chroot.tar
+
+cmp ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar /tmp/debian-chroot.tar \
+  || diffoscope ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar /tmp/debian-chroot.tar

tests/essential-hook

@@ -2,13 +2,13 @@
 set -eu
 export LC_ALL=C.UTF-8
 trap "rm -rf /tmp/debian-chroot; rm -f /tmp/essential.sh" EXIT INT TERM
-cat << 'SCRIPT' > /tmp/essential.sh
+cat <<'SCRIPT' >/tmp/essential.sh
 #!/bin/sh
 echo tzdata tzdata/Zones/Europe select Berlin | chroot "$1" debconf-set-selections
 SCRIPT
 chmod +x /tmp/essential.sh
 {{ CMD }} --mode=root --variant=apt --include=tzdata --essential-hook='echo tzdata tzdata/Areas select Europe | chroot "$1" debconf-set-selections' --essential-hook=/tmp/essential.sh {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-echo Europe/Berlin | cmp /tmp/debian-chroot/etc/timezone
+[ "$(readlink /tmp/debian-chroot/etc/localtime)" = "/usr/share/zoneinfo/Europe/Berlin" ]
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort \
   | grep -v '^./etc/localtime' \
   | grep -v '^./etc/timezone' \
@@ -17,5 +17,4 @@ tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort \
   | grep -v '^./usr/share/lintian/overrides/tzdata' \
   | grep -v '^./usr/share/zoneinfo' \
   | grep -v '^./var/lib/dpkg/info/tzdata.' \
-	| grep -v '^./var/lib/apt/extended_states$' \
   | diff -u tar1.txt -

tests/fail-installing-to-existing-file

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap "rm -f /tmp/exists" EXIT INT TERM
+
 touch /tmp/exists
 ret=0
 {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/exists {{ MIRROR }} || ret=$?

tests/fail-with-path-with-quotes

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap 'rm -rf /tmp/quoted\"path' EXIT INT TERM
+
 ret=0
 {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/quoted\"path {{ MIRROR }} || ret=$?
 if [ "$ret" = 0 ]; then

tests/fail-without-etc-subuid

@@ -5,8 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
   echo "this test modifies the system and should only be run inside a container" >&2
   exit 1
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+useradd --home-dir /home/user --create-home user
 rm /etc/subuid
 ret=0
 runuser -u user -- {{ CMD }} --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }} || ret=$?
@@ -14,4 +13,4 @@ if [ "$ret" = 0 ]; then
   echo expected failure but got exit $ret >&2
   exit 1
 fi
-rm -r /tmp/debian-chroot
+[ ! -e /tmp/debian-chroot ]

tests/fail-without-username-in-etc-subuid

@@ -5,9 +5,8 @@ if [ ! -e /mmdebstrap-testenv ]; then
   echo "this test modifies the system and should only be run inside a container" >&2
   exit 1
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
-awk -F: '$1!="user"' /etc/subuid > /etc/subuid.tmp
+useradd --home-dir /home/user --create-home user
+awk -F: '$1!="user"' /etc/subuid >/etc/subuid.tmp
 mv /etc/subuid.tmp /etc/subuid
 ret=0
 runuser -u user -- {{ CMD }} --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }} || ret=$?
@@ -15,4 +14,4 @@ if [ "$ret" = 0 ]; then
   echo expected failure but got exit $ret >&2
   exit 1
 fi
-rm -r /tmp/debian-chroot
+[ ! -e /tmp/debian-chroot ]

tests/file-mirror-automount-hook

@@ -5,8 +5,8 @@ if [ ! -e /mmdebstrap-testenv ]; then
   echo "this test requires the cache directory to be mounted on /mnt and should only be run inside a container" >&2
   exit 1
 fi
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	adduser --gecos user --disabled-password user
+if [ "$(id -u)" -eq 0 ] && ! id -u user >/dev/null 2>&1; then
+  useradd --home-dir /home/user --create-home user
 fi
 prefix=
 [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"

tests/hook-directory

@@ -4,12 +4,12 @@ export LC_ALL=C.UTF-8
 for h in hookA hookB; do
   mkdir /tmp/$h
   for s in setup extract essential customize; do
-		cat << SCRIPT > /tmp/$h/${s}00.sh
+    cat <<SCRIPT >/tmp/$h/${s}00.sh
 #!/bin/sh
 echo $h/${s}00 >> "\$1/$s"
 SCRIPT
     chmod +x /tmp/$h/${s}00.sh
-		cat << SCRIPT > /tmp/$h/${s}01.sh
+    cat <<SCRIPT >/tmp/$h/${s}01.sh
 echo $h/${s}01 >> "\$1/$s"
 SCRIPT
     chmod +x /tmp/$h/${s}01.sh

tests/i386-which-can-be-executed-without-qemu

@@ -6,11 +6,12 @@ if [ ! -e /mmdebstrap-testenv ]; then
   exit 1
 fi
 # remove qemu just to be sure
-apt-get remove --yes qemu-user-static binfmt-support qemu-user
+apt-get remove --yes qemu-user-binfmt binfmt-support qemu-user
 {{ CMD }} --mode={{ MODE }} --variant=apt --architectures=i386 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 # we ignore differences between architectures by ignoring some files
 # and renaming others
-{ tar -tf /tmp/debian-chroot.tar \
+{
+  tar -tf /tmp/debian-chroot.tar \
     | grep -v '^\./usr/bin/i386$' \
     | grep -v '^\./usr/lib/ld-linux\.so\.2$' \
     | grep -v '^\./usr/lib/i386-linux-gnu/ld-linux\.so\.2$' \
@@ -20,10 +21,10 @@ apt-get remove --yes qemu-user-static binfmt-support qemu-user
     | grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.i386\.gz$' \
     | sed 's/i386-linux-gnu/x86_64-linux-gnu/' \
     | sed 's/i386/amd64/' \
-	| sed 's/\/stubs-32.ph$/\/stubs-64.ph/';
-} | sort > tar2.txt
-{ < tar1.txt \
-	grep -v '^\./usr/bin/i386$' \
+    | sed 's/\/stubs-32.ph$/\/stubs-64.ph/'
+} | sort >tar2.txt
+{
+  grep <tar1.txt -v '^\./usr/bin/i386$' \
     | grep -v '^\./usr/bin/x86_64$' \
     | grep -v '^\./usr/lib32/$' \
     | grep -v '^\./lib32$' \
@@ -36,6 +37,6 @@ apt-get remove --yes qemu-user-static binfmt-support qemu-user
     | grep -v '^\./usr/lib/x86_64-linux-gnu/libmvec\.so\.1$' \
     | grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.amd64\.gz$' \
     | grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
-	| grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$';
+    | grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$'
 } | sort | diff -u - tar2.txt >&2
 rm /tmp/debian-chroot.tar

tests/include

@@ -3,10 +3,9 @@ set -eu
 export LC_ALL=C.UTF-8
 trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
 {{ CMD }} --mode=root --variant=apt --include=doc-debian {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
-rm /tmp/debian-chroot/var/lib/apt/extended_states
 rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.list
 rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.md5sums
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/include-deb-file

@@ -3,11 +3,25 @@
 set -eu
 export LC_ALL=C.UTF-8
 
+trap "rm -rf /tmp/dummypkg.deb /tmp/dummypkg" EXIT INT TERM
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+    if [ ! -e /mmdebstrap-testenv ]; then
+      echo "this test modifies the system and should only be run inside a container" >&2
+      exit 1
+    fi
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
+fi
+
 # instead of obtaining a .deb from our cache, we create a new package because
 # otherwise apt might decide to download the package with the same name and
 # version from the cache instead of using the local .deb
 mkdir -p /tmp/dummypkg/DEBIAN
-cat << END > "/tmp/dummypkg/DEBIAN/control"
+cat <<END >"/tmp/dummypkg/DEBIAN/control"
 Package: dummypkg
 Priority: optional
 Section: oldlibs
@@ -20,7 +34,7 @@ Description: dummypkg
 END
 dpkg-deb --build "/tmp/dummypkg" "/tmp/dummypkg.deb"
 
-{{ CMD }} --variant=apt --include="/tmp/dummypkg.deb" \
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include="/tmp/dummypkg.deb" \
   --hook-dir=./hooks/file-mirror-automount \
   --customize-hook='chroot "$1" dpkg-query -W -f="\${Status}\n" dummypkg | grep "^install ok installed$"' \
   {{ DIST }} /dev/null {{ MIRROR }}

tests/include-foreign-libmagic-mgc

@@ -6,20 +6,41 @@
 #   - installs only few files
 #   - doesn't change its name regularly (like gcc-*-base)
 
+case "$(dpkg --print-architecture)" in
+  arm64)
+    native_arch=arm64
+    foreign_arch=amd64
+    ;;
+  amd64)
+    native_arch=amd64
+    foreign_arch=arm64
+    ;;
+  *)
+    echo "unsupported native architecture" >&2
+    exit 1
+    ;;
+esac
+
 set -eu
 export LC_ALL=C.UTF-8
-{{ CMD }} --mode=root --variant=apt --architectures=amd64,arm64 --include=libmagic-mgc:arm64 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-{ echo "amd64"; echo "arm64"; } | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
-rm /tmp/debian-chroot/var/lib/apt/extended_states
-rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
-rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
+{{ CMD }} --mode=root --variant=apt \
+  --architectures="$native_arch,$foreign_arch" \
+  --include="libmagic-mgc:$foreign_arch" \
+  {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+{
+  echo "$native_arch"
+  echo "$foreign_arch"
+} | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
 rm /tmp/debian-chroot/usr/lib/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian
+rm -f /tmp/debian-chroot/usr/share/doc/libmagic-mgc/"changelog.Debian.$foreign_arch.gz"
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
 rm /tmp/debian-chroot/usr/share/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/misc/magic.mgc
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
 rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/
 rmdir /tmp/debian-chroot/usr/share/file/magic/
 rmdir /tmp/debian-chroot/usr/share/file/

tests/include-foreign-libmagic-mgc-with-multiple-arch-options

@@ -0,0 +1,50 @@
+#!/bin/sh
+#
+# to test foreign architecture package installation we choose a package which
+#   - is not part of the native installation set
+#   - does not have any dependencies
+#   - installs only few files
+#   - doesn't change its name regularly (like gcc-*-base)
+
+case "$(dpkg --print-architecture)" in
+  arm64)
+    native_arch=arm64
+    foreign_arch=amd64
+    ;;
+  amd64)
+    native_arch=amd64
+    foreign_arch=arm64
+    ;;
+  *)
+    echo "unsupported native architecture" >&2
+    exit 1
+    ;;
+esac
+
+set -eu
+export LC_ALL=C.UTF-8
+{{ CMD }} --mode=root --variant=apt \
+  --architectures="$native_arch" \
+  --architectures="$foreign_arch" \
+  --include="libmagic-mgc:$foreign_arch" \
+  {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+{
+  echo "$native_arch"
+  echo "$foreign_arch"
+} | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
+rm /tmp/debian-chroot/usr/lib/file/magic.mgc
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian
+rm -f /tmp/debian-chroot/usr/share/doc/libmagic-mgc/"changelog.Debian.$foreign_arch.gz"
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
+rm /tmp/debian-chroot/usr/share/file/magic.mgc
+rm /tmp/debian-chroot/usr/share/misc/magic.mgc
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
+rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/
+rmdir /tmp/debian-chroot/usr/share/file/magic/
+rmdir /tmp/debian-chroot/usr/share/file/
+rmdir /tmp/debian-chroot/usr/lib/file/
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+rm -r /tmp/debian-chroot

tests/include-libmagic-mgc-arm64-with-multiple-arch-options

@@ -1,21 +0,0 @@
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-{{ CMD }} --mode=root --variant=apt --architectures=amd64 --architectures=arm64 --include=libmagic-mgc:arm64 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-{ echo "amd64"; echo "arm64"; } | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
-rm /tmp/debian-chroot/var/lib/apt/extended_states
-rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
-rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
-rm /tmp/debian-chroot/usr/lib/file/magic.mgc
-rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian
-rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz
-rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
-rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
-rm /tmp/debian-chroot/usr/share/file/magic.mgc
-rm /tmp/debian-chroot/usr/share/misc/magic.mgc
-rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/
-rmdir /tmp/debian-chroot/usr/share/file/magic/
-rmdir /tmp/debian-chroot/usr/share/file/
-rmdir /tmp/debian-chroot/usr/lib/file/
-tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
-rm -r /tmp/debian-chroot

tests/install-busybox-based-sub-essential-system

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
 pkgs=base-files,base-passwd,busybox,debianutils,dpkg,libc-bin,mawk,tar
 # busybox --install -s will install symbolic links into the rootfs, leaving
 # existing files untouched. It has to run after extraction (otherwise there is
@@ -12,23 +15,27 @@ pkgs=base-files,base-passwd,busybox,debianutils,dpkg,libc-bin,mawk,tar
   --setup-hook='printf "root:x:0:\nmail:x:8:\nutmp:x:43:\n" > "$1/etc/group"' \
   --extract-hook='chroot "$1" busybox --install -s' \
   {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-echo "$pkgs" | tr ',' '\n' > /tmp/expected
+echo "$pkgs" | tr ',' '\n' >/tmp/expected
 chroot /tmp/debian-chroot dpkg-query -f '${binary:Package}\n' -W \
   | comm -12 - /tmp/expected \
   | diff -u - /tmp/expected
 rm /tmp/expected
 for cmd in echo cat sed grep; do
   test -L /tmp/debian-chroot/bin/$cmd
-	test "$(readlink /tmp/debian-chroot/bin/$cmd)" = "/bin/busybox"
+  test "$(readlink /tmp/debian-chroot/bin/$cmd)" = "/usr/bin/busybox"
 done
 for cmd in sort tee; do
   test -L /tmp/debian-chroot/usr/bin/$cmd
-	test "$(readlink /tmp/debian-chroot/usr/bin/$cmd)" = "/bin/busybox"
+  test "$(readlink /tmp/debian-chroot/usr/bin/$cmd)" = "/usr/bin/busybox"
 done
+
+# if /bin or /sbin are not symlinks, add /bin and /sbin to PATH
+if [ ! -L /tmp/debian-chroot/bin ] || [ ! -L /tmp/debian-chroot/sbin ]; then
+  export PATH="$PATH:/sbin:/bin"
+fi
 chroot /tmp/debian-chroot echo foobar \
   | chroot /tmp/debian-chroot cat \
   | chroot /tmp/debian-chroot sort \
   | chroot /tmp/debian-chroot tee /dev/null \
   | chroot /tmp/debian-chroot sed 's/foobar/blubber/' \
   | chroot /tmp/debian-chroot grep blubber >/dev/null
-rm -r /tmp/debian-chroot

tests/install-doc-debian

@@ -1,21 +1,30 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
+[ {{ VARIANT }} = "custom" ]
+[ {{ MODE }} = "chrootless" ]
+
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
     if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
     fi
-	adduser --gecos user --disabled-password user
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
-$prefix {{ CMD }} --mode=chrootless --variant=custom --include=doc-debian {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --include=doc-debian {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 tar -C /tmp/debian-chroot --owner=0 --group=0 --numeric-owner --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/debian-chroot.tar .
-tar tvf /tmp/debian-chroot.tar > doc-debian.tar.list
+tar tvf /tmp/debian-chroot.tar >doc-debian.tar.list
 rm /tmp/debian-chroot.tar
 # delete contents of doc-debian
-rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
 # delete real files

tests/install-doc-debian-and-output-tarball

@@ -2,15 +2,22 @@
 set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
+[ {{ VARIANT }} = "custom" ]
+[ {{ MODE }} = "chrootless" ]
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
     if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
     fi
-	adduser --gecos user --disabled-password user
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
-$prefix {{ CMD }} --mode=chrootless --variant=custom --include=doc-debian {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --include=doc-debian {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar tvf /tmp/debian-chroot.tar | grep -v ' ./dev' | diff -u doc-debian.tar.list -
 rm /tmp/debian-chroot.tar

tests/install-doc-debian-and-test-hooks

@@ -2,23 +2,32 @@
 set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
+[ {{ VARIANT }} = "custom" ]
+[ {{ MODE }} = "chrootless" ]
+
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
     if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
     fi
-	adduser --gecos user --disabled-password user
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
-$prefix {{ CMD }} --mode=chrootless --skip=cleanup/tmp --variant=custom --include=doc-debian --setup-hook='touch "$1/tmp/setup"' --customize-hook='touch "$1/tmp/customize"' {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+
+$prefix {{ CMD }} --mode={{ MODE }} --skip=cleanup/tmp --variant={{ VARIANT }} --include=doc-debian --setup-hook='touch "$1/tmp/setup"' --customize-hook='touch "$1/tmp/customize"' {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 rm /tmp/debian-chroot/tmp/setup
 rm /tmp/debian-chroot/tmp/customize
 tar -C /tmp/debian-chroot --owner=0 --group=0 --numeric-owner --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/debian-chroot.tar .
 tar tvf /tmp/debian-chroot.tar | grep -v ' ./dev' | diff -u doc-debian.tar.list -
 rm /tmp/debian-chroot.tar
 # delete contents of doc-debian
-rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
 # delete real files

tests/install-libmagic-mgc-on-foreign

@@ -1,22 +1,43 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
+[ {{ VARIANT }} = "custom" ]
+[ {{ MODE }} = "chrootless" ]
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
     if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
     fi
-	adduser --gecos user --disabled-password user
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
-$prefix {{ CMD }} --mode=chrootless --variant=custom --architectures=arm64 --include=libmagic-mgc {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+
+case "$(dpkg --print-architecture)" in
+  arm64)
+    foreign_arch=amd64
+    ;;
+  amd64)
+    foreign_arch=arm64
+    ;;
+  *)
+    echo "unsupported native architecture" >&2
+    exit 1
+    ;;
+esac
+
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --architectures="$foreign_arch" --include=libmagic-mgc {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 # delete contents of libmagic-mgc
 rm /tmp/debian-chroot/usr/lib/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
+rm -f /tmp/debian-chroot/usr/share/doc/libmagic-mgc/"changelog.Debian.$foreign_arch.gz"
 rm /tmp/debian-chroot/usr/share/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/misc/magic.mgc
 # delete real files

tests/jessie-or-older

@@ -1,39 +1,43 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
-	echo "this test modifies the system and should only be run inside a container" >&2
-	exit 1
-fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
+trap "rm -f /tmp/debian-chroot-{{ MODE }}.tar /tmp/debian-chroot-root-normal.tar" EXIT INT TERM
+
+[ "$(id -u)" -eq 0 ]
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+    if [ ! -e /mmdebstrap-testenv ]; then
+      echo "this test modifies the system and should only be run inside a container" >&2
+      exit 1
+    fi
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
+fi
+
+MMTARFILTER=
+[ -x /usr/bin/mmtarfilter ] && MMTARFILTER=/usr/bin/mmtarfilter
+[ -x ./tarfilter ] && MMTARFILTER=./tarfilter
+
 filter() {
-	./tarfilter \
+  "$MMTARFILTER" \
     --path-exclude=/usr/bin/uncompress \
     --path-exclude=/var/cache/debconf/config.dat-old \
     --path-exclude=/var/cache/debconf/templates.dat-old \
     --path-exclude=/var/lib/dpkg/available \
+    --path-exclude=/var/lib/dpkg/diversions \
+    --path-exclude=/var/lib/dpkg/diversions-old \
     --path-exclude=/var/lib/dpkg/cmethopt \
     --path-exclude=/var/lib/dpkg/status-old \
     --path-exclude=/var/lib/shells.state
 }
 
 # base for comparison without jessie-or-older hook
-{{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-root-normal.tar
+{{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} - {{ MIRROR }} >/tmp/debian-chroot-root-normal.tar
 
-# root
-{{ CMD }} --mode=root --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-root.tar
-cmp /tmp/debian-chroot-root-normal.tar /tmp/debian-chroot-root.tar
-rm /tmp/debian-chroot-root.tar
-# unshare
-runuser -u user -- {{ CMD }} --mode=unshare --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-unshare.tar
-cmp /tmp/debian-chroot-root-normal.tar /tmp/debian-chroot-unshare.tar
-rm /tmp/debian-chroot-unshare.tar
-# fakechroot
-runuser -u user -- {{ CMD }} --mode=fakechroot --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-fakechroot.tar
-cmp /tmp/debian-chroot-root-normal.tar /tmp/debian-chroot-fakechroot.tar
-rm /tmp/debian-chroot-fakechroot.tar
-
-rm /tmp/debian-chroot-root-normal.tar
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter >/tmp/debian-chroot-{{ MODE }}.tar
+filter </tmp/debian-chroot-root-normal.tar | cmp - /tmp/debian-chroot-{{ MODE }}.tar

tests/keyring

@@ -5,7 +5,12 @@ if [ ! -e /mmdebstrap-testenv ]; then
   echo "this test modifies the system and should only be run inside a container" >&2
   exit 1
 fi
-rm /etc/apt/trusted.gpg.d/*.gpg
+for f in /etc/apt/trusted.gpg.d/*.gpg /etc/apt/trusted.gpg.d/*.asc; do
+  [ -e "$f" ] || continue
+  rm "$f"
+done
+rmdir /etc/apt/trusted.gpg.d
+mkdir /etc/apt/trusted.gpg.d
 {{ CMD }} --mode=root --variant=apt --keyring=/usr/share/keyrings/debian-archive-keyring.gpg --keyring=/usr/share/keyrings/ {{ DIST }} /tmp/debian-chroot "deb {{ MIRROR }} {{ DIST }} main"
 # make sure that no [signedby=...] managed to make it into the sources.list
 echo "deb {{ MIRROR }} {{ DIST }} main" | cmp /tmp/debian-chroot/etc/apt/sources.list -

tests/logfile

@@ -1,20 +1,23 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap "rm -rf /tmp/debian-chroot /tmp/log /tmp/trimmed" EXIT INT TERM
+
 # we check the full log to also prevent debug printfs to accidentally make it into a commit
 {{ CMD }} --mode=root --variant=apt --logfile=/tmp/log {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 # omit the last line which should contain the runtime
-head --lines=-1 /tmp/log > /tmp/trimmed
-cat << LOG | diff -u - /tmp/trimmed
+head --lines=-1 /tmp/log >/tmp/trimmed
+cat <<LOG | diff -u - /tmp/trimmed
 I: chroot architecture {{ HOSTARCH }} is equal to the host's architecture
+I: finding correct signed-by value...
 I: automatically chosen format: directory
 I: running apt-get update...
 I: downloading packages with apt...
 I: extracting archives...
 I: installing essential packages...
+I: installing remaining packages inside the chroot...
 I: cleaning package lists and apt cache...
 LOG
 tail --lines=1 /tmp/log | grep '^I: success in .* seconds$'
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
-rm -r /tmp/debian-chroot
-rm /tmp/log /tmp/trimmed

tests/merged-fakechroot-inside-unmerged-chroot

@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# make sure that the $FAKECHROOT_CMD_SUBST environment variable is set up
+# such that one can create a merged-/usr chroot from an unmerged-/usr system
+
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+trap "rm -f /tmp/chroot-fakechroot.tar /tmp/chroot-root.tar" EXIT INT TERM
+[ "$(whoami)" = "root" ]
+{{ CMD }} --mode=root --variant=apt --hook-dir=./hooks/merged-usr {{ DIST }} /tmp/chroot-root.tar {{ MIRROR }}
+cat <<'SCRIPT' >script.sh
+#!/bin/sh
+set -exu
+rootfs="$1"
+mkdir -p "$rootfs/mnt/hooks"
+[ -e /usr/libexec/mmdebstrap/ldconfig.fakechroot ] && cp -a /usr/libexec/mmdebstrap/ldconfig.fakechroot "$rootfs/mnt"
+[ -e ./ldconfig.fakechroot ] && cp -a ./ldconfig.fakechroot "$rootfs/mnt"
+[ -e /usr/share/mmdebstrap/hooks/merged-usr ] && cp -a /usr/share/mmdebstrap/hooks/merged-usr "$rootfs/mnt/hooks"
+[ -e ./hooks/merged-usr ] && cp -a ./hooks/merged-usr "$rootfs/mnt/hooks"
+[ -e /usr/bin/mmdebstrap ] && cp -aT /usr/bin/mmdebstrap "$rootfs/usr/bin/mmdebstrap"
+[ -e ./mmdebstrap ] && cp -aT ./mmdebstrap "$rootfs/mnt/mmdebstrap"
+chroot "$rootfs" env --chdir=/mnt \
+	runuser -u user -- \
+	{{ CMD }} --mode=fakechroot --variant=apt \
+	--hook-dir=./hooks/merged-usr \
+	--customize-hook='chroot "$1" echo "$FAKECHROOT_CMD_SUBST" | tr ":" "\n" | sort' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(readlink /bin)\" = usr/bin"' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(realpath -e /bin/ldd)\" = /usr/bin/ldd"' \
+	--customize-hook='chroot "$1" echo ":$FAKECHROOT_CMD_SUBST" | grep --quiet :/usr/bin/ldd=' \
+	--customize-hook='chroot "$1" echo ":$FAKECHROOT_CMD_SUBST" | grep --quiet :/bin/ldd=' \
+	--customize-hook='chroot "$1" env PATH=/bin ldd /bin/true 2>&1 | grep --quiet "fakeldd: objdump: command not found: install binutils package"' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(readlink /sbin)\" = usr/sbin"' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(realpath -e /sbin/ldconfig)\" = /usr/sbin/ldconfig"' \
+	--customize-hook='chroot "$1" echo ":$FAKECHROOT_CMD_SUBST" | grep --quiet :/usr/sbin/ldconfig=' \
+	--customize-hook='chroot "$1" echo ":$FAKECHROOT_CMD_SUBST" | grep --quiet :/sbin/ldconfig=' \
+	--customize-hook='chroot "$1" env PATH=/sbin ldconfig 2>&1 | grep --quiet "/usr/bin/env: ‘python3’: No such file or directory"' \
+	{{ DIST }} /tmp/chroot-fakechroot.tar {{ MIRROR }}
+SCRIPT
+chmod +x script.sh
+{{ CMD }} --mode=root --variant=apt --include=perl,python3,passwd,fakeroot,fakechroot \
+  --hook-dir=./hooks/no-merged-usr \
+  --customize-hook='chroot "$1" useradd --home-dir /home/user --create-home user' \
+  --customize-hook='chroot "$1" sh -c "exec test \"\$(realpath -e /usr/bin/ldd)\" = /usr/bin/ldd"' \
+  --customize-hook='chroot "$1" sh -c "exec test ! -e /usr/sbin/ldconfig"' \
+  --customize-hook=./script.sh \
+  --customize-hook="copy-out /tmp/chroot-fakechroot.tar /tmp" \
+  {{ DIST }} /dev/null {{ MIRROR }}
+cmp /tmp/chroot-fakechroot.tar /tmp/chroot-root.tar || diffoscope /tmp/chroot-fakechroot.tar /tmp/chroot-root.tar

tests/mirror-is-real-file

@@ -2,7 +2,7 @@
 set -eu
 export LC_ALL=C.UTF-8
 trap "rm -f /tmp/debian-chroot.tar /tmp/sources.list" EXIT INT TERM
-echo "deb {{ MIRROR }} {{ DIST }} main" > /tmp/sources.list
+echo "deb {{ MIRROR }} {{ DIST }} main" >/tmp/sources.list
 {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar /tmp/sources.list
 tar -tf /tmp/debian-chroot.tar \
   | sed 's#^./etc/apt/sources.list.d/0000sources.list$#./etc/apt/sources.list#' \

tests/missing-dev-sys-proc-inside-the-chroot

@@ -1,10 +1,24 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
+
+[ {{ MODE }} = "unshare" ]
+[ {{ VARIANT }} = "custom" ]
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+    if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
+    fi
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
-runuser -u user -- {{ CMD }} --mode=unshare --variant=custom --include=dpkg,dash,diffutils,coreutils,libc-bin,sed {{ DIST }} /dev/null {{ MIRROR }}
+
+# do not install base-files, so that /proc, /sys and /dev are missing
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} \
+  --setup-hook='for dir in bin lib lib32 lib64 libo32 libx32 sbin; do ln -s "usr/$dir" "$1/$dir"; done' \
+  --include=dpkg,dash,diffutils,coreutils,libc-bin,sed \
+  {{ DIST }} /dev/null {{ MIRROR }}

tests/missing-device-nodes-outside-the-chroot

@@ -6,8 +6,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
   exit 1
 fi
 rm /dev/console
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+useradd --home-dir /home/user --create-home user
 runuser -u user -- {{ CMD }} --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar

tests/mmdebstrap

@@ -0,0 +1,31 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+
+[ "$(id -u)" -eq 0 ]
+[ {{ MODE }} = "root" ]
+case {{ FORMAT }} in tar | squashfs | ext2 | ext4) : ;; *) exit 1 ;; esac
+
+{{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} /tmp/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} {{ MIRROR }}
+# creating an ext4 image on a 9p filesystem produces different results compared
+# to creating it on a tmpfs or ext4 fs because 9p does not support discards and
+# even when running with -E nodiscard, the number of written bytes will differ
+# https://lore.kernel.org/linux-ext4/171484520952.2626447.2160419274451668597@localhost/T/#t
+mv /tmp/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }}
+if [ "{{ FORMAT }}" = tar ]; then
+  printf 'ustar\0' | cmp --bytes=6 --ignore-initial=257:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar -
+elif [ "{{ FORMAT }}" = squashfs ]; then
+  printf 'hsqs' | cmp --bytes=4 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.squashfs -
+elif [ "{{ FORMAT }}" = ext2 ]; then
+  printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext2 -
+  printf '\000\000\000\000\000\000\000\000\000\000\000\000' | cmp --bytes=12 --ignore-initial=1116:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext2 -
+elif [ "{{ FORMAT }}" = ext4 ]; then
+  printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext4 -
+  printf '\074\020\000\000\302\042\000\000\153\004\000\000' | cmp --bytes=12 --ignore-initial=1116:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext4 -
+
+  [ "$(/sbin/blkid --match-tag UUID --output value ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext4)" = "$(uuidgen --sha1 --namespace="$(uuidgen --sha1 --namespace='@dns' --name mister-muffin.de)" --name $SOURCE_DATE_EPOCH)" ]
+else
+  echo "unknown format: {{ FORMAT }}" >&2
+  exit 1
+fi

tests/multiple-include

@@ -3,24 +3,18 @@ set -eu
 export LC_ALL=C.UTF-8
 trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
 {{ CMD }} --mode=root --variant=apt --include=doc-debian --include=tzdata {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
 rm /tmp/debian-chroot/usr/share/lintian/overrides/tzdata
 rm /tmp/debian-chroot/etc/localtime
 rm /tmp/debian-chroot/etc/timezone
-rm /tmp/debian-chroot/usr/sbin/tzconfig
 rm -r /tmp/debian-chroot/usr/share/doc/tzdata
 rm -r /tmp/debian-chroot/usr/share/zoneinfo
-rm /tmp/debian-chroot/var/lib/apt/extended_states
-rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.list
-rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.md5sums
-rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.list
-rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.md5sums
-rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.config
-rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.postinst
-rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.postrm
-rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.templates
-rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.preinst
-rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.prerm
+for p in doc-debian tzdata; do
+  for f in list md5sums config postinst postrm templates preinst prerm; do
+    [ -e "/tmp/debian-chroot/var/lib/dpkg/info/$p.$f" ] || continue
+    rm "/tmp/debian-chroot/var/lib/dpkg/info/$p.$f"
+  done
+done
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/no-sbin-in-path

@@ -7,16 +7,22 @@
 
 set -eu
 export LC_ALL=C.UTF-8
-[ "{{ MODE }}" = "fakechroot" ]
+
 trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
+[ "{{ MODE }}" = "fakechroot" ]
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
     if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
     fi
-	adduser --gecos user --disabled-password user
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
-$prefix env PATH=/usr/bin:/bin fakechroot fakeroot {{ CMD }} --mode=fakechroot --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+
+$prefix env PATH=/usr/bin:/bin fakechroot fakeroot {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/not-having-to-install-apt-in-include-because-a-hook-did-it-before

@@ -4,6 +4,6 @@ export LC_ALL=C.UTF-8
 trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 {{ CMD }} --mode={{ MODE }} --variant=essential --include=apt \
   --essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get update' \
-	--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$1" apt' \
+  --essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install apt' \
   {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/pass-distribution-but-implicitly-write-to-stdout

@@ -5,10 +5,10 @@ if [ ! -e /mmdebstrap-testenv ]; then
   echo "this test modifies the system and should only be run inside a container" >&2
   exit 1
 fi
-cat << HOSTS >> /etc/hosts
+cat <<HOSTS >>/etc/hosts
 127.0.0.1 deb.debian.org
 127.0.0.1 security.debian.org
 HOSTS
-{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} > /tmp/debian-chroot.tar
+{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} >/tmp/debian-chroot.tar
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar

tests/pivot_root

@@ -4,52 +4,51 @@ export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 trap "rm -f /tmp/chroot1.tar /tmp/chroot2.tar /tmp/chroot3.tar /tmp/mmdebstrap" EXIT INT TERM
 
-if [ ! -e /mmdebstrap-testenv ]; then
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+  if ! id "${SUDO_USER:-user}" 2>/dev/null; then
+    if [ ! -e /mmdebstrap-testenv ]; then
       echo "this test modifies the system and should only be run inside a container" >&2
       exit 1
+    fi
+    useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+  fi
+  prefix="runuser -u ${SUDO_USER:-user} --"
 fi
 
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	adduser --gecos user --disabled-password user
-fi
-
-prefix=
-[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
-
-MMDEBSTRAP=
-[ -e /usr/bin/mmdebstrap ] && MMDEBSTRAP=/usr/bin/mmdebstrap
-[ -e ./mmdebstrap ] && MMDEBSTRAP=./mmdebstrap
-
 $prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
-	--include=mount \
+  --include=mount,perl \
   {{ DIST }} /tmp/chroot1.tar {{ MIRROR }}
 
 if [ {{ MODE }} = "unshare" ]; then
   # calling pivot_root in root mode does not work for mysterious reasons:
   # pivot_root: failed to change root from `.' to `mnt': Invalid argument
-	$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount \
-		--customize-hook="upload $MMDEBSTRAP /$MMDEBSTRAP" \
-		--customize-hook='chmod +x "$1"/'"$MMDEBSTRAP" \
-		--customize-hook='mount -o rbind "$1" /mnt && cd /mnt && /sbin/pivot_root . mnt' \
+  $prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount,perl \
+    --customize-hook='mkdir -p "$1/mnt" "$1/oldroot"' \
+    --customize-hook='[ ! -e /usr/bin/mmdebstrap ] || cp -aT /usr/bin/mmdebstrap "$1/usr/bin/mmdebstrap"' \
+    --customize-hook='[ ! -e ./mmdebstrap ] || cp -aT ./mmdebstrap "$1/mnt/mmdebstrap"' \
+    --customize-hook='mount -o rbind "$1" /mnt && cd /mnt && /sbin/pivot_root . oldroot' \
     --customize-hook='unshare -U echo nested unprivileged unshare' \
-		--customize-hook="/$MMDEBSTRAP"' --mode=unshare --variant=apt --include=mount {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
+    --customize-hook='env --chdir=/mnt {{ CMD }} --mode=unshare --variant=apt --include=mount,perl {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
     --customize-hook='copy-out /tmp/chroot3.tar /tmp' \
-		--customize-hook='rm "$1/'"$MMDEBSTRAP"'"' \
-		--customize-hook='umount -l mnt sys' \
+    --customize-hook='rm -f "/usr/bin/mmdebstrap" "/mnt/mmdebstrap"' \
+    --customize-hook='umount -l oldroot sys' \
+    --customize-hook='rmdir /oldroot' \
     {{ DIST }} /tmp/chroot2.tar {{ MIRROR }}
 
-		cmp /tmp/chroot1.tar /tmp/chroot2.tar
-		cmp /tmp/chroot1.tar /tmp/chroot3.tar
+  cmp /tmp/chroot1.tar /tmp/chroot2.tar || diffoscope /tmp/chroot1.tar /tmp/chroot2.tar
+  cmp /tmp/chroot1.tar /tmp/chroot3.tar || diffoscope /tmp/chroot1.tar /tmp/chroot3.tar
   rm /tmp/chroot2.tar /tmp/chroot3.tar
 fi
 
-$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount \
-	--customize-hook="upload $MMDEBSTRAP /$MMDEBSTRAP" \
-	--customize-hook='chmod +x "$1"/'"$MMDEBSTRAP" \
-	--chrooted-customize-hook="/$MMDEBSTRAP"' --mode=unshare --variant=apt --include=mount {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount,perl \
+  --customize-hook='mkdir -p "$1/mnt"' \
+  --customize-hook='[ ! -e /usr/bin/mmdebstrap ] || cp -aT /usr/bin/mmdebstrap "$1/usr/bin/mmdebstrap"' \
+  --customize-hook='[ ! -e ./mmdebstrap ] || cp -aT ./mmdebstrap "$1/mnt/mmdebstrap"' \
+  --chrooted-customize-hook='env --chdir=/mnt {{ CMD }} --mode=unshare --variant=apt --include=mount,perl {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
   --customize-hook='copy-out /tmp/chroot3.tar /tmp' \
-	--customize-hook='rm "$1/'"$MMDEBSTRAP"'"' \
+  --customize-hook='rm -f "$1/usr/bin/mmdebstrap" "$1/mnt/mmdebstrap"' \
   {{ DIST }} /tmp/chroot2.tar {{ MIRROR }}
 
-cmp /tmp/chroot1.tar /tmp/chroot2.tar
-cmp /tmp/chroot1.tar /tmp/chroot3.tar
+cmp /tmp/chroot1.tar /tmp/chroot2.tar || diffoscope /tmp/chroot1.tar /tmp/chroot2.tar
+cmp /tmp/chroot1.tar /tmp/chroot3.tar || diffoscope /tmp/chroot1.tar /tmp/chroot3.tar

tests/preserve-mode-of-etc-resolv-conf-and-etc-hostname

@@ -7,7 +7,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 fi
 for f in /etc/resolv.conf /etc/hostname; do
   # preserve original content
-	cat "$f" > "$f.bak"
+  cat "$f" >"$f.bak"
   # in case $f is a symlink, we replace it by a real file
   if [ -L "$f" ]; then
     rm "$f"

tests/progress-bars-on-fake-tty

@@ -2,5 +2,5 @@
 set -eu
 export LC_ALL=C.UTF-8
 trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
-script -qfc "{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}" /dev/null
+script -qfec "{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}" /dev/null
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/read-from-stdin-write-to-stdout

@@ -2,5 +2,5 @@
 set -eu
 export LC_ALL=C.UTF-8
 trap "rm /tmp/debian-chroot.tar" EXIT INT TERM
-echo "deb {{ MIRROR }} {{ DIST }} main" | {{ CMD }} --mode={{ MODE }} --variant=apt > /tmp/debian-chroot.tar
+echo "deb {{ MIRROR }} {{ DIST }} main" | {{ CMD }} --mode={{ MODE }} --variant=apt >/tmp/debian-chroot.tar
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -