release 1.2.5

/etc/subgid contains a mapping of user names (not group names) to group
ids as defined in man subgid.

.mailmap

@@ -1,2 +1,4 @@
 Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
 Johannes Schauer Marin Rodrigues <josch@mister-muffin.de> <j.schauer@email.de>
+Helmut Grohne <helmut@subdivi.de> <helmut.grohne@intenta.de>
+Benjamin Drung <benjamin.drung@ionos.com> <benjamin.drung@cloud.ionos.com>

CHANGELOG.md

@@ -1,3 +1,71 @@
+1.2.5 (2023-01-04)
+------------------
+
+ - bugfix release
+
+1.2.4 (2022-12-23)
+------------------
+
+ - bugfix release
+ - add jessie-or-older extract hook
+
+1.2.3 (2022-11-16)
+------------------
+
+ - use Text::ParseWords::shellwords instead of spawning a new shell
+ - mount and unmount once, instead for each run_chroot() call
+
+1.2.2 (2022-10-27)
+------------------
+
+ - allow /etc/apt/trusted.gpg.d/ not to exist
+ - always create /var/lib/dpkg/arch to make foreign architecture chrootless
+   tarballs bit-by-bit identical
+ - write an empty /etc/machine-id instead of writing 'uninitialized'
+ - only print progress bars on interactive terminals that are wide enough
+
+1.2.1 (2022-09-08)
+------------------
+
+ - bugfix release
+
+1.2.0 (2022-09-05)
+------------------
+
+ - remove proot mode
+ - error out if stdout is an interactive terminal
+ - replace taridshift by tarfilter --idshift
+ - tarfilter: add --transform option
+ - multiple --skip options can be separated by comma or whitespace
+ - also cleanup the contents of /run
+ - support apt patterns and paths with commas and whitespace in --include
+ - hooks: store the values of the --include option in MMDEBSTRAP_INCLUDE
+ - add new --skip options: chroot/start-stop-daemon, chroot/policy-rc.d
+   chroot/mount, chroot/mount/dev, chroot/mount/proc, chroot/mount/sys,
+   cleanup/run
+
+1.1.0 (2022-07-26)
+----------------
+
+ - mount a new /dev/pts instance into the chroot to make posix_openpt work
+ - adjust merged-/usr hook to work the same way as debootstrap
+ - add no-merged-usr hook
+
+1.0.1 (2022-05-29)
+------------------
+
+ - bugfix release
+
+1.0.0 (2022-05-28)
+------------------
+
+ - all documented interfaces are now considered stable
+ - allow file:// mirrors
+ - /var/cache/apt/archives/ is now allowed to contain *.deb packages
+ - add file-mirror-automount hook-dir
+ - set $MMDEBSTRAP_VERBOSITY in hooks
+ - rewrite coverage with multiple individual and skippable shell scripts
+
 0.8.6 (2022-03-25)
 ------------------
 

README.md

@@ -34,7 +34,7 @@ Summary:
  - chroot with apt in 11 seconds
  - gzipped tarball with apt is 27M small
  - bit-by-bit reproducible output
- - unprivileged operation using Linux user namespaces, fakechroot or proot
+ - unprivileged operation using Linux user namespaces or fakechroot
  - can operate on filesystems mounted with nodev
  - foreign architecture chroots with qemu-user
  - variant installing only Essential:yes packages and dependencies
@@ -78,9 +78,9 @@ privileges to create a file (the chroot tarball) in one's home directory.
 Thus, mmdebstrap provides multiple options to create a chroot tarball with the
 right permissions **without superuser privileges**. This avoids a whole class
 of bugs like #921815. Depending on what is available, it uses either Linux user
-namespaces, fakechroot or proot.  Debootstrap supports fakechroot but will not
+namespaces or fakechroot. Debootstrap supports fakechroot but will not
 create a tarball with the right permissions by itself. Support for Linux user
-namespaces and proot is missing (see bugs #829134 and #698347, respectively).
+namespaces is missing (see #829134).
 
 When creating a chroot tarball with debootstrap, the temporary chroot directory
 cannot be on a filesystem that has been mounted with nodev. In unprivileged
@@ -137,6 +137,11 @@ By default, `coverage.sh` will skip running a single test which tries creating
 a Ubuntu Focal chroot. To not skip that test, run `coverage.sh` with the
 environment variable `ONLINE=yes`.
 
+If a test fails you can run individual tests by executing `coverage.py` with
+the test name and optionally limit it to a specific distribution like so:
+
+    CMD=./mmdebstrap ./coverage.py --dist unstable check-against-debootstrap-dist
+
 Bugs
 ====
 
@@ -147,11 +152,16 @@ Contributors
 ============
 
  - Johannes Schauer Marin Rodrigues (main author)
+ - Gioele Barabucci
  - Helmut Grohne
  - Benjamin Drung
- - Steve Dodd
+ - Jochen Sprickerhof
  - Josh Triplett
  - Konstantin Demin
+ - David Kalnischkies
+ - Joe Groocock
+ - Nicolas Vigier
+ - Raul Tambre
+ - Steve Dodd
  - Trent W. Buck
  - Vagrant Cascadian
- - Gioele Barabucci

coverage.py

@@ -0,0 +1,408 @@
+#!/usr/bin/env python3
+
+from debian.deb822 import Deb822, Release
+import email.utils
+import os
+import sys
+import shutil
+import subprocess
+import argparse
+import time
+from datetime import timedelta
+from collections import defaultdict
+from itertools import product
+
+have_qemu = os.getenv("HAVE_QEMU", "yes") == "yes"
+have_unshare = os.getenv("HAVE_UNSHARE", "yes") == "yes"
+have_binfmt = os.getenv("HAVE_BINFMT", "yes") == "yes"
+run_ma_same_tests = os.getenv("RUN_MA_SAME_TESTS", "yes") == "yes"
+cmd = os.getenv("CMD", "./mmdebstrap")
+
+default_dist = os.getenv("DEFAULT_DIST", "unstable")
+all_dists = ["oldstable", "stable", "testing", "unstable"]
+default_mode = "auto" if have_unshare else "root"
+all_modes = ["auto", "root", "unshare", "fakechroot", "chrootless"]
+default_variant = "apt"
+all_variants = [
+    "extract",
+    "custom",
+    "essential",
+    "apt",
+    "minbase",
+    "buildd",
+    "-",
+    "standard",
+]
+default_format = "auto"
+all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "null"]
+
+mirror = os.getenv("mirror", "http://127.0.0.1/debian")
+hostarch = subprocess.check_output(["dpkg", "--print-architecture"]).decode().strip()
+
+release_path = f"./shared/cache/debian/dists/{default_dist}/Release"
+if not os.path.exists(release_path):
+    print("path doesn't exist:", release_path, file=sys.stderr)
+    print("run ./make_mirror.sh first", file=sys.stderr)
+    exit(1)
+if os.getenv("SOURCE_DATE_EPOCH") is not None:
+    s_d_e = os.getenv("SOURCE_DATE_EPOCH")
+else:
+    with open(release_path) as f:
+        rel = Release(f)
+    s_d_e = str(email.utils.mktime_tz(email.utils.parsedate_tz(rel["Date"])))
+
+separator = (
+    "------------------------------------------------------------------------------"
+)
+
+
+def skip(condition, dist, mode, variant, fmt):
+    if not condition:
+        return ""
+    for line in condition.splitlines():
+        if not line:
+            continue
+        if eval(line):
+            return line.strip()
+    return ""
+
+
+def parse_config(confname):
+    config_dict = defaultdict(dict)
+    config_order = list()
+    all_vals = {
+        "Dists": all_dists,
+        "Modes": all_modes,
+        "Variants": all_variants,
+        "Formats": all_formats,
+    }
+    with open(confname) as f:
+        for test in Deb822.iter_paragraphs(f):
+            if "Test" not in test.keys():
+                print("Test without name", file=sys.stderr)
+                exit(1)
+            name = test["Test"]
+            config_order.append(name)
+            for k in test.keys():
+                v = test[k]
+                if k not in [
+                    "Test",
+                    "Dists",
+                    "Modes",
+                    "Variants",
+                    "Formats",
+                    "Skip-If",
+                    "Needs-QEMU",
+                    "Needs-Root",
+                ]:
+                    print(f"Unknown field name {k} in test {name}")
+                    exit(1)
+                if k in all_vals.keys():
+                    if v == "default":
+                        print(
+                            f"Setting {k} to default in Test {name} is redundant",
+                            file=sys.stderr,
+                        )
+                        exit(1)
+                    if v == "any":
+                        v = all_vals[k]
+                    else:
+                        # else, split the value by whitespace
+                        v = v.split()
+                        for i in v:
+                            if i not in all_vals[k]:
+                                print(
+                                    f"{i} is not a valid value for {k}", file=sys.stderr
+                                )
+                                exit(1)
+                config_dict[name][k] = v
+    return config_order, config_dict
+
+
+def format_failed(num, total, name, dist, mode, variant, fmt, config_dict):
+    ret = f"({num}/{total}) {name}"
+    if len(config_dict[name].get("Dists", [])) > 1:
+        ret += f" --dist={dist}"
+    if len(config_dict[name].get("Modes", [])) > 1:
+        ret += f" --mode={mode}"
+    if len(config_dict[name].get("Variants", [])) > 1:
+        ret += f" --variant={variant}"
+    if len(config_dict[name].get("Formats", [])) > 1:
+        ret += f" --format={fmt}"
+    return ret
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("test", nargs="*", help="only run these tests")
+    parser.add_argument(
+        "-x",
+        "--exitfirst",
+        action="store_const",
+        dest="maxfail",
+        const=1,
+        help="exit instantly on first error or failed test.",
+    )
+    parser.add_argument(
+        "--maxfail",
+        metavar="num",
+        action="store",
+        type=int,
+        dest="maxfail",
+        default=0,
+        help="exit after first num failures or errors.",
+    )
+    parser.add_argument(
+        "--mode",
+        metavar="mode",
+        help=f"only run tests with this mode (Default = {default_mode})",
+    )
+    parser.add_argument(
+        "--dist",
+        metavar="dist",
+        help=f"only run tests with this dist (Default = {default_dist})",
+    )
+    parser.add_argument(
+        "--variant",
+        metavar="variant",
+        help=f"only run tests with this variant (Default = {default_variant})",
+    )
+    parser.add_argument(
+        "--format",
+        metavar="format",
+        help=f"only run tests with this format (Default = {default_format})",
+    )
+    parser.add_argument(
+        "--skip", metavar="test", action="append", help="skip this test"
+    )
+    args = parser.parse_args()
+
+    # copy over files from git or as distributed
+    for (git, dist, target) in [
+        ("./mmdebstrap", "/usr/bin/mmdebstrap", "mmdebstrap"),
+        ("./tarfilter", "/usr/bin/mmtarfilter", "tarfilter"),
+        (
+            "./proxysolver",
+            "/usr/lib/apt/solvers/mmdebstrap-dump-solution",
+            "proxysolver",
+        ),
+        (
+            "./ldconfig.fakechroot",
+            "/usr/libexec/mmdebstrap/ldconfig.fakechroot",
+            "ldconfig.fakechroot",
+        ),
+    ]:
+        if os.path.exists(git):
+            shutil.copy(git, f"shared/{target}")
+        else:
+            shutil.copy(dist, f"shared/{target}")
+    # copy over hooks from git or as distributed
+    if os.path.exists("hooks"):
+        shutil.copytree("hooks", "shared/hooks", dirs_exist_ok=True)
+    else:
+        shutil.copytree(
+            "/usr/share/mmdebstrap/hooks", "shared/hooks", dirs_exist_ok=True
+        )
+
+    # parse coverage.txt
+    config_order, config_dict = parse_config("coverage.txt")
+
+    indirbutnotcovered = set(
+        [d for d in os.listdir("tests") if not d.startswith(".")]
+    ) - set(config_order)
+    if indirbutnotcovered:
+        print(
+            "test(s) missing from coverage.txt: %s"
+            % (", ".join(sorted(indirbutnotcovered))),
+            file=sys.stderr,
+        )
+        exit(1)
+    coveredbutnotindir = set(config_order) - set(
+        [d for d in os.listdir("tests") if not d.startswith(".")]
+    )
+    if coveredbutnotindir:
+        print(
+            "test(s) missing from ./tests: %s"
+            % (", ".join(sorted(coveredbutnotindir))),
+            file=sys.stderr,
+        )
+
+        exit(1)
+
+    # produce the list of tests using the cartesian product of all allowed
+    # dists, modes, variants and formats of a given test
+    tests = []
+    for name in config_order:
+        test = config_dict[name]
+        for dist, mode, variant, fmt in product(
+            test.get("Dists", [default_dist]),
+            test.get("Modes", [default_mode]),
+            test.get("Variants", [default_variant]),
+            test.get("Formats", [default_format]),
+        ):
+            skipreason = skip(test.get("Skip-If"), dist, mode, variant, fmt)
+            if skipreason:
+                tt = ("skip", skipreason)
+            elif have_qemu:
+                tt = "qemu"
+            elif test.get("Needs-QEMU", "false") == "true":
+                tt = ("skip", "test needs QEMU")
+            elif test.get("Needs-Root", "false") == "true":
+                tt = "sudo"
+            elif mode == "auto" and not have_unshare:
+                tt = "sudo"
+            elif mode == "root":
+                tt = "sudo"
+            elif mode == "unshare" and not have_unshare:
+                tt = ("skip", "test needs unshare")
+            else:
+                tt = "null"
+            tests.append((tt, name, dist, mode, variant, fmt))
+
+    torun = []
+    num_tests = len(tests)
+    if args.test:
+        # check if all given tests are either a valid name or a valid number
+        for test in args.test:
+            if test in [name for (_, name, _, _, _, _) in tests]:
+                continue
+            if not test.isdigit():
+                print(f"cannot find test named {test}", file=sys.stderr)
+                exit(1)
+            if int(test) >= len(tests) or int(test) <= 0 or str(int(test)) != test:
+                print(f"test number {test} doesn't exist", file=sys.stderr)
+                exit(1)
+
+        for i, (_, name, _, _, _, _) in enumerate(tests):
+            # if either the number or test name matches, then we use this test,
+            # otherwise we skip it
+            if name in args.test:
+                torun.append(i)
+            if str(i + 1) in args.test:
+                torun.append(i)
+        num_tests = len(torun)
+
+    starttime = time.time()
+    skipped = defaultdict(list)
+    failed = []
+    num_success = 0
+    num_finished = 0
+    for i, (test, name, dist, mode, variant, fmt) in enumerate(tests):
+        if torun and i not in torun:
+            continue
+        print(separator, file=sys.stderr)
+        print("(%d/%d) %s" % (i + 1, len(tests), name), file=sys.stderr)
+        print("dist: %s" % dist, file=sys.stderr)
+        print("mode: %s" % mode, file=sys.stderr)
+        print("variant: %s" % variant, file=sys.stderr)
+        print("format: %s" % fmt, file=sys.stderr)
+        if num_finished > 0:
+            currenttime = time.time()
+            timeleft = timedelta(
+                seconds=int(
+                    (num_tests - num_finished)
+                    * (currenttime - starttime)
+                    / num_finished
+                )
+            )
+            print("time left: %s" % timeleft, file=sys.stderr)
+        if failed:
+            print("failed: %d" % len(failed), file=sys.stderr)
+        num_finished += 1
+        with open("tests/" + name) as fin, open("shared/test.sh", "w") as fout:
+            for line in fin:
+                line = line.replace("{{ CMD }}", cmd)
+                line = line.replace("{{ SOURCE_DATE_EPOCH }}", s_d_e)
+                line = line.replace("{{ DIST }}", dist)
+                line = line.replace("{{ MIRROR }}", mirror)
+                line = line.replace("{{ MODE }}", mode)
+                line = line.replace("{{ VARIANT }}", variant)
+                line = line.replace("{{ FORMAT }}", fmt)
+                line = line.replace("{{ HOSTARCH }}", hostarch)
+                fout.write(line)
+        # ignore:
+        # SC2016 Expressions don't expand in single quotes, use double quotes for that.
+        # SC2050 This expression is constant. Did you forget the $ on a variable?
+        # SC2194 This word is constant. Did you forget the $ on a variable?
+        shellcheck = subprocess.run(
+            [
+                "shellcheck",
+                "--exclude=SC2050,SC2194,SC2016",
+                "-f",
+                "gcc",
+                "shared/test.sh",
+            ],
+            check=False,
+            stdout=subprocess.PIPE,
+        ).stdout.decode()
+        argv = None
+        match test:
+            case "qemu":
+                argv = ["./run_qemu.sh"]
+            case "sudo":
+                argv = ["./run_null.sh", "SUDO"]
+            case "null":
+                argv = ["./run_null.sh"]
+            case ("skip", reason):
+                skipped[reason].append(
+                    ("(%d/%d) %s" % (i + 1, len(tests), name), dist, mode, variant, fmt)
+                )
+                print(f"skipped because of {reason}", file=sys.stderr)
+                continue
+        print(separator, file=sys.stderr)
+        if args.skip and name in args.skip:
+            print(f"skipping because of --skip={name}", file=sys.stderr)
+            continue
+        if args.dist and args.dist != dist:
+            print(f"skipping because of --dist={args.dist}", file=sys.stderr)
+            continue
+        if args.mode and args.mode != mode:
+            print(f"skipping because of --mode={args.mode}", file=sys.stderr)
+            continue
+        if args.variant and args.variant != variant:
+            print(f"skipping because of --variant={args.variant}", file=sys.stderr)
+            continue
+        if args.format and args.format != fmt:
+            print(f"skipping because of --format={args.format}", file=sys.stderr)
+            continue
+        proc = subprocess.Popen(argv)
+        try:
+            proc.wait()
+        except KeyboardInterrupt:
+            proc.terminate()
+            proc.wait()
+            break
+        print(separator, file=sys.stderr)
+        if proc.returncode != 0 or shellcheck != "":
+            if shellcheck != "":
+                print(shellcheck)
+            failed.append(
+                format_failed(
+                    i + 1, len(tests), name, dist, mode, variant, fmt, config_dict
+                )
+            )
+            print("result: FAILURE", file=sys.stderr)
+        else:
+            print("result: SUCCESS", file=sys.stderr)
+            num_success += 1
+        if args.maxfail and len(failed) >= args.maxfail:
+            break
+    print(
+        "successfully ran %d tests" % num_success,
+        file=sys.stderr,
+    )
+    if skipped:
+        print("skipped %d:" % sum([len(v) for v in skipped.values()]), file=sys.stderr)
+        for reason, l in skipped.items():
+            print(f"skipped because of {reason}:", file=sys.stderr)
+            for t in l:
+                print(f"    {t}", file=sys.stderr)
+    if failed:
+        print("failed %d:" % len(failed), file=sys.stderr)
+        for f in failed:
+            print(f, file=sys.stderr)
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

coverage.txt

@@ -0,0 +1,360 @@
+Test: check-against-debootstrap-dist
+Dists: any
+Variants: minbase buildd -
+Needs-Root: true
+
+Test: as-debootstrap-unshare-wrapper
+Needs-QEMU: true
+
+Test: help
+
+Test: man
+
+Test: version
+
+Test: create-directory
+Needs-Root: true
+
+Test: unshare-as-root-user
+Needs-Root: true
+
+Test: dist-using-codename
+Dists: any
+
+Test: fail-without-etc-subuid
+Needs-QEMU: true
+
+Test: fail-without-username-in-etc-subuid
+Needs-QEMU: true
+
+Test: unshare-as-root-user-inside-chroot
+Needs-Root: true
+
+Test: root-mode-inside-chroot
+Needs-Root: true
+
+Test: root-mode-inside-unshare-chroot
+Needs-QEMU: true
+
+Test: root-without-cap-sys-admin
+Needs-Root: true
+
+Test: mount-is-missing
+Needs-QEMU: true
+
+Test: check-for-bit-by-bit-identical-format-output
+Needs-QEMU: true
+Formats: tar squashfs ext2
+Variants: essential apt minbase buildd - standard
+Skip-If:
+ variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
+ variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
+ fmt == "squashfs" and dist == "oldstable" # squashfs-tools-ng is not available
+ fmt == "ext2" and dist == "oldstable" # genext2fs does not support SOURCE_DATE_EPOCH
+
+Test: tarfilter-idshift
+Needs-QEMU: true
+Skip-If: dist == "oldstable" # python3 tarfile module does not preserve xattrs
+
+Test: progress-bars-on-fake-tty
+
+Test: debug-output-on-fake-tty
+
+Test: existing-empty-directory
+Needs-Root: true
+
+Test: existing-directory-with-lost-found
+Needs-Root: true
+
+Test: fail-installing-to-non-empty-lost-found
+
+Test: fail-installing-to-non-empty-target-directory
+
+Test: missing-device-nodes-outside-the-chroot
+Needs-QEMU: true
+
+Test: missing-dev-sys-proc-inside-the-chroot
+Needs-QEMU: true
+
+Test: chroot-directory-not-accessible-by-apt-user
+Needs-Root: true
+
+Test: cwd-directory-not-accessible-by-unshared-user
+Needs-QEMU: true
+
+Test: create-gzip-compressed-tarball
+Needs-QEMU: true
+
+Test: custom-tmpdir
+Needs-QEMU: true
+
+Test: xz-compressed-tarball
+
+Test: directory-ending-in-tar
+Modes: root
+Needs-Root: true
+
+Test: auto-mode-without-unshare-capabilities
+Needs-QEMU: true
+
+Test: fail-with-missing-lz4
+
+Test: fail-with-path-with-quotes
+
+Test: create-tarball-with-tmp-mounted-nodev
+Needs-QEMU: true
+
+Test: read-from-stdin-write-to-stdout
+
+Test: supply-components-manually
+Modes: root
+Needs-Root: true
+
+Test: stable-default-mirror
+Needs-QEMU: true
+
+Test: pass-distribution-but-implicitly-write-to-stdout
+Needs-QEMU: true
+
+Test: aspcud-apt-solver
+
+Test: mirror-is-stdin
+
+Test: copy-mirror
+Needs-QEMU: true
+
+Test: file-mirror
+Needs-QEMU: true
+
+Test: file-mirror-automount-hook
+Modes: root unshare fakechroot
+Needs-QEMU: true
+
+Test: mirror-is-deb
+
+Test: mirror-is-real-file
+
+Test: deb822-1-2
+Modes: root
+Needs-Root: true
+
+Test: deb822-2-2
+Modes: root
+Needs-Root: true
+
+Test: automatic-mirror-from-suite
+Needs-QEMU: true
+
+Test: invalid-mirror
+
+Test: fail-installing-to-root
+Modes: root
+Needs-Root: true
+
+Test: fail-installing-to-existing-file
+Modes: root
+Needs-Root: true
+
+Test: arm64-without-qemu-support
+Needs-QEMU: true
+Skip-If: hostarch != "amd64"
+
+Test: i386-which-can-be-executed-without-qemu
+Needs-QEMU: true
+Skip-If:
+ hostarch != "amd64"
+ not run_ma_same_tests
+
+Test: include-libmagic-mgc-arm64
+Needs-Root: true
+Skip-If:
+ hostarch != "amd64"
+ not run_ma_same_tests
+
+Test: include-libmagic-mgc-arm64-with-multiple-arch-options
+Needs-Root: true
+Skip-If:
+ hostarch != "amd64"
+ not run_ma_same_tests
+
+Test: aptopt
+Needs-Root: true
+
+Test: keyring
+Needs-QEMU: true
+
+Test: keyring-overwrites
+Needs-Root: true
+
+Test: signed-by-without-host-keys
+Needs-QEMU: true
+
+Test: ascii-armored-keys
+Needs-QEMU: true
+
+Test: signed-by-with-host-keys
+Needs-Root: true
+
+Test: dpkgopt
+Needs-Root: true
+
+Test: include
+Needs-Root: true
+
+Test: multiple-include
+Needs-Root: true
+
+Test: include-with-multiple-apt-sources
+Needs-Root: true
+
+Test: essential-hook
+Needs-Root: true
+
+Test: customize-hook
+Needs-Root: true
+
+Test: failing-customize-hook
+Needs-Root: true
+
+Test: sigint-during-customize-hook
+Needs-Root: true
+
+Test: hook-directory
+Needs-Root: true
+
+Test: eatmydata-via-hook-dir
+Needs-Root: true
+
+Test: special-hooks-using-helpers
+Needs-Root: true
+
+Test: special-hooks-using-helpers-and-env-vars
+Needs-Root: true
+
+Test: special-hooks-with-mode-mode
+Modes: root unshare fakechroot
+Needs-QEMU: true
+
+Test: debootstrap-no-op-options
+Needs-Root: true
+
+Test: verbose
+Needs-Root: true
+
+Test: debug
+Needs-Root: true
+
+Test: quiet
+Needs-Root: true
+
+Test: logfile
+Needs-Root: true
+
+Test: without-etc-resolv-conf-and-etc-hostname
+Needs-QEMU: true
+
+Test: preserve-mode-of-etc-resolv-conf-and-etc-hostname
+Modes: root
+Needs-QEMU: true
+
+Test: not-having-to-install-apt-in-include-because-a-hook-did-it-before
+
+Test: remove-start-stop-daemon-and-policy-rc-d-in-hook
+
+Test: skip-start-stop-daemon-policy-rc
+
+Test: skip-mount
+Modes: unshare
+
+Test: compare-output-with-pre-seeded-var-cache-apt-archives
+Needs-QEMU: true
+Variants: any
+Skip-If:
+ variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
+ variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
+
+Test: create-directory-dry-run
+
+Test: create-tarball-dry-run
+Variants: any
+Modes: any
+
+Test: unpack-doc-debian
+Needs-QEMU: true
+Modes: any
+Variants: extract
+
+Test: install-doc-debian
+Modes: chrootless
+Variants: custom
+
+Test: chrootless
+Variants: essential
+Modes: chrootless
+Needs-Root: true
+Skip-If:
+ dist in ["oldstable", "stable"]
+
+Test: chrootless-fakeroot
+Variants: essential
+Modes: chrootless
+Needs-QEMU: true
+Skip-If:
+ dist in ["oldstable", "stable"]
+
+Test: chrootless-foreign
+Variants: essential
+Modes: chrootless
+Skip-If:
+ dist in ["oldstable", "stable"]
+ hostarch != "amd64"
+ not run_ma_same_tests
+Needs-QEMU: true
+
+Test: install-doc-debian-and-output-tarball
+Variants: custom
+Modes: chrootless
+
+Test: install-doc-debian-and-test-hooks
+Variants: custom
+Modes: chrootless
+
+Test: install-libmagic-mgc-on-arm64
+Skip-If:
+ hostarch != "amd64"
+ not have_binfmt
+
+Test: install-busybox-based-sub-essential-system
+Needs-Root: true
+
+Test: create-arm64-tarball
+Modes: root unshare fakechroot
+Skip-If:
+ hostarch != "amd64"
+ mode == "fakechroot" and not run_ma_same_tests
+ not have_binfmt
+
+Test: no-sbin-in-path
+Modes: fakechroot
+
+Test: dev-ptmx
+Modes: root unshare
+Needs-QEMU: true
+
+Test: error-if-stdout-is-tty
+
+Test: variant-custom-timeout
+
+Test: include-deb-file
+
+Test: unshare-include-deb
+Modes: unshare
+Needs-QEMU: true
+
+Test: pivot_root
+Modes: root unshare
+Needs-QEMU: true
+
+Test: jessie-or-older
+Needs-QEMU: true
+Variants: essential apt minbase

hooks/busybox/extract00.sh

@@ -1,6 +1,10 @@
 #!/bin/sh
 
-set -exu
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
 
 rootdir="$1"
 

hooks/busybox/setup00.sh

@@ -1,6 +1,10 @@
 #!/bin/sh
 
-set -exu
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
 
 rootdir="$1"
 

hooks/eatmydata/customize.sh

@@ -1,6 +1,10 @@
 #!/bin/sh
 
-set -exu
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
 
 rootdir="$1"
 
@@ -9,7 +13,7 @@ if [ -e "$rootdir/var/lib/dpkg/arch" ]; then
 else
 	chrootarch=$(dpkg --print-architecture)
 fi
-libdir="/usr/lib/$(dpkg-architecture -a $chrootarch -q DEB_HOST_MULTIARCH)"
+libdir="/usr/lib/$(dpkg-architecture -a "$chrootarch" -q DEB_HOST_MULTIARCH)"
 
 # if eatmydata was actually installed properly, then we are not removing
 # anything here

hooks/eatmydata/extract.sh

@@ -1,6 +1,10 @@
 #!/bin/sh
 
-set -exu
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
 
 rootdir="$1"
 
@@ -10,8 +14,10 @@ else
 	chrootarch=$(dpkg --print-architecture)
 fi
 
-eval $(apt-config shell trusted Dir::Etc::trusted/f)
-eval $(apt-config shell trustedparts Dir::Etc::trustedparts/d)
+trusted=
+eval "$(apt-config shell trusted Dir::Etc::trusted/f)"
+trustedparts=
+eval "$(apt-config shell trustedparts Dir::Etc::trustedparts/d)"
 tmpfile=$(mktemp --tmpdir="$rootdir/tmp")
 cat << END > "$tmpfile"
 Apt::Architecture "$chrootarch";
@@ -25,8 +31,8 @@ END
 # nothing will be printed for them
 tmpdir=$(mktemp --directory --tmpdir="$rootdir/tmp")
 env --chdir="$tmpdir" APT_CONFIG="$tmpfile" apt-get download --print-uris eatmydata libeatmydata1 \
-	| sed -ne "s/^'\([^']\+\)'\s\+\([^\s]\+\)\s\+\([0-9]\+\)\s\+\(SHA256:[a-f0-9]\+\)$/\1 \2 \3 \4/p" \
-	| while read uri fname size hash; do
+	| sed -ne "s/^'\([^']\+\)'\s\+\(\S\+\)\s\+\([0-9]\+\)\s\+\(SHA256:[a-f0-9]\+\)$/\1 \2 \3 \4/p" \
+	| while read -r uri fname size hash; do
 		echo "processing $fname" >&2
 		if [ -e "$tmpdir/$fname" ]; then
 			echo "$tmpdir/$fname already exists" >&2
@@ -41,7 +47,7 @@ env --chdir="$tmpdir" APT_CONFIG="$tmpfile" apt-get download --print-uris eatmyd
 					| tar --directory="$rootdir/usr/bin" --strip-components=3 --extract --verbose ./usr/bin/eatmydata
 				;;
 			libeatmydata1_*_$chrootarch.deb)
-				libdir="/usr/lib/$(dpkg-architecture -a $chrootarch -q DEB_HOST_MULTIARCH)"
+				libdir="/usr/lib/$(dpkg-architecture -a "$chrootarch" -q DEB_HOST_MULTIARCH)"
 				mkdir -p "$rootdir$libdir"
 				dpkg-deb --fsys-tarfile "$tmpdir/$fname" \
 					| tar --directory="$rootdir$libdir" --strip-components=4 --extract --verbose --wildcards ".$libdir/libeatmydata.so*"

hooks/file-mirror-automount/customize00.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+#
+# shellcheck disable=SC2086
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
+
+rootdir="$1"
+
+if [ ! -e "$rootdir/run/mmdebstrap/file-mirror-automount" ]; then
+	exit 0
+fi
+
+xargsopts="--null --no-run-if-empty -I {} --max-args=1"
+
+case $MMDEBSTRAP_MODE in
+	root|unshare)
+		echo "unmounting the following mountpoints:" >&2 ;;
+	*)
+		echo "removing the following directories:" >&2 ;;
+esac
+
+< "$rootdir/run/mmdebstrap/file-mirror-automount" \
+	xargs $xargsopts echo "    $rootdir/{}"
+
+case $MMDEBSTRAP_MODE in
+	root|unshare)
+		< "$rootdir/run/mmdebstrap/file-mirror-automount" \
+			xargs $xargsopts umount "$rootdir/{}"
+		;;
+	*)
+		< "$rootdir/run/mmdebstrap/file-mirror-automount" \
+			xargs $xargsopts rm -r "$rootdir/{}"
+		;;
+esac
+
+rm "$rootdir/run/mmdebstrap/file-mirror-automount"
+rmdir --ignore-fail-on-non-empty "$rootdir/run/mmdebstrap"

hooks/file-mirror-automount/setup00.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
+
+rootdir="$1"
+
+# process all configured apt repositories
+env APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get indextargets --no-release-info --format '$(REPO_URI)' \
+	| sed -ne 's/^file:\/\+//p' \
+	| sort -u \
+	| while read -r path; do
+		mkdir -p "$rootdir/run/mmdebstrap"
+		case $MMDEBSTRAP_MODE in
+			root|unshare)
+				echo "bind-mounting /$path into the chroot" >&2
+				mkdir -p "$rootdir/$path"
+				mount -o ro,bind "/$path" "$rootdir/$path"
+				;;
+			*)
+				echo "copying /$path into the chroot" >&2
+				mkdir -p "$rootdir/$(dirname "$path")"
+				cp -av "/$path" "$rootdir/$(dirname "$path")"
+				;;
+		esac
+		printf '/%s\0' "$path" >> "$rootdir/run/mmdebstrap/file-mirror-automount"
+	done
+
+# process all files given via --include
+set -f # turn off pathname expansion
+IFS=',' # split by comma
+for pkg in $MMDEBSTRAP_INCLUDE; do
+	set +f; unset IFS
+	case $pkg in
+		./*|../*|/*) : ;; # we are interested in this case
+		*) continue ;; # not a file
+	esac
+	# undo escaping
+	pkg="$(printf '%s' "$pkg" | sed 's/%2C/,/g; s/%25/%/g')"
+	# check for existance
+	if [ ! -f "$pkg" ]; then
+		echo "$pkg does not exist" >&2
+		continue
+	fi
+	# make path absolute
+	pkg="$(realpath "$pkg")"
+	mkdir -p "$rootdir/run/mmdebstrap"
+	mkdir -p "$rootdir/$(dirname "$pkg")"
+	case $MMDEBSTRAP_MODE in
+		root|unshare)
+			echo "bind-mounting $pkg into the chroot" >&2
+			touch "$rootdir/$pkg"
+			mount -o bind "$pkg" "$rootdir/$pkg"
+			;;
+		*)
+			echo "copying $pkg into the chroot" >&2
+			cp -av "$pkg" "$rootdir/$pkg"
+			;;
+	esac
+	printf '/%s\0' "$pkg" >> "$rootdir/run/mmdebstrap/file-mirror-automount"
+done
+set +f; unset IFS

hooks/jessie-or-older/extract00.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
+
+TARGET="$1"
+
+for f in available diversions cmethopt; do
+	if [ ! -e "$TARGET/var/lib/dpkg/$f" ]; then
+		touch "$TARGET/var/lib/dpkg/$f"
+	fi
+done
+
+if [ -z "${MMDEBSTRAP_ESSENTIAL+x}" ]; then
+	MMDEBSTRAP_ESSENTIAL=
+	for f in "$TARGET/var/cache/apt/archives/"*.deb; do
+		[ -f "$f" ] || continue
+		f="${f#"$TARGET"}"
+		MMDEBSTRAP_ESSENTIAL="$MMDEBSTRAP_ESSENTIAL $f"
+	done
+fi
+
+fname_base_passwd=
+fname_base_files=
+fname_dpkg=
+for pkg in $MMDEBSTRAP_ESSENTIAL; do
+	pkgname=$(dpkg-deb --show --showformat='${Package}' "$TARGET/$pkg")
+	# shellcheck disable=SC2034
+	case $pkgname in
+		base-passwd) fname_base_passwd=$pkg;;
+		base-files)  fname_base_files=$pkg;;
+		dpkg)        fname_dpkg=$pkg;;
+	esac
+done
+
+for var in base_passwd base_files dpkg; do
+	eval 'val=$fname_'"$var"
+	[ -z "$val" ] && continue
+	chroot "$TARGET" dpkg --install --force-depends "$val"
+done
+
+# shellcheck disable=SC2086
+chroot "$TARGET" dpkg --unpack --force-depends $MMDEBSTRAP_ESSENTIAL
+
+chroot "$TARGET" dpkg --configure --pending

hooks/merged-usr/essential00.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+        set -x
+fi
+
+TARGET="$1"
+
+if [ "${MMDEBSTRAP_MODE:-}" = "chrootless" ]; then
+	APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install \
+		-oDPkg::Options::=--force-not-root \
+		-oDPkg::Options::=--force-script-chrootless \
+		-oDPkg::Options::=--root="$TARGET" \
+		-oDPkg::Options::=--log="$TARGET/var/log/dpkg.log" \
+		usr-is-merged
+	export DPKG_ROOT="$TARGET"
+	dpkg-query --showformat '${db:Status-Status}\n' --show usr-is-merged | grep -q '^installed$'
+	dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
+	dpkg --compare-versions "1" "lt" "$(dpkg-query --showformat '${Version}\n' --show usr-is-merged)"
+else
+	APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$TARGET" usr-is-merged
+	chroot "$TARGET" dpkg-query --showformat '${db:Status-Status}\n' --show usr-is-merged | grep -q '^installed$'
+	chroot "$TARGET" dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
+	dpkg --compare-versions "1" "lt" "$(chroot "$TARGET" dpkg-query --showformat '${Version}\n' --show usr-is-merged)"
+fi

hooks/merged-usr/setup00.sh

@@ -39,22 +39,23 @@
 # out merged-/usr is bad from the dpkg point-of-view and completely opposite of
 # the vision with which in mind I wrote mmdebstrap.
 
-set -exu
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
 
 TARGET="$1"
 
-if [ -e "$TARGET/var/lib/dpkg/arch" ]; then
-	ARCH=$(head -1 "$TARGET/var/lib/dpkg/arch")
-else
-	ARCH=$(dpkg --print-architecture)
-fi
+ARCH=$(dpkg --print-architecture)
+eval "$(APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-config shell ARCH Apt::Architecture)"
 
 if [ -e /usr/share/debootstrap/functions ]; then
+	# shellcheck disable=SC1091
 	. /usr/share/debootstrap/functions
-	doing_variant () { [ $1 != "buildd" ]; }
+	doing_variant () { [ "$1" != "buildd" ]; }
+	# shellcheck disable=SC2034
 	MERGED_USR="yes"
-	# until https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/48 gets merged
-	link_dir=""
 	setup_merged_usr
 else
 	link_dir=""
@@ -79,3 +80,34 @@ else
 		mkdir -p "$TARGET/usr/$dir"
 	done
 fi
+
+# now install an empty "usr-is-merged" package to avoid installing the
+# usrmerge package on this system even after init-system-helpers starts
+# depending on "usrmerge | usr-is-merged".
+#
+# This package will not end up in the final chroot because the essential
+# hook replaces it with the actual usr-is-merged package from src:usrmerge.
+
+tmpdir=$(mktemp --directory --tmpdir="$TARGET/tmp")
+mkdir -p "$tmpdir/usr-is-merged/DEBIAN"
+
+cat << END > "$tmpdir/usr-is-merged/DEBIAN/control"
+Package: usr-is-merged
+Priority: optional
+Section: oldlibs
+Maintainer: Johannes Schauer Marin Rodrigues <josch@debian.org>
+Architecture: all
+Multi-Arch: foreign
+Source: mmdebstrap-dummy-usr-is-merged
+Version: 1
+Description: dummy package created by mmdebstrap merged-usr setup hook
+ This package was generated and installed by the mmdebstrap merged-usr
+ setup hook at /usr/share/mmdebstrap/hooks/merged-usr.
+ .
+ If this package is installed in the final chroot, then this is a bug
+ in mmdebstrap. Please report: https://gitlab.mister-muffin.de/josch/mmdebstrap
+END
+dpkg-deb --build "$tmpdir/usr-is-merged" "$tmpdir/usr-is-merged.deb"
+dpkg --root="$TARGET" --log="$TARGET/var/log/dpkg.log" --install "$tmpdir/usr-is-merged.deb"
+rm "$tmpdir/usr-is-merged.deb" "$tmpdir/usr-is-merged/DEBIAN/control"
+rmdir "$tmpdir/usr-is-merged/DEBIAN" "$tmpdir/usr-is-merged" "$tmpdir"

hooks/no-merged-usr/essential00.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+        set -x
+fi
+
+TARGET="$1"
+
+APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$TARGET" usr-is-merged
+
+chroot "$TARGET" dpkg-query --showformat '${db:Status-Status}\n' --show usr-is-merged | grep -q '^installed$'
+chroot "$TARGET" dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
+dpkg --compare-versions "1" "lt" "$(chroot "$TARGET" dpkg-query --showformat '${Version}\n' --show usr-is-merged)"

hooks/no-merged-usr/setup00.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+#
+# mmdebstrap does have a --no-merged-usr option but only as a no-op for
+# debootstrap compatibility
+#
+# Using this hook script, you can emulate what debootstrap does to set up
+# a system without merged-/usr even after the essential init-system-helpers
+# package added a dependency on "usrmerge | usr-is-merged". By installing
+# a dummy usr-is-merged package, it avoids pulling in the dependencies of
+# the usrmerge package.
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+        set -x
+fi
+
+TARGET="$1"
+
+echo "Warning: starting with Debian 12 (Bookworm), systems without merged-/usr are not supported anymore" >&2
+
+echo "this system will not be supported in the future" > "$TARGET/etc/unsupported-skip-usrmerge-conversion"
+
+# now install an empty "usr-is-merged" package to avoid installing the
+# usrmerge package on this system even after init-system-helpers starts
+# depending on "usrmerge | usr-is-merged".
+#
+# This package will not end up in the final chroot because the essential
+# hook replaces it with the actual usr-is-merged package from src:usrmerge.
+
+tmpdir=$(mktemp --directory --tmpdir="$TARGET/tmp")
+mkdir -p "$tmpdir/usr-is-merged/DEBIAN"
+
+cat << END > "$tmpdir/usr-is-merged/DEBIAN/control"
+Package: usr-is-merged
+Priority: optional
+Section: oldlibs
+Maintainer: Johannes Schauer Marin Rodrigues <josch@debian.org>
+Architecture: all
+Multi-Arch: foreign
+Source: mmdebstrap-dummy-usr-is-merged
+Version: 1
+Description: dummy package created by mmdebstrap no-merged-usr setup hook
+ This package was generated and installed by the mmdebstrap no-merged-usr
+ setup hook at /usr/share/mmdebstrap/hooks/no-merged-usr.
+ .
+ If this package is installed in the final chroot, then this is a bug
+ in mmdebstrap. Please report: https://gitlab.mister-muffin.de/josch/mmdebstrap
+END
+dpkg-deb --build "$tmpdir/usr-is-merged" "$tmpdir/usr-is-merged.deb"
+dpkg --root="$TARGET" --log="$TARGET/var/log/dpkg.log" --install "$tmpdir/usr-is-merged.deb"
+rm "$tmpdir/usr-is-merged.deb" "$tmpdir/usr-is-merged/DEBIAN/control"
+rmdir "$tmpdir/usr-is-merged/DEBIAN" "$tmpdir/usr-is-merged" "$tmpdir"

make_mirror.sh

@@ -58,11 +58,9 @@ deletecache() {
 				;;
 		esac
 	done
-	if [ -e $dir/debian-*.qcow ]; then
-		rm --one-file-system "$dir"/debian-*.qcow
-	else
-		echo "does not exist: $dir/debian-*.qcow" >&2
-	fi
+	for f in "$dir/debian-"*.qcow; do
+		rm --one-file-system "$f"
+	done
 	if [ -e "$dir/debian/pool/main" ]; then
 		rm --one-file-system --recursive "$dir/debian/pool/main"
 	else
@@ -103,7 +101,7 @@ get_oldaptnames() {
 	xz -dc "$1/$2" \
 		| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename '' \
 		| paste -sd "    \n" \
-		| while read name ver arch fname; do
+		| while read -r name ver arch fname; do
 			if [ ! -e "$1/$fname" ]; then
 				continue
 			fi
@@ -131,7 +129,7 @@ get_newaptnames() {
 	xz -dc "$1/$2" \
 		| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename,SHA256 '' \
 		| paste -sd "     \n" \
-		| while read name ver arch fname hash; do
+		| while read -r name ver arch fname hash; do
 			# sanity check for the hash because sometimes the
 			# archive switches the hash algorithm
 			if [ "${#hash}" -ne 64 ]; then
@@ -150,7 +148,7 @@ get_newaptnames() {
 				# since we move hardlinks around, the same hardlink might've been
 				# moved already into the same place by another distribution.
 				# mv(1) refuses to copy A to B if both are hardlinks of each other.
-				if [ "$aptname" -ef "$1/$fname" ]; then
+				if [ -e "$aptname" ] && [ -e "$1/$fname" ] && [ "$(stat -c "%d %i" "$aptname")" = "$(stat -c "%d %i" "$1/$fname")" ]; then
 					# both files are already the same so we just need to
 					# delete the source
 					rm "$aptname"
@@ -232,7 +230,7 @@ Acquire::https::Dl-Limit "1000";
 Acquire::Retries "5";
 END
 
-	> "$rootdir/var/lib/dpkg/status"
+	: > "$rootdir/var/lib/dpkg/status"
 
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update
 
@@ -264,8 +262,16 @@ END
 			--or --field=Priority important --or --field=Priority standard \
 			\))
 
-	pkgs="$(echo $pkgs) build-essential busybox gpg eatmydata"
+	pkgs="$pkgs build-essential busybox gpg eatmydata"
 
+	# we need usr-is-merged to simulate debootstrap behaviour for all dists
+	# starting from Debian 12 (Bullseye)
+	case "$dist" in
+		oldstable|stable) : ;;
+		*) pkgs="$pkgs usr-is-merged usrmerge" ;;
+	esac
+
+	# shellcheck disable=SC2086
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs
 
 	# to be able to also test gpg verification, we need to create a mirror
@@ -329,7 +335,7 @@ END
 	# new one anymore
 	comm -23 "$rootdir/oldaptnames" "$rootdir/newaptnames" | xargs --delimiter="\n" --no-run-if-empty rm
 	# now the apt cache should be empty
-	if [ ! -z "$(ls -1qA "$rootdir/var/cache/apt/archives/")" ]; then
+	if [ -n "$(ls -1qA "$rootdir/var/cache/apt/archives/")" ]; then
 		echo "$rootdir/var/cache/apt/archives not empty:"
 		ls -la "$rootdir/var/cache/apt/archives/"
 		exit 1
@@ -385,7 +391,6 @@ components=main
 : "${DEFAULT_DIST:=unstable}"
 : "${HAVE_QEMU:=yes}"
 : "${RUN_MA_SAME_TESTS:=yes}"
-: "${HAVE_PROOT:=yes}"
 # by default, use the mmdebstrap executable in the current directory
 : "${CMD:=./mmdebstrap}"
 
@@ -413,7 +418,7 @@ fi
 for nativearch in $arches; do
 	for dist in oldstable stable testing unstable; do
 		# non-host architectures are only downloaded for $DEFAULT_DIST
-		if [ $nativearch != $HOSTARCH ] && [ $DEFAULT_DIST != $dist ]; then
+		if [ "$nativearch" != "$HOSTARCH" ] && [ "$DEFAULT_DIST" != "$dist" ]; then
 			continue
 		fi
 		# we need a first pass without updates and security patches
@@ -458,12 +463,10 @@ cleanuptmpdir() {
 	if [ ! -e "$tmpdir" ]; then
 		return
 	fi
-	for f in "$tmpdir/extlinux.conf" \
-		"$tmpdir/worker.sh" \
+	for f in "$tmpdir/worker.sh" \
 		"$tmpdir/mini-httpd" "$tmpdir/hosts" \
 		"$tmpdir/debian-chroot.tar" \
-		"$tmpdir/mmdebstrap.service" \
-		"$tmpdir/debian-$DEFAULT_DIST.img"; do
+		"$tmpdir/mmdebstrap.service"; do
 		if [ ! -e "$f" ]; then
 			echo "does not exist: $f" >&2
 			continue
@@ -473,16 +476,17 @@ cleanuptmpdir() {
 	rmdir "$tmpdir"
 }
 
-export SOURCE_DATE_EPOCH=$(date --date="$(grep-dctrl -s Date -n '' "$newmirrordir/dists/$DEFAULT_DIST/Release")" +%s)
+SOURCE_DATE_EPOCH="$(date --date="$(grep-dctrl -s Date -n '' "$newmirrordir/dists/$DEFAULT_DIST/Release")" +%s)"
+export SOURCE_DATE_EPOCH
 
 if [ "$HAVE_QEMU" = "yes" ]; then
 	case "$HOSTARCH" in
-		amd64|i386)
+		amd64|i386|arm64)
 			# okay
 			;;
 		*)
-			echo "qemu support is only available on amd64 and i386" >&2
-			echo "because syslinux is only available on those arches" >&2
+			echo "qemu support is only available on amd64, i386 and arm64" >&2
+			echo "because grub is only available on those arches" >&2
 			exit 1
 			;;
 	esac
@@ -496,13 +500,10 @@ if [ "$HAVE_QEMU" = "yes" ]; then
 	tmpdir="$(mktemp -d)"
 	trap "cleanuptmpdir; cleanup_newcachedir" EXIT INT TERM
 
-	pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils
+	pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,grub-efi
 	if [ "$DEFAULT_DIST" != "oldstable" ]; then
 		pkgs="$pkgs,squashfs-tools-ng,genext2fs"
 	fi
-	if [ "$HAVE_PROOT" = "yes" ]; then
-		pkgs="$pkgs,proot"
-	fi
 	if [ ! -e ./mmdebstrap ]; then
 		pkgs="$pkgs,mmdebstrap"
 	fi
@@ -527,21 +528,12 @@ if [ "$HAVE_QEMU" = "yes" ]; then
 	else
 		arches=$HOSTARCH
 	fi
-	$CMD --variant=apt --architectures=$arches --include="$pkgs" \
+	$CMD --variant=apt --architectures="$arches" --include="$pkgs" \
 		--aptopt='Acquire::http::Dl-Limit "1000"' \
 		--aptopt='Acquire::https::Dl-Limit "1000"' \
 		--aptopt='Acquire::Retries "5"' \
-		$DEFAULT_DIST - "$mirror" > "$tmpdir/debian-chroot.tar"
+		"$DEFAULT_DIST" - "$mirror" > "$tmpdir/debian-chroot.tar"
 
-	cat << END > "$tmpdir/extlinux.conf"
-default linux
-timeout 0
-
-label linux
-kernel /vmlinuz
-append initrd=/initrd.img root=/dev/vda1 rw console=ttyS0,115200
-serial 0 115200
-END
 	cat << END > "$tmpdir/mmdebstrap.service"
 [Unit]
 Description=mmdebstrap worker script
@@ -622,15 +614,32 @@ END
 	# buildd variant will not be 400MB but 1.3GB large and needs a 10G
 	# disk.
 	if [ -z ${DISK_SIZE+x} ]; then
-		DISK_SIZE=3G
+		DISK_SIZE=10G
 	fi
-	guestfish -N "$tmpdir/debian-$DEFAULT_DIST.img"=disk:$DISK_SIZE -- \
-		part-disk /dev/sda mbr : \
-		mkfs ext2 /dev/sda1 : \
-		mount /dev/sda1 / : \
-		tar-in "$tmpdir/debian-chroot.tar" / : \
+	case "$HOSTARCH" in
+		amd64) GRUB_TARGET=x86_64-efi;;
+		i386) GRUB_TARGET=i386-efi;;
+		arm64) GRUB_TARGET=arm64-efi;;
+	esac
+	case "$HOSTARCH" in
+		arm64) SERIAL="loglevel=3 console=tty0 console=ttyAMA0,115200n8" ;;
+		*) SERIAL="loglevel=3 console=tty0 console=ttyS0,115200n8" ;;
+	esac
+	guestfish -- \
+		disk-create "$newcachedir/debian-$DEFAULT_DIST.qcow" qcow2 "$DISK_SIZE" : \
+		add-drive "$newcachedir/debian-$DEFAULT_DIST.qcow" format:qcow2 : \
+		launch : \
+		part-init /dev/sda gpt : \
+		part-add /dev/sda primary 8192 262144 : \
+		part-add /dev/sda primary 262145 -34 : \
+		part-set-gpt-type /dev/sda 1 C12A7328-F81F-11D2-BA4B-00A0C93EC93B  : \
+		mkfs ext2 /dev/sda2 : \
+		mount /dev/sda2 / : \
+		tar-in "$tmpdir/debian-chroot.tar" / xattrs:true : \
+		mkdir-p /boot/efi : \
+		mkfs vfat /dev/sda1 : \
+		mount /dev/sda1 /boot/efi : \
 		command /sbin/ldconfig : \
-		copy-in "$tmpdir/extlinux.conf" / : \
 		mkdir-p /etc/systemd/system/multi-user.target.wants : \
 		ln-s ../mmdebstrap.service /etc/systemd/system/multi-user.target.wants/mmdebstrap.service : \
 		copy-in "$tmpdir/mmdebstrap.service" /etc/systemd/system/ : \
@@ -638,15 +647,16 @@ END
 		copy-in "$tmpdir/mini-httpd" /etc/default : \
 		copy-in "$tmpdir/hosts" /etc/ : \
 		touch /mmdebstrap-testenv : \
-		upload /usr/lib/EXTLINUX/mbr.bin /mbr.bin : \
-		copy-file-to-device /mbr.bin /dev/sda size:440 : \
-		rm /mbr.bin : \
-		extlinux / : \
+		command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda2) / ext4 errors=remount-ro 0 1 > /etc/fstab'" : \
+		command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda1) /boot/efi vfat errors=remount-ro 0 2 >> /etc/fstab'" : \
+		command "sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=/GRUB_CMDLINE_LINUX_DEFAULT=\"biosdevname=0 net.ifnames=0 consoleblank=0 rw $SERIAL\"/' /etc/default/grub" : \
+		command "update-initramfs -u" : \
+		command "grub-mkconfig -o /boot/grub/grub.cfg" : \
+		command "grub-install /dev/sda --target=$GRUB_TARGET --no-nvram --force-extra-removable --no-floppy --modules=part_gpt --grub-mkdevicemap=/boot/grub/device.map" : \
 		sync : \
+		umount /boot/efi : \
 		umount / : \
-		part-set-bootable /dev/sda 1 true : \
 		shutdown
-	qemu-img convert -O qcow2 "$tmpdir/debian-$DEFAULT_DIST.img" "$newcachedir/debian-$DEFAULT_DIST.qcow"
 	cleanuptmpdir
 	trap "cleanup_newcachedir" EXIT INT TERM
 fi
@@ -654,15 +664,23 @@ fi
 mirror="http://127.0.0.1/debian"
 for dist in oldstable stable testing unstable; do
 	for variant in minbase buildd -; do
-		echo "running debootstrap --no-merged-usr --variant=$variant $dist \${TEMPDIR} $mirror"
+		echo "running debootstrap --variant=$variant $dist \${TEMPDIR} $mirror"
 		cat << END > shared/test.sh
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
+echo "SOURCE_DATE_EPOCH=\$SOURCE_DATE_EPOCH"
 tmpdir="\$(mktemp -d)"
 chmod 755 "\$tmpdir"
-debootstrap --no-merged-usr --variant=$variant $dist "\$tmpdir" $mirror
+case "$dist" in
+	oldstable|stable)
+		debootstrap --no-merged-usr --variant=$variant $dist "\$tmpdir" $mirror
+		;;
+	*)
+		debootstrap --merged-usr --variant=$variant $dist "\$tmpdir" $mirror
+		;;
+esac
 tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system --xattrs -C "\$tmpdir" -c . > "$newcache/debian-$dist-$variant.tar"
 rm -r "\$tmpdir"
 END

mmdebstrap-autopkgtest-build-qemu

@@ -145,7 +145,7 @@ case "$nativearch" in
 		[ $BOOT = bios ] || [ $BOOT = efi ]
 		if [ $BOOT = bios ]; then
 			include="linux-image-686-pae grub-pc"
-			grub_target="i386-efi"
+			grub_target="i386-pc"
 		elif [ $BOOT = efi ]; then
 			include="linux-image-686-pae grub-efi"
 			grub_target="i386-efi"

run_qemu.sh

@@ -25,20 +25,45 @@ cleanup() {
 
 trap cleanup INT TERM EXIT
 
+ARCH=$(dpkg --print-architecture)
+case $ARCH in
+	i386)
+		MACHINE="accel=kvm:tcg"
+		CODE="/usr/share/OVMF/OVMF32_CODE_4M.secboot.fd"
+		QEMUARCH="i386"
+		;;
+	amd64)
+		MACHINE="accel=kvm:tcg"
+		CODE="/usr/share/OVMF/OVMF_CODE.fd"
+		QEMUARCH="x86_64"
+		;;
+	arm64)
+		MACHINE="type=virt,gic-version=host,accel=kvm"
+		CODE="/usr/share/AAVMF/AAVMF_CODE.fd"
+		QEMUARCH="aarch64"
+		;;
+	*) echo "qemu kvm not supported on $ARCH" >&2;;
+esac
+
 # the path to debian-$DEFAULT_DIST.qcow must be absolute or otherwise qemu will
 # look for the path relative to debian-$DEFAULT_DIST-overlay.qcow
-qemu-img create -f qcow2 -b "$(realpath $cachedir)/debian-$DEFAULT_DIST.qcow" -F qcow2 "$tmpdir/debian-$DEFAULT_DIST-overlay.qcow"
+qemu-img create -f qcow2 -b "$(realpath "$cachedir")/debian-$DEFAULT_DIST.qcow" -F qcow2 "$tmpdir/debian-$DEFAULT_DIST-overlay.qcow"
 # to connect to serial use:
 #   minicom -D 'unix#/tmp/ttyS0'
+#
+# or this (quit with ctrl+q):
+#   socat stdin,raw,echo=0,escape=0x11 unix-connect:/tmp/ttyS0
 ret=0
-timeout 20m qemu-system-x86_64 \
+timeout --foreground 40m qemu-system-"$QEMUARCH" \
+	-cpu host \
 	-no-user-config \
-	-M accel=kvm:tcg -m 1G -nographic \
+	-M "$MACHINE" -m 4G -nographic \
 	-object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \
 	-monitor unix:/tmp/monitor,server,nowait \
 	-serial unix:/tmp/ttyS0,server,nowait \
 	-serial unix:/tmp/ttyS1,server,nowait \
 	-net nic,model=virtio -net user \
+	-drive if=pflash,format=raw,unit=0,read-only=on,file="$CODE" \
 	-virtfs local,id=mmdebstrap,path="$(pwd)/shared",security_model=none,mount_tag=mmdebstrap \
 	-drive file="$tmpdir/debian-$DEFAULT_DIST-overlay.qcow",cache=unsafe,index=0,if=virtio \
 	>"$tmpdir/log" 2>&1 || ret=$?

tarfilter

@@ -43,17 +43,53 @@ class PaxFilterAction(argparse.Action):
         setattr(namespace, "paxfilter", items)
 
 
+class TransformAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = getattr(namespace, "trans", [])
+        # This function mimics what src/transform.c from tar does
+        if not values.startswith("s"):
+            raise ValueError("regex must start with an 's'")
+        if len(values) <= 4:
+            # minimum regex: s/x//
+            raise ValueError("invalid regex (too short)")
+        d = values[1]
+        if values.startswith(f"s{d}{d}"):
+            raise ValueError("empty regex")
+        values = values.removeprefix(f"s{d}")
+        flags = 0
+        if values.endswith(f"{d}i"):
+            # trailing flags
+            flags = re.IGNORECASE
+            values = values.removesuffix(f"{d}i")
+        # This regex only finds non-empty tokens.
+        # Finding empty tokens would require a variable length look-behind
+        # or \K in order to find escaped delimiters which is not supported by
+        # the python re module.
+        tokens = re.findall(rf"(?:\\[\\{d}]|[^{d}])+", values)
+        match len(tokens):
+            case 0:
+                raise ValueError("invalid regex: not enough terms")
+            case 1:
+                repl = ""
+            case 2:
+                repl = tokens[1]
+            case _:
+                raise ValueError("invalid regex: too many terms: %s" % tokens)
+        items.append((re.compile(tokens[0], flags), repl))
+        setattr(namespace, "trans", items)
+
+
 def main():
     parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
         description="""\
 Filters a tarball on standard input by the same rules as the dpkg --path-exclude
 and --path-include options and writes resulting tarball to standard output. See
-dpkg(1) for information on how these two options work in detail. Since this is
-meant for filtering tarballs storing a rootfs, notice that paths must be given
-as /path and not as ./path even though they might be stored as such in the
-tarball.
+dpkg(1) for information on how these two options work in detail. To reuse the
+exact same semantics as used by dpkg, paths must be given as /path and not as
+./path even though they might be stored as such in the tarball.
 
-Similarly, filter out unwanted pax extended headers. This is useful in cases
+Secondly, filter out unwanted pax extended headers. This is useful in cases
 where a tool only accepts certain xattr prefixes. For example tar2sqfs only
 supports SCHILY.xattr.user.*, SCHILY.xattr.trusted.* and
 SCHILY.xattr.security.* but not SCHILY.xattr.system.posix_acl_default.*.
@@ -65,41 +101,65 @@ Both types of options use Unix shell-style wildcards:
    [seq] matches any character in seq
   [!seq] matches any character not in seq
 
-Thirdly, strip leading directory components off of tar members. Just as with
+Thirdly, transform the path of tar members using a sed expression just as with
+GNU tar --transform.
+
+Fourthly, strip leading directory components off of tar members. Just as with
 GNU tar --strip-components, tar members that have less or equal components in
 their path are not passed through.
-"""
+
+Lastly, shift user id and group id of each entry by the value given by the
+--idshift argument. The resulting uid or gid must not be negative.
+""",
     )
     parser.add_argument(
         "--path-exclude",
         metavar="pattern",
         action=PathFilterAction,
-        help="Exclude path matching the given shell pattern.",
+        help="Exclude path matching the given shell pattern. "
+        "This option can be specified multiple times.",
     )
     parser.add_argument(
         "--path-include",
         metavar="pattern",
         action=PathFilterAction,
-        help="Re-include a pattern after a previous exclusion.",
+        help="Re-include a pattern after a previous exclusion. "
+        "This option can be specified multiple times.",
     )
     parser.add_argument(
         "--pax-exclude",
         metavar="pattern",
         action=PaxFilterAction,
-        help="Exclude pax header matching the given globbing pattern.",
+        help="Exclude pax header matching the given globbing pattern. "
+        "This option can be specified multiple times.",
     )
     parser.add_argument(
         "--pax-include",
         metavar="pattern",
         action=PaxFilterAction,
-        help="Re-include a pax header after a previous exclusion.",
+        help="Re-include a pax header after a previous exclusion. "
+        "This option can be specified multiple times.",
+    )
+    parser.add_argument(
+        "--transform",
+        "--xform",
+        metavar="EXPRESSION",
+        action=TransformAction,
+        help="Use sed replace EXPRESSION to transform file names. "
+        "This option can be specified multiple times.",
     )
     parser.add_argument(
         "--strip-components",
-        metavar="number",
+        metavar="NUMBER",
         type=int,
         help="Strip NUMBER leading components from file names",
     )
+    parser.add_argument(
+        "--idshift",
+        metavar="NUM",
+        type=int,
+        help="Integer value by which to shift the uid and gid of each entry",
+    )
     args = parser.parse_args()
     if (
         not hasattr(args, "pathfilter")
@@ -157,6 +217,8 @@ their path are not passed through.
                 continue
             if args.strip_components:
                 comps = member.name.split("/")
+                # just as with GNU tar, archive members with less or equal
+                # number of components are not passed through at all
                 if len(comps) <= args.strip_components:
                     continue
                 member.name = "/".join(comps[args.strip_components :])
@@ -165,6 +227,18 @@ their path are not passed through.
                 for k, v in member.pax_headers.items()
                 if not pax_filter_should_skip(k)
             }
+            if args.idshift:
+                if args.idshift < 0 and -args.idshift > member.uid:
+                    print("uid cannot be negative", file=sys.stderr)
+                    exit(1)
+                if args.idshift < 0 and -args.idshift > member.gid:
+                    print("gid cannot be negative", file=sys.stderr)
+                    exit(1)
+                member.uid += args.idshift
+                member.gid += args.idshift
+            if hasattr(args, "trans"):
+                for r, s in args.trans:
+                    member.name = r.sub(s, member.name)
             if member.isfile():
                 with in_tar.extractfile(member) as file:
                     out_tar.addfile(member, file)

taridshift

@@ -1,67 +0,0 @@
-#!/usr/bin/env python3
-#
-# This script is in the public domain
-#
-# Author: Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
-#
-# This script accepts a tarball on standard input and prints a tarball on
-# standard output with the same contents but all uid and gid ownership
-# information shifted by the value given as first command line argument.
-#
-# A tool like this should be written in C but libarchive has issues:
-# https://github.com/libarchive/libarchive/issues/587
-# https://github.com/libarchive/libarchive/pull/1288/ (needs 3.4.1)
-# Should these issues get fixed, then a good template is tarfilter.c in the
-# examples directory of libarchive.
-#
-# We are not using Perl either, because Archive::Tar slurps the whole tarball
-# into memory.
-#
-# We could also use Go but meh...
-# https://stackoverflow.com/a/59542307/784669
-
-import tarfile
-import sys
-import argparse
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="""\
-Accepts a tarball on standard input and prints a tarball on standard output
-with the same contents but all uid and gid ownership information shifted by the
-value given as first command line argument.
-"""
-    )
-    parser.add_argument(
-        "idshift",
-        metavar="NUM",
-        type=int,
-        help="Integer value by which to shift the uid and gid of each entry",
-    )
-    args = parser.parse_args()
-
-    # starting with Python 3.8, the default format became PAX_FORMAT, so this
-    # is only for compatibility with older versions of Python 3
-    with tarfile.open(fileobj=sys.stdin.buffer, mode="r|*") as in_tar, tarfile.open(
-        fileobj=sys.stdout.buffer, mode="w|", format=tarfile.PAX_FORMAT
-    ) as out_tar:
-        for member in in_tar:
-            if args.idshift < 0 and -args.idshift > member.uid:
-                print("uid cannot be negative", file=sys.stderr)
-                exit(1)
-            if args.idshift < 0 and -args.idshift > member.gid:
-                print("gid cannot be negative", file=sys.stderr)
-                exit(1)
-
-            member.uid += args.idshift
-            member.gid += args.idshift
-            if member.isfile():
-                with in_tar.extractfile(member) as file:
-                    out_tar.addfile(member, file)
-            else:
-                out_tar.addfile(member)
-
-
-if __name__ == "__main__":
-    main()

tests/aptopt

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot; rm -f /tmp/config" EXIT INT TERM
+echo 'Acquire::Languages "none";' > /tmp/config
+{{ CMD }} --mode=root --variant=apt --aptopt='Acquire::Check-Valid-Until "false"' --aptopt=/tmp/config {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+printf 'Acquire::Check-Valid-Until "false";\nAcquire::Languages "none";\n' | cmp /tmp/debian-chroot/etc/apt/apt.conf.d/99mmdebstrap -
+rm /tmp/debian-chroot/etc/apt/apt.conf.d/99mmdebstrap
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/arm64-without-qemu-support

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+apt-get remove --yes qemu-user-static binfmt-support qemu-user
+# the following is not necessary anymore since systemd-binfmt
+# successfully disables support upon removal of qemu-user with
+# the upload of src:systemd 251.2-4: https://bugs.debian.org/1012163
+#echo 0 > /proc/sys/fs/binfmt_misc/qemu-aarch64
+ret=0
+{{ CMD }} --mode={{ MODE }} --variant=apt --architectures=arm64 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }} || ret=$?
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi

tests/as-debootstrap-unshare-wrapper

@@ -0,0 +1,89 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+sysctl -w kernel.unprivileged_userns_clone=1
+adduser --gecos user --disabled-password user
+runuser -u user -- {{ CMD }} --variant=custom --mode=unshare --setup-hook='env container=lxc debootstrap unstable "$1" {{ MIRROR }}' - /tmp/debian-mm.tar {{ MIRROR }}
+
+mkdir /tmp/debian-mm
+tar --xattrs --xattrs-include='*' -C /tmp/debian-mm -xf /tmp/debian-mm.tar
+
+mkdir /tmp/debian-debootstrap
+tar --xattrs --xattrs-include='*' -C /tmp/debian-debootstrap -xf "cache/debian-unstable--.tar"
+
+# diff cannot compare device nodes, so we use tar to do that for us and then
+# delete the directory
+tar -C /tmp/debian-debootstrap -cf dev1.tar ./dev
+tar -C /tmp/debian-mm -cf dev2.tar ./dev
+cmp dev1.tar dev2.tar >&2
+rm dev1.tar dev2.tar
+rm -r /tmp/debian-debootstrap/dev /tmp/debian-mm/dev
+
+# remove downloaded deb packages
+rm /tmp/debian-debootstrap/var/cache/apt/archives/*.deb
+# remove aux-cache
+rm /tmp/debian-debootstrap/var/cache/ldconfig/aux-cache
+# remove logs
+rm /tmp/debian-debootstrap/var/log/dpkg.log \
+	/tmp/debian-debootstrap/var/log/bootstrap.log \
+	/tmp/debian-debootstrap/var/log/alternatives.log \
+	/tmp/debian-mm/var/log/bootstrap.log
+
+# clear out /run except for /run/lock
+find /tmp/debian-debootstrap/run/ -mindepth 1 -maxdepth 1 ! -name lock -print0 | xargs --no-run-if-empty -0 rm -r
+
+# debootstrap doesn't clean apt
+rm /tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_main_binary-{{ HOSTARCH }}_Packages \
+	/tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_Release \
+	/tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_Release.gpg
+
+rm /tmp/debian-debootstrap/etc/machine-id /tmp/debian-mm/etc/machine-id
+rm /tmp/debian-mm/var/cache/apt/archives/lock
+rm /tmp/debian-mm/var/lib/apt/lists/lock
+rm /tmp/debian-mm/var/lib/dpkg/arch
+
+# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
+# also needed for users that are created by systemd-sysusers before systemd 252
+# https://github.com/systemd/systemd/pull/24534
+for f in shadow shadow-; do
+	if ! cmp /tmp/debian-debootstrap/etc/$f /tmp/debian-mm/etc/$f >&2; then
+		echo patching /etc/$f >&2
+		awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-mm/etc/$f > /tmp/debian-mm/etc/$f.bak
+		cat /tmp/debian-mm/etc/$f.bak > /tmp/debian-mm/etc/$f
+		rm /tmp/debian-mm/etc/$f.bak
+	else
+		echo no difference for /etc/$f >&2
+	fi
+done
+
+# check if the file content differs
+diff --no-dereference --recursive /tmp/debian-debootstrap /tmp/debian-mm >&2
+
+# check permissions, ownership, symlink targets, modification times using tar
+# mtimes of directories created by mmdebstrap will differ, thus we equalize them first
+for d in etc/apt/preferences.d/ etc/apt/sources.list.d/ etc/dpkg/dpkg.cfg.d/ var/log/apt/; do
+	touch --date="@{{ SOURCE_DATE_EPOCH }}" /tmp/debian-debootstrap/$d /tmp/debian-mm/$d
+done
+# debootstrap never ran apt -- fixing permissions
+for d in ./var/lib/apt/lists/partial ./var/cache/apt/archives/partial; do
+	chroot /tmp/debian-debootstrap chmod 0700 $d
+	chroot /tmp/debian-debootstrap chown _apt:root $d
+done
+tar -C /tmp/debian-debootstrap --numeric-owner --xattrs --xattrs-include='*' --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/root1.tar .
+tar -C /tmp/debian-mm --numeric-owner --xattrs --xattrs-include='*' --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/root2.tar .
+tar --full-time --verbose -tf /tmp/root1.tar > /tmp/root1.tar.list
+tar --full-time --verbose -tf /tmp/root2.tar > /tmp/root2.tar.list
+# despite SOURCE_DATE_EPOCH and --clamp-mtime, the timestamps in the tarball
+# will slightly differ from each other in the sub-second precision (last
+# decimals) so the tarballs will not be identical, so we use diff to compare
+# content and tar to compare attributes
+diff -u /tmp/root1.tar.list /tmp/root2.tar.list >&2
+rm /tmp/root1.tar /tmp/root2.tar /tmp/root1.tar.list /tmp/root2.tar.list
+
+rm /tmp/debian-mm.tar
+rm -r /tmp/debian-debootstrap /tmp/debian-mm

tests/ascii-armored-keys

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+for f in /usr/share/keyrings/*.gpg; do
+	name=$(basename "$f" .gpg)
+	gpg --enarmor < "/usr/share/keyrings/$name.gpg" \
+		| sed 's/ PGP ARMORED FILE/ PGP PUBLIC KEY BLOCK/;/^Comment: /d' \
+		> "/etc/apt/trusted.gpg.d/$name.asc"
+done
+rm /etc/apt/trusted.gpg.d/*.gpg
+rm /usr/share/keyrings/*.gpg
+{{ CMD }} --mode=root --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+rm -r /tmp/debian-chroot.tar

tests/aspcud-apt-solver

@@ -0,0 +1,11 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+{{ CMD }} --mode={{ MODE }} --variant=custom \
+    --include "$(tr '\n' ',' < pkglist.txt)" \
+    --aptopt='APT::Solver "aspcud"' \
+    {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar | sort \
+    | grep -v '^./etc/apt/apt.conf.d/99mmdebstrap$' \
+    | diff -u tar1.txt -

tests/auto-mode-without-unshare-capabilities

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+adduser --gecos user --disabled-password user
+sysctl -w kernel.unprivileged_userns_clone=0
+runuser -u user -- {{ CMD }} --mode=auto --variant=apt {{ DIST }} /tmp/debian-chroot.tar.gz {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar.gz | sort | diff -u tar1.txt -
+rm /tmp/debian-chroot.tar.gz

tests/automatic-mirror-from-suite

@@ -0,0 +1,14 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+cat << HOSTS >> /etc/hosts
+127.0.0.1 deb.debian.org
+127.0.0.1 security.debian.org
+HOSTS
+{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+rm /tmp/debian-chroot.tar

tests/check-against-debootstrap-dist

@@ -0,0 +1,225 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+
+echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
+
+# we create the apt user ourselves or otherwise its uid/gid will differ
+# compared to the one chosen in debootstrap because of different installation
+# order in comparison to the systemd users
+# https://bugs.debian.org/969631
+# we cannot use useradd because passwd is not Essential:yes
+{{ CMD }} --variant={{ VARIANT }} --mode={{ MODE }} \
+	--essential-hook='case {{ DIST }} in oldstable|stable) if [ {{ VARIANT }} = - ]; then echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd; fi;; esac' \
+	"$(case {{ DIST }} in oldstable|stable) echo --merged-usr ;; *) echo --hook-dir=./hooks/merged-usr ;; esac)" \
+	{{ DIST }} /tmp/debian-{{ DIST }}-mm.tar {{ MIRROR }}
+
+mkdir /tmp/debian-{{ DIST }}-mm
+tar --xattrs --xattrs-include='*' -C /tmp/debian-{{ DIST }}-mm -xf /tmp/debian-{{ DIST }}-mm.tar
+rm /tmp/debian-{{ DIST }}-mm.tar
+
+mkdir /tmp/debian-{{ DIST }}-debootstrap
+tar --xattrs --xattrs-include='*' -C /tmp/debian-{{ DIST }}-debootstrap -xf "cache/debian-{{ DIST }}-{{ VARIANT }}.tar"
+
+# diff cannot compare device nodes, so we use tar to do that for us and then
+# delete the directory
+tar -C /tmp/debian-{{ DIST }}-debootstrap -cf /tmp/dev1.tar ./dev
+tar -C /tmp/debian-{{ DIST }}-mm -cf /tmp/dev2.tar ./dev
+ret=0
+cmp /tmp/dev1.tar /tmp/dev2.tar >&2 || ret=$?
+if [ "$ret" -ne 0 ]; then
+	if type diffoscope >/dev/null; then
+		diffoscope /tmp/dev1.tar /tmp/dev2.tar
+		exit 1
+	else
+		echo "no diffoscope installed" >&2
+	fi
+	if type base64 >/dev/null; then
+		base64 /tmp/dev1.tar
+		base64 /tmp/dev2.tar
+		exit 1
+	else
+		echo "no base64 installed" >&2
+	fi
+	if type xxd >/dev/null; then
+		xxd /tmp/dev1.tar
+		xxd /tmp/dev2.tar
+		exit 1
+	else
+		echo "no xxd installed" >&2
+	fi
+	exit 1
+fi
+rm /tmp/dev1.tar /tmp/dev2.tar
+rm -r /tmp/debian-{{ DIST }}-debootstrap/dev /tmp/debian-{{ DIST }}-mm/dev
+
+# remove downloaded deb packages
+rm /tmp/debian-{{ DIST }}-debootstrap/var/cache/apt/archives/*.deb
+# remove aux-cache
+rm /tmp/debian-{{ DIST }}-debootstrap/var/cache/ldconfig/aux-cache
+# remove logs
+rm /tmp/debian-{{ DIST }}-debootstrap/var/log/dpkg.log \
+	/tmp/debian-{{ DIST }}-debootstrap/var/log/bootstrap.log \
+	/tmp/debian-{{ DIST }}-debootstrap/var/log/alternatives.log
+# remove *-old files
+rm /tmp/debian-{{ DIST }}-debootstrap/var/cache/debconf/config.dat-old \
+	/tmp/debian-{{ DIST }}-mm/var/cache/debconf/config.dat-old
+rm /tmp/debian-{{ DIST }}-debootstrap/var/cache/debconf/templates.dat-old \
+	/tmp/debian-{{ DIST }}-mm/var/cache/debconf/templates.dat-old
+rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status-old \
+	/tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status-old
+# remove dpkg files
+rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/available
+rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/cmethopt
+# remove /var/lib/dpkg/arch
+rm /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/arch
+# since we installed packages directly from the .deb files, Priorities differ
+# thus we first check for equality and then remove the files
+chroot /tmp/debian-{{ DIST }}-debootstrap dpkg --list > /tmp/dpkg1
+chroot /tmp/debian-{{ DIST }}-mm dpkg --list > /tmp/dpkg2
+diff -u /tmp/dpkg1 /tmp/dpkg2 >&2
+rm /tmp/dpkg1 /tmp/dpkg2
+grep -v '^Priority: ' /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status > /tmp/status1
+grep -v '^Priority: ' /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status > /tmp/status2
+diff -u /tmp/status1 /tmp/status2 >&2
+rm /tmp/status1 /tmp/status2
+rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/dpkg/status /tmp/debian-{{ DIST }}-mm/var/lib/dpkg/status
+# debootstrap exposes the hosts's kernel version
+if [ -e /tmp/debian-{{ DIST }}-debootstrap/etc/apt/apt.conf.d/01autoremove-kernels ]; then
+	rm /tmp/debian-{{ DIST }}-debootstrap/etc/apt/apt.conf.d/01autoremove-kernels
+fi
+if [ -e /tmp/debian-{{ DIST }}-mm/etc/apt/apt.conf.d/01autoremove-kernels ]; then
+	rm /tmp/debian-{{ DIST }}-mm/etc/apt/apt.conf.d/01autoremove-kernels
+fi
+# clear out /run except for /run/lock
+find /tmp/debian-{{ DIST }}-debootstrap/run/ -mindepth 1 -maxdepth 1 ! -name lock -print0 | xargs --no-run-if-empty -0 rm -r
+# debootstrap doesn't clean apt
+rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_{{ DIST }}_main_binary-{{ HOSTARCH }}_Packages \
+	/tmp/debian-{{ DIST }}-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_{{ DIST }}_Release \
+	/tmp/debian-{{ DIST }}-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_{{ DIST }}_Release.gpg
+
+if [ "{{ VARIANT }}" = "-" ]; then
+	rm /tmp/debian-{{ DIST }}-debootstrap/etc/machine-id
+	rm /tmp/debian-{{ DIST }}-mm/etc/machine-id
+	rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/systemd/catalog/database
+	rm /tmp/debian-{{ DIST }}-mm/var/lib/systemd/catalog/database
+
+	cap=$(chroot /tmp/debian-{{ DIST }}-debootstrap /sbin/getcap /bin/ping)
+	expected="/bin/ping cap_net_raw=ep"
+	if [ "{{ DIST }}" = oldstable ]; then
+		expected="/bin/ping = cap_net_raw+ep"
+	fi
+	if [ "$cap" != "$expected" ]; then
+		echo "expected bin/ping to have capabilities $expected" >&2
+		echo "but debootstrap produced: $cap" >&2
+		exit 1
+	fi
+	cap=$(chroot /tmp/debian-{{ DIST }}-mm /sbin/getcap /bin/ping)
+	if [ "$cap" != "$expected" ]; then
+		echo "expected bin/ping to have capabilities $expected" >&2
+		echo "but mmdebstrap produced: $cap" >&2
+		exit 1
+	fi
+fi
+rm /tmp/debian-{{ DIST }}-mm/var/cache/apt/archives/lock
+rm /tmp/debian-{{ DIST }}-mm/var/lib/apt/extended_states
+rm /tmp/debian-{{ DIST }}-mm/var/lib/apt/lists/lock
+
+# the list of shells might be sorted wrongly
+# /var/lib/dpkg/triggers/File might be sorted wrongly
+for f in "/var/lib/dpkg/triggers/File" "/etc/shells"; do
+	f1="/tmp/debian-{{ DIST }}-debootstrap/$f"
+	f2="/tmp/debian-{{ DIST }}-mm/$f"
+	# both chroots must have the file
+	if [ ! -e "$f1" ] || [ ! -e "$f2" ]; then
+		continue
+	fi
+	# the file must be different
+	if cmp "$f1" "$f2" >&2; then
+		continue
+	fi
+	# then sort both
+	sort -o "$f1" "$f1"
+	sort -o "$f2" "$f2"
+done
+
+# Because of unreproducible uids (#969631) we created the _apt user ourselves
+# and because passwd is not Essential:yes we didn't use useradd. But newer
+# versions of adduser and shadow will create a different /etc/shadow
+if [ "{{ VARIANT }}" = "-" ]; then
+case {{ DIST }} in oldstable|stable)
+for f in shadow shadow-; do
+	if grep -q '^_apt:!:' /tmp/debian-{{ DIST }}-debootstrap/etc/$f; then
+		sed -i 's/^_apt:\*:\([^:]\+\):0:99999:7:::$/_apt:!:\1::::::/' /tmp/debian-{{ DIST }}-mm/etc/$f
+	fi
+done;;
+esac
+fi
+
+for log in faillog lastlog; do
+	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/var/log/$log /tmp/debian-{{ DIST }}-mm/var/log/$log >&2;then
+		# if the files differ, make sure they are all zeroes
+		cmp -n "$(stat -c %s "/tmp/debian-{{ DIST }}-debootstrap/var/log/$log")" "/tmp/debian-{{ DIST }}-debootstrap/var/log/$log" /dev/zero >&2
+		cmp -n "$(stat -c %s "/tmp/debian-{{ DIST }}-mm/var/log/$log")" "/tmp/debian-{{ DIST }}-mm/var/log/$log" /dev/zero >&2
+		# then delete them
+		rm /tmp/debian-{{ DIST }}-debootstrap/var/log/$log /tmp/debian-{{ DIST }}-mm/var/log/$log
+	fi
+done
+
+# the order in which systemd and cron get installed differ and thus the order
+# of lines in /etc/group and /etc/gshadow differs
+if [ "{{ VARIANT }}" = "-" ]; then
+	case {{ DIST }} in testing|unstable)
+		for f in group group- gshadow gshadow-; do
+			cmp /tmp/debian-{{ DIST }}-mm/etc/$f /tmp/debian-{{ DIST }}-debootstrap/etc/$f 2>/dev/null && exit 1
+			for d in mm debootstrap; do
+				sort /tmp/debian-{{ DIST }}-$d/etc/$f > /tmp/debian-{{ DIST }}-$d/etc/$f.bak
+				mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
+			done
+		done
+		;;
+	esac
+fi
+
+# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
+case {{ DIST }} in oldstable|stable)
+for f in shadow shadow-; do
+	if [ ! -e /tmp/debian-{{ DIST }}-mm/etc/$f ]; then
+		continue
+	fi
+	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/etc/$f /tmp/debian-{{ DIST }}-mm/etc/$f >&2; then
+		echo patching /etc/$f on {{ DIST }} {{ VARIANT }} >&2
+		awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-{{ DIST }}-mm/etc/$f > /tmp/debian-{{ DIST }}-mm/etc/$f.bak
+		cat /tmp/debian-{{ DIST }}-mm/etc/$f.bak > /tmp/debian-{{ DIST }}-mm/etc/$f
+		rm /tmp/debian-{{ DIST }}-mm/etc/$f.bak
+	else
+		echo no difference for /etc/$f on {{ DIST }} {{ VARIANT }} >&2
+	fi
+done;;
+esac
+
+# check if the file content differs
+diff --unified --no-dereference --recursive /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm >&2
+
+# check permissions, ownership, symlink targets, modification times using tar
+# directory mtimes will differ, thus we equalize them first
+find /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm -type d -print0 | xargs -0 touch --date="@{{ SOURCE_DATE_EPOCH }}"
+# debootstrap never ran apt -- fixing permissions
+for d in ./var/lib/apt/lists/partial ./var/cache/apt/archives/partial; do
+	chroot /tmp/debian-{{ DIST }}-debootstrap chmod 0700 $d
+	chroot /tmp/debian-{{ DIST }}-debootstrap chown "$(id -u _apt):root" $d
+done
+tar -C /tmp/debian-{{ DIST }}-debootstrap --numeric-owner --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/root1.tar .
+tar -C /tmp/debian-{{ DIST }}-mm --numeric-owner --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/root2.tar .
+tar --full-time --verbose -tf /tmp/root1.tar > /tmp/root1.tar.list
+tar --full-time --verbose -tf /tmp/root2.tar > /tmp/root2.tar.list
+diff -u /tmp/root1.tar.list /tmp/root2.tar.list >&2
+rm /tmp/root1.tar /tmp/root2.tar /tmp/root1.tar.list /tmp/root2.tar.list
+
+# check if file properties (permissions, ownership, symlink names, modification time) differ
+#
+# we cannot use this (yet) because it cannot cope with paths that have [ or @ in them
+#fmtree -c -p /tmp/debian-{{ DIST }}-debootstrap -k flags,gid,link,mode,size,time,uid | sudo fmtree -p /tmp/debian-{{ DIST }}-mm
+
+rm -r /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm

tests/check-for-bit-by-bit-identical-format-output

@@ -0,0 +1,37 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+adduser --gecos user --disabled-password user
+sysctl -w kernel.unprivileged_userns_clone=1
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+{{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-root.{{ FORMAT }} {{ MIRROR }}
+if [ "{{ FORMAT }}" = tar ]; then
+	printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 /tmp/debian-chroot-root.tar -
+elif [ "{{ FORMAT }}" = squashfs ]; then
+	printf 'hsqs' | cmp --bytes=4 /tmp/debian-chroot-root.squashfs -
+elif [ "{{ FORMAT }}" = ext2 ]; then
+	printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 /tmp/debian-chroot-root.ext2 -
+else
+	echo "unknown format: {{ FORMAT }}" >&2
+	exit 1
+fi
+runuser -u user -- {{ CMD }} --mode=unshare --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-unshare.{{ FORMAT }} {{ MIRROR }}
+cmp /tmp/debian-chroot-root.{{ FORMAT }} /tmp/debian-chroot-unshare.{{ FORMAT }}
+rm /tmp/debian-chroot-unshare.{{ FORMAT }}
+case {{ VARIANT }} in essential|apt|minbase|buildd)
+	# variants important and standard differ because permissions drwxr-sr-x
+	# and extended attributes of ./var/log/journal/ cannot be preserved
+	# in fakechroot mode
+	runuser -u user -- {{ CMD }} --mode=fakechroot --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-fakechroot.{{ FORMAT }} {{ MIRROR }}
+	cmp /tmp/debian-chroot-root.{{ FORMAT }} /tmp/debian-chroot-fakechroot.{{ FORMAT }}
+	rm /tmp/debian-chroot-fakechroot.{{ FORMAT }}
+	;;
+esac
+# we cannot test chrootless mode here, because mmdebstrap relies on the
+# usrmerge package to set up merged-/usr and that doesn't work in chrootless
+# mode
+rm /tmp/debian-chroot-root.{{ FORMAT }}

tests/chroot-directory-not-accessible-by-apt-user

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+mkdir /tmp/debian-chroot
+chmod 700 /tmp/debian-chroot
+{{ CMD }} --mode=root --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/chrootless

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+if dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libpam-runtime)" le 1.5.2-5; then
+	# https://bugs.debian.org/1022952
+	exit 0
+fi
+trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
+# we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
+# DPKG_ROOT
+for INCLUDE in '' 'systemd-sysv'; do
+	for MODE in root chrootless; do
+		{{ CMD }} --mode=$MODE --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
+			${INCLUDE:+--include="$INCLUDE"} \
+			{{ DIST }} "/tmp/$MODE.tar" {{ MIRROR }}
+	done
+	cmp /tmp/root.tar /tmp/chrootless.tar
+	rm /tmp/chrootless.tar /tmp/root.tar
+done

tests/chrootless-fakeroot

@@ -0,0 +1,29 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	if [ ! -e /mmdebstrap-testenv ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
+		exit 1
+	fi
+	adduser --gecos user --disabled-password user
+fi
+prefix=
+[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+# we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
+# DPKG_ROOT
+# permissions drwxr-sr-x and extended attributes of ./var/log/journal/ cannot
+# be preserved under fakeroot
+for INCLUDE in '' 'systemd-sysv'; do
+	{{ CMD }} --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
+		--customize-hook='if [ -d "$1"/var/log/journal ]; then rmdir "$1"/var/log/journal; mkdir --mode=2755 "$1"/var/log/journal; chroot "$1" chown root:systemd-journal /var/log/journal; fi' \
+		${INCLUDE:+--include="$INCLUDE"} \
+		{{ DIST }} /tmp/root.tar {{ MIRROR }}
+	$prefix fakeroot {{ CMD }} --mode=chrootless --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
+		${INCLUDE:+--include="$INCLUDE"} \
+		{{ DIST }} /tmp/chrootless.tar {{ MIRROR }}
+	cmp /tmp/root.tar /tmp/chrootless.tar
+	rm /tmp/chrootless.tar /tmp/root.tar
+done

tests/chrootless-foreign

@@ -0,0 +1,46 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+
+[ "$(id -u)" -eq 0 ]
+[ -e /proc/sys/fs/binfmt_misc/qemu-aarch64 ]
+
+# we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
+# DPKG_ROOT
+for INCLUDE in '' 'systemd-sysv'; do
+	echo 1 > /proc/sys/fs/binfmt_misc/qemu-aarch64
+	arch-test arm64
+	{{ CMD }} --mode=root --architecture=arm64 --variant={{ VARIANT }} \
+		--hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
+		{{ DIST }} "/tmp/root.tar" {{ MIRROR }}
+	echo 0 > /proc/sys/fs/binfmt_misc/qemu-aarch64
+	arch-test arm64 && exit 1
+	{{ CMD }} --mode=chrootless --architecture=arm64 --variant={{ VARIANT }} \
+		--hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
+		{{ DIST }} "/tmp/chrootless.tar" {{ MIRROR }}
+	# when creating a foreign architecture chroot, the tarballs are not
+	# bit-by-bit identical but contain a few remaining differences:
+	#
+	#  * /etc/ld.so.cache -- hard problem, must be solved in glibc upstream
+	#  * /var/lib/dpkg/triggers -- #990712
+	#  * /var/cache/debconf/*.dat-old -- needs investigation
+	for tar in root chrootless; do
+		<"/tmp/$tar.tar" \
+		./tarfilter \
+			--path-exclude=/var/cache/debconf/config.dat-old \
+			--path-exclude=/var/cache/debconf/templates.dat-old \
+			--path-exclude=/etc/ld.so.cache \
+			--path-exclude=/var/lib/dpkg/triggers/File \
+			--path-exclude=/var/lib/dpkg/triggers/ldconfig \
+			> "/tmp/$tar.tar.tmp"
+		mv "/tmp/$tar.tar.tmp" "/tmp/$tar.tar"
+	done
+	cmp /tmp/root.tar /tmp/chrootless.tar
+	rm /tmp/chrootless.tar /tmp/root.tar
+done

tests/compare-output-with-pre-seeded-var-cache-apt-archives

@@ -0,0 +1,44 @@
+#!/bin/sh
+#
+# test that the user can drop archives into /var/cache/apt/archives as well as
+# into /var/cache/apt/archives/partial
+
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test requires the cache directory to be mounted on /mnt and should only be run inside a container" >&2
+	exit 1
+fi
+tmpdir=$(mktemp -d)
+trap 'rm -f "$tmpdir"/*.deb /tmp/orig.tar /tmp/test1.tar /tmp/test2.tar; rmdir "$tmpdir"' EXIT INT TERM
+
+include="--include=doc-debian"
+if [ "{{ VARIANT }}" = "custom" ]; then
+	include="$include,base-files,base-passwd,coreutils,dash,diffutils,dpkg,libc-bin,sed"
+fi
+{{ CMD }} $include --mode={{ MODE }} --variant={{ VARIANT }} \
+	--setup-hook='mkdir -p "$1"/var/cache/apt/archives/partial' \
+	--setup-hook='touch "$1"/var/cache/apt/archives/lock' \
+	--setup-hook='chmod 0640 "$1"/var/cache/apt/archives/lock' \
+	{{ DIST }} - {{ MIRROR }} > /tmp/orig.tar
+# somehow, when trying to create a tarball from the 9p mount, tar throws the
+# following error: tar: ./doc-debian_6.4_all.deb: File shrank by 132942 bytes; padding with zeros
+# to reproduce, try: tar --directory /mnt/cache/debian/pool/main/d/doc-debian/ --create --file - . | tar --directory /tmp/ --extract --file -
+# this will be different:
+# md5sum /mnt/cache/debian/pool/main/d/doc-debian/*.deb /tmp/*.deb
+# another reason to copy the files into a new directory is, that we can use shell globs
+cp /mnt/cache/debian/pool/main/b/busybox/busybox_*"_{{ HOSTARCH }}.deb" /mnt/cache/debian/pool/main/a/apt/apt_*"_{{ HOSTARCH }}.deb" "$tmpdir"
+{{ CMD }} $include --mode={{ MODE }} --variant={{ VARIANT }} \
+	--setup-hook='mkdir -p "$1"/var/cache/apt/archives/partial' \
+	--setup-hook='sync-in "'"$tmpdir"'" /var/cache/apt/archives/partial' \
+	{{ DIST }} - {{ MIRROR }} > /tmp/test1.tar
+cmp /tmp/orig.tar /tmp/test1.tar
+{{ CMD }} $include --mode={{ MODE }} --variant={{ VARIANT }} \
+	--customize-hook='touch "$1"/var/cache/apt/archives/partial' \
+	--setup-hook='mkdir -p "$1"/var/cache/apt/archives/' \
+	--setup-hook='sync-in "'"$tmpdir"'" /var/cache/apt/archives/' \
+	--setup-hook='chmod 0755 "$1"/var/cache/apt/archives/' \
+	--customize-hook='find "'"$tmpdir"'" -type f -exec md5sum "{}" \; | sed "s|"'"$tmpdir"'"|$1/var/cache/apt/archives|" | md5sum --check' \
+	{{ DIST }} - {{ MIRROR }} > /tmp/test2.tar
+cmp /tmp/orig.tar /tmp/test2.tar

tests/copy-mirror

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test requires the cache directory to be mounted on /mnt and should only be run inside a container" >&2
+	exit 1
+fi
+{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar "deb copy:///mnt/cache/debian {{ DIST }} main"
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+rm /tmp/debian-chroot.tar

tests/create-arm64-tarball

@@ -0,0 +1,68 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	if [ ! -e /mmdebstrap-testenv ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
+		exit 1
+	fi
+	adduser --gecos user --disabled-password user
+fi
+if [ "{{ MODE }}" = unshare ]; then
+	if [ ! -e /mmdebstrap-testenv ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
+		exit 1
+	fi
+	sysctl -w kernel.unprivileged_userns_clone=1
+fi
+prefix=
+[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
+[ "{{ MODE }}" = "fakechroot" ] && prefix="$prefix fakechroot fakeroot"
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --architectures=arm64 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+# we ignore differences between architectures by ignoring some files
+# and renaming others
+{ tar -tf /tmp/debian-chroot.tar \
+	| grep -v '^\./usr/lib/ld-linux-aarch64\.so\.1$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/ld-linux-aarch64\.so\.1$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/int-ll64\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/types\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm-generic/unistd\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/sigcontext\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/sve_context\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/asm/types\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-extra\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-id\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs-prregset\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/bits/procfs\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/gnu/stubs-lp64\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/linux/types\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/sys/procfs\.ph$' \
+	| grep -v '^\./usr/lib/aarch64-linux-gnu/perl/5\.[0-9]\+\.0/sys/user\.ph$' \
+	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.arm64\.gz$' \
+	| sed 's/aarch64-linux-gnu/x86_64-linux-gnu/' \
+	| sed 's/arm64/amd64/';
+} | sort > tar2.txt
+{ < tar1.txt \
+	grep -v '^\./usr/bin/i386$' \
+	| grep -v '^\./usr/bin/x86_64$' \
+	| grep -v '^\./lib32$' \
+	| grep -v '^\./lib64$' \
+	| grep -v '^\./libx32$' \
+	| grep -v '^\./usr/lib32/$' \
+	| grep -v '^\./usr/libx32/$' \
+	| grep -v '^\./usr/lib64/$' \
+	| grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/ld-linux-x86-64\.so\.2$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/libmvec\.so\.1$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_32\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_64\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/posix_types_x32\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_32\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_64\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/asm/unistd_x32\.ph$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/perl/5\.[0-9]\+\.0/gnu/stubs-64\.ph$' \
+	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.amd64\.gz$' \
+	| grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
+	| grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$';
+} | sort | diff -u - tar2.txt >&2
+rm /tmp/debian-chroot.tar

tests/create-directory

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+{{ CMD }} --mode=root --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+chroot /tmp/debian-chroot dpkg-query --showformat '${binary:Package}\n' --show > pkglist.txt
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort > tar1.txt
+rm -r /tmp/debian-chroot

tests/create-directory-dry-run

@@ -0,0 +1,29 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+{{ CMD }} --mode={{ MODE }} --dry-run --variant=apt \
+	--setup-hook="exit 1" \
+	--essential-hook="exit 1" \
+	--customize-hook="exit 1" \
+	{{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+rm /tmp/debian-chroot/dev/console
+rm /tmp/debian-chroot/dev/fd
+rm /tmp/debian-chroot/dev/full
+rm /tmp/debian-chroot/dev/null
+rm /tmp/debian-chroot/dev/ptmx
+rm /tmp/debian-chroot/dev/random
+rm /tmp/debian-chroot/dev/stderr
+rm /tmp/debian-chroot/dev/stdin
+rm /tmp/debian-chroot/dev/stdout
+rm /tmp/debian-chroot/dev/tty
+rm /tmp/debian-chroot/dev/urandom
+rm /tmp/debian-chroot/dev/zero
+rm /tmp/debian-chroot/etc/apt/sources.list
+rm /tmp/debian-chroot/etc/fstab
+rm /tmp/debian-chroot/etc/hostname
+rm /tmp/debian-chroot/etc/resolv.conf
+rm /tmp/debian-chroot/var/lib/apt/lists/lock
+rm /tmp/debian-chroot/var/lib/dpkg/status
+rm /tmp/debian-chroot/var/lib/dpkg/arch
+# the rest should be empty directories that we can rmdir recursively
+find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir

tests/create-gzip-compressed-tarball

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+adduser --gecos user --disabled-password user
+sysctl -w kernel.unprivileged_userns_clone=1
+runuser -u user -- {{ CMD }} --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot.tar.gz {{ MIRROR }}
+printf '\037\213\010' | cmp --bytes=3 /tmp/debian-chroot.tar.gz -
+tar -tf /tmp/debian-chroot.tar.gz | sort | diff -u tar1.txt -
+rm /tmp/debian-chroot.tar.gz

tests/create-tarball-dry-run

@@ -0,0 +1,35 @@
+#!/bin/sh
+#
+# we are testing all variants here because with 0.7.5 we had a bug:
+# mmdebstrap sid /dev/null --simulate ==> E: cannot read /var/cache/apt/archives/
+
+set -eu
+export LC_ALL=C.UTF-8
+prefix=
+include=,
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != root ] && [ "{{ MODE }}" != auto ]; then
+	# this must be qemu
+	if ! id -u user >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		adduser --gecos user --disabled-password user
+	fi
+	if [ "{{ MODE }}" = unshare ]; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		sysctl -w kernel.unprivileged_userns_clone=1
+	fi
+	prefix="runuser -u user --"
+	if [ "{{ VARIANT }}" = extract ] || [ "{{ VARIANT }}" = custom ]; then
+		include="$(tr '\n' ',' < pkglist.txt)"
+	fi
+fi
+$prefix {{ CMD }} --mode={{ MODE }} --include="$include" --dry-run --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+if [ -e /tmp/debian-chroot.tar ]; then
+	echo "/tmp/debian-chroot.tar must not be created with --dry-run" >&2
+	exit 1
+fi

tests/create-tarball-with-tmp-mounted-nodev

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+mount -t tmpfs -o nodev,nosuid,size=400M tmpfs /tmp
+# use --customize-hook to exercise the mounting/unmounting code of block devices in root mode
+{{ CMD }} --mode=root --variant=apt --customize-hook='mount | grep /dev/full' --customize-hook='test "$(echo foo | tee /dev/full 2>&1 1>/dev/null)" = "tee: /dev/full: No space left on device"' {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+rm /tmp/debian-chroot.tar

tests/custom-tmpdir

@@ -0,0 +1,26 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+# https://www.etalabs.net/sh_tricks.html
+quote () { printf %s\\n "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" ; }
+adduser --gecos user --disabled-password user
+sysctl -w kernel.unprivileged_userns_clone=1
+homedir=$(runuser -u user -- sh -c 'cd && pwd')
+# apt:test/integration/test-apt-key
+TMPDIR_ADD="This is fü\$\$ing cràzy, \$(apt -v)\$!"
+runuser -u user -- mkdir "$homedir/$TMPDIR_ADD"
+# make sure the unshared user can traverse into the TMPDIR
+chmod 711 "$homedir"
+# set permissions and sticky bit like the real /tmp
+chmod 1777 "$homedir/$TMPDIR_ADD"
+runuser -u user -- env TMPDIR="$homedir/$TMPDIR_ADD" {{ CMD }} --mode=unshare --variant=apt \
+	--setup-hook='case "$1" in '"$(quote "$homedir/$TMPDIR_ADD/mmdebstrap.")"'??????????) exit 0;; *) echo "$1"; exit 1;; esac' \
+	 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+# use rmdir as a quick check that nothing is remaining in TMPDIR
+runuser -u user -- rmdir "$homedir/$TMPDIR_ADD"
+rm /tmp/debian-chroot.tar

tests/customize-hook

@@ -0,0 +1,16 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot; rm -f /tmp/customize.sh" EXIT INT TERM
+cat << 'SCRIPT' > /tmp/customize.sh
+#!/bin/sh
+chroot "$1" whoami > "$1/output2"
+chroot "$1" pwd >> "$1/output2"
+SCRIPT
+chmod +x /tmp/customize.sh
+{{ CMD }} --mode=root --variant=apt --customize-hook='chroot "$1" sh -c "whoami; pwd" > "$1/output1"' --customize-hook=/tmp/customize.sh {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+printf "root\n/\n" | cmp /tmp/debian-chroot/output1
+printf "root\n/\n" | cmp /tmp/debian-chroot/output2
+rm /tmp/debian-chroot/output1
+rm /tmp/debian-chroot/output2
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/cwd-directory-not-accessible-by-unshared-user

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+adduser --gecos user --disabled-password user
+sysctl -w kernel.unprivileged_userns_clone=1
+mkdir /tmp/debian-chroot
+chmod 700 /tmp/debian-chroot
+chown user:user /tmp/debian-chroot
+if [ "{{ CMD }}" = "./mmdebstrap" ]; then
+	set -- "$(realpath --canonicalize-existing ./mmdebstrap)"
+elif [ "{{ CMD }}" = "perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap" ]; then
+	set -- perl -MDevel::Cover=-silent,-nogcov "$(realpath --canonicalize-existing ./mmdebstrap)"
+else
+	set -- {{ CMD }}
+fi
+env --chdir=/tmp/debian-chroot runuser -u user -- "$@" --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+rm /tmp/debian-chroot.tar

tests/deb822-1-2

@@ -0,0 +1,45 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot; rm -f /tmp/sources.list /tmp/deb822.sources" EXIT INT TERM
+cat << SOURCES > /tmp/deb822.sources
+Types: deb
+URIs: {{ MIRROR }}1
+Suites: {{ DIST }}
+Components: main
+SOURCES
+echo "deb {{ MIRROR }}2 {{ DIST }} main" > /tmp/sources.list
+echo "deb {{ MIRROR }}3 {{ DIST }} main" \
+	| {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} \
+		/tmp/debian-chroot \
+		/tmp/deb822.sources \
+		{{ MIRROR }}4 \
+		- \
+		"deb {{ MIRROR }}5 {{ DIST }} main" \
+		{{ MIRROR }}6 \
+		/tmp/sources.list
+test ! -e /tmp/debian-chroot/etc/apt/sources.list
+cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0000deb822.sources -
+Types: deb
+URIs: {{ MIRROR }}1
+Suites: {{ DIST }}
+Components: main
+SOURCES
+cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0001main.list -
+deb {{ MIRROR }}4 {{ DIST }} main
+
+deb {{ MIRROR }}3 {{ DIST }} main
+
+deb {{ MIRROR }}5 {{ DIST }} main
+
+deb {{ MIRROR }}6 {{ DIST }} main
+SOURCES
+echo "deb {{ MIRROR }}2 {{ DIST }} main" | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0002sources.list -
+tar -C /tmp/debian-chroot --one-file-system -c . \
+	| {
+		tar -t \
+			| grep -v "^./etc/apt/sources.list.d/0000deb822.sources$" \
+			| grep -v "^./etc/apt/sources.list.d/0001main.list$" \
+			| grep -v "^./etc/apt/sources.list.d/0002sources.list";
+		printf "./etc/apt/sources.list\n";
+	} | sort | diff -u tar1.txt -

tests/deb822-2-2

@@ -0,0 +1,44 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot; rm -f /tmp/sources /tmp/deb822" EXIT INT TERM
+cat << SOURCES > /tmp/deb822
+Types: deb
+URIs: {{ MIRROR }}1
+Suites: {{ DIST }}
+Components: main
+SOURCES
+echo "deb {{ MIRROR }}2 {{ DIST }} main" > /tmp/sources
+cat << SOURCES | {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} \
+		/tmp/debian-chroot \
+		/tmp/deb822 \
+		- \
+		/tmp/sources
+Types: deb
+URIs: {{ MIRROR }}3
+Suites: {{ DIST }}
+Components: main
+SOURCES
+test ! -e /tmp/debian-chroot/etc/apt/sources.list
+ls -lha /tmp/debian-chroot/etc/apt/sources.list.d/
+cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0000deb822.sources -
+Types: deb
+URIs: {{ MIRROR }}1
+Suites: {{ DIST }}
+Components: main
+SOURCES
+cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0001main.sources -
+Types: deb
+URIs: {{ MIRROR }}3
+Suites: {{ DIST }}
+Components: main
+SOURCES
+echo "deb {{ MIRROR }}2 {{ DIST }} main" | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0002sources.list -
+tar -C /tmp/debian-chroot --one-file-system -c . \
+	| {
+		tar -t \
+			| grep -v "^./etc/apt/sources.list.d/0000deb822.sources$" \
+			| grep -v "^./etc/apt/sources.list.d/0001main.sources$" \
+			| grep -v "^./etc/apt/sources.list.d/0002sources.list$";
+		printf "./etc/apt/sources.list\n";
+	} | sort | diff -u tar1.txt -

tests/debootstrap-no-op-options

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+{{ CMD }} --mode=root --variant=apt --resolve-deps --merged-usr --no-merged-usr --force-check-gpg {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+rm -r /tmp/debian-chroot

tests/debug

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+{{ CMD }} --mode=root --variant=apt --debug {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+rm -r /tmp/debian-chroot

tests/debug-output-on-fake-tty

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+script -qfc "{{ CMD }} --mode={{ MODE }} --debug --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}" /dev/null
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/dev-ptmx

@@ -0,0 +1,146 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+
+if [ {{ MODE }} != unshare ] && [ {{ MODE }} != root ]; then
+	echo "test requires root or unshare mode" >&2
+	exit 1
+fi
+
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	adduser --gecos user --disabled-password user
+fi
+prefix=
+[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
+
+# this mimics what apt does in apt-pkg/deb/dpkgpm.cc/pkgDPkgPM::StartPtyMagic()
+cat > /tmp/test.c << 'END'
+#define _GNU_SOURCE
+
+#include <stdlib.h>
+#include <fcntl.h>
+#include <termios.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <signal.h>
+
+int main() {
+	int ret;
+	int fd = posix_openpt(O_RDWR | O_NOCTTY);
+	if (fd < 0) {
+		perror("posix_openpt");
+		return 1;
+	}
+	char buf[64]; // 64 is used by apt
+	ret = ptsname_r(fd, buf, sizeof(buf));
+	if (ret != 0) {
+		perror("ptsname_r");
+		return 1;
+	}
+	ret = grantpt(fd);
+	if (ret == -1) {
+		perror("grantpt");
+		return 1;
+	}
+	struct termios origtt;
+	ret = tcgetattr(STDIN_FILENO, &origtt);
+	if (ret != 0) {
+		perror("tcgetattr1");
+		return 1;
+	}
+	struct termios tt;
+	ret = tcgetattr(STDOUT_FILENO, &tt);
+	if (ret != 0) {
+		perror("tcgetattr2");
+		return 1;
+	}
+	struct winsize win;
+	ret = ioctl(STDOUT_FILENO, TIOCGWINSZ, &win);
+	if (ret < 0) {
+		perror("ioctl stdout TIOCGWINSZ");
+		return 1;
+	}
+	ret = ioctl(fd, TIOCSWINSZ, &win);
+	if (ret < 0) {
+		perror("ioctl fd TIOCGWINSZ");
+		return 1;
+	}
+	ret = tcsetattr(fd, TCSANOW, &tt);
+	if (ret != 0) {
+		perror("tcsetattr1");
+		return 1;
+	}
+	cfmakeraw(&tt);
+	tt.c_lflag &= ~ECHO;
+	tt.c_lflag |= ISIG;
+	sigset_t sigmask;
+	sigset_t sigmask_old;
+	ret = sigemptyset(&sigmask);
+	if (ret != 0) {
+		perror("sigemptyset");
+		return 1;
+	}
+	ret = sigaddset(&sigmask, SIGTTOU);
+	if (ret != 0) {
+		perror("sigaddset");
+		return 1;
+	}
+	ret = sigprocmask(SIG_BLOCK,&sigmask, &sigmask_old);
+	if (ret != 0) {
+		perror("sigprocmask1");
+		return 1;
+	}
+	ret = tcsetattr(STDIN_FILENO, TCSAFLUSH, &tt);
+	if (ret != 0) {
+		perror("tcsetattr2");
+		return 1;
+	}
+	ret = sigprocmask(SIG_BLOCK,&sigmask_old, NULL);
+	if (ret != 0) {
+		perror("sigprocmask2");
+		return 1;
+	}
+	ret = tcsetattr(STDIN_FILENO, TCSAFLUSH, &origtt);
+	if (ret != 0) {
+		perror("tcsetattr3");
+		return 1;
+	}
+	return 0;
+}
+END
+
+# use script to create a fake tty
+# run all tests as root and as a normal user (the latter requires ptmxmode=666)
+script -qfc "$prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
+	--include=gcc,libc6-dev,python3,adduser \
+	--customize-hook='chroot \"\$1\" adduser --gecos user --disabled-password user' \
+	--customize-hook='chroot \"\$1\" python3 -c \"import pty; print(pty.openpty())\"' \
+	--customize-hook='chroot \"\$1\" runuser -u user -- python3 -c \"import pty; print(pty.openpty())\"' \
+	--customize-hook='chroot \"\$1\" script -c \"echo foobar\"' \
+	--customize-hook='chroot \"\$1\" runuser -u user -- env --chdir=/home/user script -c \"echo foobar\"' \
+	--customize-hook='chroot \"\$1\" apt-get install --yes doc-debian 2>&1 | tee /tmp/log' \
+	--customize-hook=\"copy-in /tmp/test.c /tmp\" \
+	--customize-hook='chroot \"\$1\" gcc /tmp/test.c -o /tmp/test' \
+	--customize-hook='chroot \"\$1\" /tmp/test' \
+	--customize-hook='chroot \"\$1\" runuser -u user -- /tmp/test' \
+	--customize-hook='rm \"\$1\"/tmp/test \"\$1\"/tmp/test.c' \
+	{{ DIST }} /dev/null {{ MIRROR }}" /dev/null
+
+fail=0
+[ -r /tmp/log ] || fail=1
+grep '^E:' /tmp/log && fail=1
+grep 'Can not write log' /tmp/log && fail=1
+grep 'posix_openpt' /tmp/log && fail=1
+grep 'No such file or directory' /tmp/log && fail=1
+if [ $fail -eq 1 ]; then
+	echo "apt failed to write log:" >&2
+	cat /tmp/log >&2
+	exit 1
+fi
+
+rm /tmp/test.c /tmp/log

tests/directory-ending-in-tar

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+[ "$(whoami)" = "root" ]
+trap "rm -rf /tmp/debian-chroot.tar" EXIT INT TERM
+{{ CMD }} --mode={{ MODE }} --variant=apt --format=directory {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+ftype=$(stat -c %F /tmp/debian-chroot.tar)
+if [ "$ftype" != directory ]; then
+	echo "expected directory but got: $ftype" >&2
+	exit 1
+fi
+tar -C /tmp/debian-chroot.tar --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/dist-using-codename

@@ -0,0 +1,11 @@
+#!/bin/sh
+#
+# make sure that using codenames works https://bugs.debian.org/1003191
+
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f Release; rm -rf /tmp/debian-chroot" EXIT INT TERM
+/usr/lib/apt/apt-helper download-file "{{ MIRROR }}/dists/{{ DIST }}/Release" Release
+codename=$(awk '/^Codename: / { print $2; }' Release)
+{{ CMD }} --mode={{ MODE }} --variant=apt "$codename" /tmp/debian-chroot {{ MIRROR }}
+echo "deb {{ MIRROR }} $codename main" | diff -u - /tmp/debian-chroot/etc/apt/sources.list

tests/dpkgopt

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot; rm -f /tmp/config" EXIT INT TERM
+echo no-pager > /tmp/config
+{{ CMD }} --mode=root --variant=apt --dpkgopt="path-exclude=/usr/share/doc/*" --dpkgopt=/tmp/config --dpkgopt="path-include=/usr/share/doc/dpkg/copyright" {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+printf 'path-exclude=/usr/share/doc/*\nno-pager\npath-include=/usr/share/doc/dpkg/copyright\n' | cmp /tmp/debian-chroot/etc/dpkg/dpkg.cfg.d/99mmdebstrap -
+rm /tmp/debian-chroot/etc/dpkg/dpkg.cfg.d/99mmdebstrap
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort > tar2.txt
+{ grep -v '^./usr/share/doc/.' tar1.txt; echo ./usr/share/doc/dpkg/; echo ./usr/share/doc/dpkg/copyright; } | sort | diff -u - tar2.txt

tests/eatmydata-via-hook-dir

@@ -0,0 +1,39 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+cat << SCRIPT > /tmp/checkeatmydata.sh
+#!/bin/sh
+set -exu
+cat << EOF | diff - "\$1"/usr/bin/dpkg
+#!/bin/sh
+exec /usr/bin/eatmydata /usr/bin/dpkg.distrib "\\\$@"
+EOF
+[ -e "\$1"/usr/bin/eatmydata ]
+SCRIPT
+chmod +x /tmp/checkeatmydata.sh
+# first four bytes: magic
+elfheader="\\177ELF"
+# fifth byte: bits
+case "$(dpkg-architecture -qDEB_HOST_ARCH_BITS)" in
+	32) elfheader="$elfheader\\001";;
+	64) elfheader="$elfheader\\002";;
+	*) echo "bits not supported"; exit 1;;
+esac
+# sixth byte: endian
+case "$(dpkg-architecture -qDEB_HOST_ARCH_ENDIAN)" in
+	little) elfheader="$elfheader\\001";;
+	big) elfheader="$elfheader\\002";;
+	*) echo "endian not supported"; exit 1;;
+esac
+# seventh and eigth byte: elf version (1) and abi (unset)
+elfheader="$elfheader\\001\\000"
+{{ CMD }} --mode=root --variant=apt \
+	--customize-hook=/tmp/checkeatmydata.sh \
+	--essential-hook=/tmp/checkeatmydata.sh \
+	--extract-hook='printf "'"$elfheader"'" | cmp --bytes=8 - "$1"/usr/bin/dpkg' \
+	--hook-dir=./hooks/eatmydata \
+	--customize-hook='printf "'"$elfheader"'" | cmp --bytes=8 - "$1"/usr/bin/dpkg' \
+	 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+rm /tmp/checkeatmydata.sh
+rm -r /tmp/debian-chroot

tests/error-if-stdout-is-tty

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -eu
+
+export LC_ALL=C.UTF-8
+
+ret=0
+script -qfec "{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} - {{ MIRROR }}" /dev/null || ret=$?
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi

tests/essential-hook

@@ -0,0 +1,21 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot; rm -f /tmp/essential.sh" EXIT INT TERM
+cat << 'SCRIPT' > /tmp/essential.sh
+#!/bin/sh
+echo tzdata tzdata/Zones/Europe select Berlin | chroot "$1" debconf-set-selections
+SCRIPT
+chmod +x /tmp/essential.sh
+{{ CMD }} --mode=root --variant=apt --include=tzdata --essential-hook='echo tzdata tzdata/Areas select Europe | chroot "$1" debconf-set-selections' --essential-hook=/tmp/essential.sh {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+echo Europe/Berlin | cmp /tmp/debian-chroot/etc/timezone
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort \
+	| grep -v '^./etc/localtime' \
+	| grep -v '^./etc/timezone' \
+	| grep -v '^./usr/sbin/tzconfig' \
+	| grep -v '^./usr/share/doc/tzdata' \
+	| grep -v '^./usr/share/lintian/overrides/tzdata' \
+	| grep -v '^./usr/share/zoneinfo' \
+	| grep -v '^./var/lib/dpkg/info/tzdata.' \
+	| grep -v '^./var/lib/apt/extended_states$' \
+	| diff -u tar1.txt -

tests/existing-directory-with-lost-found

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+mkdir /tmp/debian-chroot
+mkdir /tmp/debian-chroot/lost+found
+{{ CMD }} --mode=root --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+rmdir /tmp/debian-chroot/lost+found
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/existing-empty-directory

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+mkdir /tmp/debian-chroot
+{{ CMD }} --mode=root --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/fail-installing-to-existing-file

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+touch /tmp/exists
+ret=0
+{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/exists {{ MIRROR }} || ret=$?
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi

tests/fail-installing-to-non-empty-lost-found

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm /tmp/debian-chroot/lost+found/exists; rmdir /tmp/debian-chroot/lost+found /tmp/debian-chroot" EXIT INT TERM
+mkdir /tmp/debian-chroot
+mkdir /tmp/debian-chroot/lost+found
+touch /tmp/debian-chroot/lost+found/exists
+ret=0
+{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }} || ret=$?
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi

tests/fail-installing-to-non-empty-target-directory

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rmdir /tmp/debian-chroot/lost+found; rm /tmp/debian-chroot/exists; rmdir /tmp/debian-chroot" EXIT INT TERM
+mkdir /tmp/debian-chroot
+mkdir /tmp/debian-chroot/lost+found
+touch /tmp/debian-chroot/exists
+ret=0
+{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }} || ret=$?
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi

tests/fail-installing-to-root

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+ret=0
+{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} / {{ MIRROR }} || ret=$?
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi

tests/fail-with-missing-lz4

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+ret=0
+{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar.lz4 {{ MIRROR }} || ret=$?
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi

tests/fail-with-path-with-quotes

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+ret=0
+{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/quoted\"path {{ MIRROR }} || ret=$?
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi

tests/fail-without-etc-subuid

@@ -0,0 +1,17 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+adduser --gecos user --disabled-password user
+sysctl -w kernel.unprivileged_userns_clone=1
+rm /etc/subuid
+ret=0
+runuser -u user -- {{ CMD }} --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }} || ret=$?
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi
+rm -r /tmp/debian-chroot

tests/fail-without-username-in-etc-subuid

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+adduser --gecos user --disabled-password user
+sysctl -w kernel.unprivileged_userns_clone=1
+awk -F: '$1!="user"' /etc/subuid > /etc/subuid.tmp
+mv /etc/subuid.tmp /etc/subuid
+ret=0
+runuser -u user -- {{ CMD }} --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }} || ret=$?
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi
+rm -r /tmp/debian-chroot

tests/failing-customize-hook

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+ret=0
+{{ CMD }} --mode=root --variant=apt --customize-hook='chroot "$1" sh -c "exit 1"' {{ DIST }} /tmp/debian-chroot {{ MIRROR }} || ret=$?
+rm -r /tmp/debian-chroot
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi

tests/file-mirror

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test requires the cache directory to be mounted on /mnt and should only be run inside a container" >&2
+	exit 1
+fi
+{{ CMD }} --mode={{ MODE }} --variant=apt \
+	--setup-hook='mkdir -p "$1"/mnt/cache/debian; mount -o ro,bind /mnt/cache/debian "$1"/mnt/cache/debian' \
+	--customize-hook='umount "$1"/mnt/cache/debian; rmdir "$1"/mnt/cache/debian "$1"/mnt/cache' \
+	{{ DIST }} /tmp/debian-chroot.tar "deb file:///mnt/cache/debian {{ DIST }} main"
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+rm /tmp/debian-chroot.tar

tests/file-mirror-automount-hook

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test requires the cache directory to be mounted on /mnt and should only be run inside a container" >&2
+	exit 1
+fi
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	adduser --gecos user --disabled-password user
+fi
+prefix=
+[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
+[ "{{ MODE }}" = "fakechroot" ] && prefix="$prefix fakechroot fakeroot"
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
+	--hook-dir=./hooks/file-mirror-automount \
+	--customize-hook='[ ! -e "$1"/mnt/cache/debian/ ] || rmdir "$1"/mnt/cache/debian/' \
+	--customize-hook='rmdir "$1"/mnt/cache' \
+	{{ DIST }} /tmp/debian-chroot.tar "deb file:///mnt/cache/debian {{ DIST }} main"
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+rm /tmp/debian-chroot.tar

tests/help

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+# we redirect to /dev/null instead of using --quiet to not cause a broken pipe
+# when grep exits before mmdebstrap was able to write all its output
+{{ CMD }} --help | grep --fixed-strings 'mmdebstrap [OPTION...] [SUITE [TARGET [MIRROR...]]]' >/dev/null

tests/hook-directory

@@ -0,0 +1,49 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+for h in hookA hookB; do
+	mkdir /tmp/$h
+	for s in setup extract essential customize; do
+		cat << SCRIPT > /tmp/$h/${s}00.sh
+#!/bin/sh
+echo $h/${s}00 >> "\$1/$s"
+SCRIPT
+		chmod +x /tmp/$h/${s}00.sh
+		cat << SCRIPT > /tmp/$h/${s}01.sh
+echo $h/${s}01 >> "\$1/$s"
+SCRIPT
+		chmod +x /tmp/$h/${s}01.sh
+	done
+done
+{{ CMD }} --mode=root --variant=apt \
+	--setup-hook='echo cliA/setup >> "$1"/setup' \
+	--extract-hook='echo cliA/extract >> "$1"/extract' \
+	--essential-hook='echo cliA/essential >> "$1"/essential' \
+	--customize-hook='echo cliA/customize >> "$1"/customize' \
+	--hook-dir=/tmp/hookA \
+	--setup-hook='echo cliB/setup >> "$1"/setup' \
+	--extract-hook='echo cliB/extract >> "$1"/extract' \
+	--essential-hook='echo cliB/essential >> "$1"/essential' \
+	--customize-hook='echo cliB/customize >> "$1"/customize' \
+	--hook-dir=/tmp/hookB \
+	--setup-hook='echo cliC/setup >> "$1"/setup' \
+	--extract-hook='echo cliC/extract >> "$1"/extract' \
+	--essential-hook='echo cliC/essential >> "$1"/essential' \
+	--customize-hook='echo cliC/customize >> "$1"/customize' \
+	 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+printf "cliA/setup\nhookA/setup00\nhookA/setup01\ncliB/setup\nhookB/setup00\nhookB/setup01\ncliC/setup\n" | diff -u - /tmp/debian-chroot/setup
+printf "cliA/extract\nhookA/extract00\nhookA/extract01\ncliB/extract\nhookB/extract00\nhookB/extract01\ncliC/extract\n" | diff -u - /tmp/debian-chroot/extract
+printf "cliA/essential\nhookA/essential00\nhookA/essential01\ncliB/essential\nhookB/essential00\nhookB/essential01\ncliC/essential\n" | diff -u - /tmp/debian-chroot/essential
+printf "cliA/customize\nhookA/customize00\nhookA/customize01\ncliB/customize\nhookB/customize00\nhookB/customize01\ncliC/customize\n" | diff -u - /tmp/debian-chroot/customize
+for s in setup extract essential customize; do
+	rm /tmp/debian-chroot/$s
+done
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+for h in hookA hookB; do
+	for s in setup extract essential customize; do
+		rm /tmp/$h/${s}00.sh
+		rm /tmp/$h/${s}01.sh
+	done
+	rmdir /tmp/$h
+done
+rm -r /tmp/debian-chroot

tests/i386-which-can-be-executed-without-qemu

@@ -0,0 +1,41 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+# remove qemu just to be sure
+apt-get remove --yes qemu-user-static binfmt-support qemu-user
+{{ CMD }} --mode={{ MODE }} --variant=apt --architectures=i386 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+# we ignore differences between architectures by ignoring some files
+# and renaming others
+{ tar -tf /tmp/debian-chroot.tar \
+	| grep -v '^\./usr/bin/i386$' \
+	| grep -v '^\./usr/lib/ld-linux\.so\.2$' \
+	| grep -v '^\./usr/lib/i386-linux-gnu/ld-linux\.so\.2$' \
+	| grep -v '^\./usr/lib/gcc/i686-linux-gnu/$' \
+	| grep -v '^\./usr/lib/gcc/i686-linux-gnu/[0-9]\+/$' \
+	| grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
+	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.i386\.gz$' \
+	| sed 's/i386-linux-gnu/x86_64-linux-gnu/' \
+	| sed 's/i386/amd64/' \
+	| sed 's/\/stubs-32.ph$/\/stubs-64.ph/';
+} | sort > tar2.txt
+{ < tar1.txt \
+	grep -v '^\./usr/bin/i386$' \
+	| grep -v '^\./usr/bin/x86_64$' \
+	| grep -v '^\./usr/lib32/$' \
+	| grep -v '^\./lib32$' \
+	| grep -v '^\./lib64$' \
+	| grep -v '^\./usr/lib64/$' \
+	| grep -v '^\./usr/lib64/ld-linux-x86-64\.so\.2$' \
+	| grep -v '^\./usr/lib/gcc/x86_64-linux-gnu/$' \
+	| grep -v '^\./usr/lib/gcc/x86_64-linux-gnu/[0-9]\+/$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/ld-linux-x86-64\.so\.2$' \
+	| grep -v '^\./usr/lib/x86_64-linux-gnu/libmvec\.so\.1$' \
+	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.amd64\.gz$' \
+	| grep -v '^\./usr/share/man/man8/i386\.8\.gz$' \
+	| grep -v '^\./usr/share/man/man8/x86_64\.8\.gz$';
+} | sort | diff -u - tar2.txt >&2
+rm /tmp/debian-chroot.tar

tests/include

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+{{ CMD }} --mode=root --variant=apt --include=doc-debian {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm -r /tmp/debian-chroot/usr/share/doc/debian
+rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
+rm /tmp/debian-chroot/var/lib/apt/extended_states
+rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.list
+rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.md5sums
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/include-deb-file

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+set -eu
+export LC_ALL=C.UTF-8
+
+# instead of obtaining a .deb from our cache, we create a new package because
+# otherwise apt might decide to download the package with the same name and
+# version from the cache instead of using the local .deb
+mkdir -p /tmp/dummypkg/DEBIAN
+cat << END > "/tmp/dummypkg/DEBIAN/control"
+Package: dummypkg
+Priority: optional
+Section: oldlibs
+Maintainer: Johannes Schauer Marin Rodrigues <josch@debian.org>
+Architecture: all
+Multi-Arch: foreign
+Source: dummypkg
+Version: 1
+Description: dummypkg
+END
+dpkg-deb --build "/tmp/dummypkg" "/tmp/dummypkg.deb"
+
+{{ CMD }} --variant=apt --include="/tmp/dummypkg.deb" \
+	--hook-dir=./hooks/file-mirror-automount \
+	--customize-hook='chroot "$1" dpkg-query -W -f="\${Status}\n" dummypkg | grep "^install ok installed$"' \
+	{{ DIST }} /dev/null {{ MIRROR }}

tests/include-libmagic-mgc-arm64

@@ -0,0 +1,28 @@
+#!/bin/sh
+#
+# to test foreign architecture package installation we choose a package which
+#   - is not part of the native installation set
+#   - does not have any dependencies
+#   - installs only few files
+#   - doesn't change its name regularly (like gcc-*-base)
+
+set -eu
+export LC_ALL=C.UTF-8
+{{ CMD }} --mode=root --variant=apt --architectures=amd64,arm64 --include=libmagic-mgc:arm64 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+{ echo "amd64"; echo "arm64"; } | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
+rm /tmp/debian-chroot/var/lib/apt/extended_states
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
+rm /tmp/debian-chroot/usr/lib/file/magic.mgc
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
+rm /tmp/debian-chroot/usr/share/file/magic.mgc
+rm /tmp/debian-chroot/usr/share/misc/magic.mgc
+rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/
+rmdir /tmp/debian-chroot/usr/share/file/magic/
+rmdir /tmp/debian-chroot/usr/share/file/
+rmdir /tmp/debian-chroot/usr/lib/file/
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+rm -r /tmp/debian-chroot

tests/include-libmagic-mgc-arm64-with-multiple-arch-options

@@ -0,0 +1,21 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+{{ CMD }} --mode=root --variant=apt --architectures=amd64 --architectures=arm64 --include=libmagic-mgc:arm64 {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+{ echo "amd64"; echo "arm64"; } | cmp /tmp/debian-chroot/var/lib/dpkg/arch -
+rm /tmp/debian-chroot/var/lib/apt/extended_states
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
+rm /tmp/debian-chroot/usr/lib/file/magic.mgc
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
+rm /tmp/debian-chroot/usr/share/file/magic.mgc
+rm /tmp/debian-chroot/usr/share/misc/magic.mgc
+rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/
+rmdir /tmp/debian-chroot/usr/share/file/magic/
+rmdir /tmp/debian-chroot/usr/share/file/
+rmdir /tmp/debian-chroot/usr/lib/file/
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+rm -r /tmp/debian-chroot

tests/include-with-multiple-apt-sources

@@ -0,0 +1,10 @@
+#!/bin/sh
+#
+# This checks for https://bugs.debian.org/976166
+# Since $DEFAULT_DIST varies, we hardcode stable and unstable.
+
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+{{ CMD }} --mode=root --variant=minbase --include=doc-debian unstable /tmp/debian-chroot "deb {{ MIRROR }} unstable main" "deb {{ MIRROR }} stable main"
+chroot /tmp/debian-chroot dpkg-query --show doc-debian

tests/install-busybox-based-sub-essential-system

@@ -0,0 +1,34 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+pkgs=base-files,base-passwd,busybox,debianutils,dpkg,libc-bin,mawk,tar
+# busybox --install -s will install symbolic links into the rootfs, leaving
+# existing files untouched. It has to run after extraction (otherwise there is
+# no busybox binary) and before first configuration
+{{ CMD }} --mode=root --variant=custom \
+    --include=$pkgs \
+    --setup-hook='mkdir -p "$1/bin"' \
+    --setup-hook='echo root:x:0:0:root:/root:/bin/sh > "$1/etc/passwd"' \
+    --setup-hook='printf "root:x:0:\nmail:x:8:\nutmp:x:43:\n" > "$1/etc/group"' \
+    --extract-hook='chroot "$1" busybox --install -s' \
+    {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+echo "$pkgs" | tr ',' '\n' > /tmp/expected
+chroot /tmp/debian-chroot dpkg-query -f '${binary:Package}\n' -W \
+	| comm -12 - /tmp/expected \
+	| diff -u - /tmp/expected
+rm /tmp/expected
+for cmd in echo cat sed grep; do
+	test -L /tmp/debian-chroot/bin/$cmd
+	test "$(readlink /tmp/debian-chroot/bin/$cmd)" = "/bin/busybox"
+done
+for cmd in sort tee; do
+	test -L /tmp/debian-chroot/usr/bin/$cmd
+	test "$(readlink /tmp/debian-chroot/usr/bin/$cmd)" = "/bin/busybox"
+done
+chroot /tmp/debian-chroot echo foobar \
+	| chroot /tmp/debian-chroot cat \
+	| chroot /tmp/debian-chroot sort \
+	| chroot /tmp/debian-chroot tee /dev/null \
+	| chroot /tmp/debian-chroot sed 's/foobar/blubber/' \
+	| chroot /tmp/debian-chroot grep blubber >/dev/null
+rm -r /tmp/debian-chroot

tests/install-doc-debian

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	if [ ! -e /mmdebstrap-testenv ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
+		exit 1
+	fi
+	adduser --gecos user --disabled-password user
+fi
+prefix=
+[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+$prefix {{ CMD }} --mode=chrootless --variant=custom --include=doc-debian {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+tar -C /tmp/debian-chroot --owner=0 --group=0 --numeric-owner --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/debian-chroot.tar .
+tar tvf /tmp/debian-chroot.tar > doc-debian.tar.list
+rm /tmp/debian-chroot.tar
+# delete contents of doc-debian
+rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm -r /tmp/debian-chroot/usr/share/doc/debian
+rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
+# delete real files
+rm /tmp/debian-chroot/etc/apt/sources.list
+rm /tmp/debian-chroot/etc/fstab
+rm /tmp/debian-chroot/etc/hostname
+rm /tmp/debian-chroot/etc/resolv.conf
+rm /tmp/debian-chroot/var/lib/dpkg/status
+rm /tmp/debian-chroot/var/lib/dpkg/arch
+rm /tmp/debian-chroot/var/cache/apt/archives/lock
+rm /tmp/debian-chroot/var/lib/dpkg/lock
+rm /tmp/debian-chroot/var/lib/dpkg/lock-frontend
+rm /tmp/debian-chroot/var/lib/apt/lists/lock
+## delete merged usr symlinks
+#rm /tmp/debian-chroot/libx32
+#rm /tmp/debian-chroot/lib64
+#rm /tmp/debian-chroot/lib32
+#rm /tmp/debian-chroot/sbin
+#rm /tmp/debian-chroot/bin
+#rm /tmp/debian-chroot/lib
+# in chrootless mode, there is more to remove
+rm /tmp/debian-chroot/var/lib/dpkg/triggers/Lock
+rm /tmp/debian-chroot/var/lib/dpkg/triggers/Unincorp
+rm /tmp/debian-chroot/var/lib/dpkg/status-old
+rm /tmp/debian-chroot/var/lib/dpkg/info/format
+rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.md5sums
+rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.list
+# the rest should be empty directories that we can rmdir recursively
+find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir

tests/install-doc-debian-and-output-tarball

@@ -0,0 +1,16 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	if [ ! -e /mmdebstrap-testenv ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
+		exit 1
+	fi
+	adduser --gecos user --disabled-password user
+fi
+prefix=
+[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+$prefix {{ CMD }} --mode=chrootless --variant=custom --include=doc-debian {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar tvf /tmp/debian-chroot.tar | grep -v ' ./dev' | diff -u doc-debian.tar.list -
+rm /tmp/debian-chroot.tar

tests/install-doc-debian-and-test-hooks

@@ -0,0 +1,50 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	if [ ! -e /mmdebstrap-testenv ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
+		exit 1
+	fi
+	adduser --gecos user --disabled-password user
+fi
+prefix=
+[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+$prefix {{ CMD }} --mode=chrootless --skip=cleanup/tmp --variant=custom --include=doc-debian --setup-hook='touch "$1/tmp/setup"' --customize-hook='touch "$1/tmp/customize"' {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+rm /tmp/debian-chroot/tmp/setup
+rm /tmp/debian-chroot/tmp/customize
+tar -C /tmp/debian-chroot --owner=0 --group=0 --numeric-owner --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/debian-chroot.tar .
+tar tvf /tmp/debian-chroot.tar | grep -v ' ./dev' | diff -u doc-debian.tar.list -
+rm /tmp/debian-chroot.tar
+# delete contents of doc-debian
+rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm -r /tmp/debian-chroot/usr/share/doc/debian
+rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
+# delete real files
+rm /tmp/debian-chroot/etc/apt/sources.list
+rm /tmp/debian-chroot/etc/fstab
+rm /tmp/debian-chroot/etc/hostname
+rm /tmp/debian-chroot/etc/resolv.conf
+rm /tmp/debian-chroot/var/lib/dpkg/status
+rm /tmp/debian-chroot/var/lib/dpkg/arch
+rm /tmp/debian-chroot/var/cache/apt/archives/lock
+rm /tmp/debian-chroot/var/lib/dpkg/lock
+rm /tmp/debian-chroot/var/lib/dpkg/lock-frontend
+rm /tmp/debian-chroot/var/lib/apt/lists/lock
+## delete merged usr symlinks
+#rm /tmp/debian-chroot/libx32
+#rm /tmp/debian-chroot/lib64
+#rm /tmp/debian-chroot/lib32
+#rm /tmp/debian-chroot/sbin
+#rm /tmp/debian-chroot/bin
+#rm /tmp/debian-chroot/lib
+# in chrootless mode, there is more to remove
+rm /tmp/debian-chroot/var/lib/dpkg/triggers/Lock
+rm /tmp/debian-chroot/var/lib/dpkg/triggers/Unincorp
+rm /tmp/debian-chroot/var/lib/dpkg/status-old
+rm /tmp/debian-chroot/var/lib/dpkg/info/format
+rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.md5sums
+rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.list
+# the rest should be empty directories that we can rmdir recursively
+find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir

tests/install-libmagic-mgc-on-arm64

@@ -0,0 +1,48 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	if [ ! -e /mmdebstrap-testenv ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
+		exit 1
+	fi
+	adduser --gecos user --disabled-password user
+fi
+prefix=
+[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+$prefix {{ CMD }} --mode=chrootless --variant=custom --architectures=arm64 --include=libmagic-mgc {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+# delete contents of libmagic-mgc
+rm /tmp/debian-chroot/usr/lib/file/magic.mgc
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.Debian.gz
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
+rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
+rm /tmp/debian-chroot/usr/share/file/magic.mgc
+rm /tmp/debian-chroot/usr/share/misc/magic.mgc
+# delete real files
+rm /tmp/debian-chroot/etc/apt/sources.list
+rm /tmp/debian-chroot/etc/fstab
+rm /tmp/debian-chroot/etc/hostname
+rm /tmp/debian-chroot/etc/resolv.conf
+rm /tmp/debian-chroot/var/lib/dpkg/status
+rm /tmp/debian-chroot/var/cache/apt/archives/lock
+rm /tmp/debian-chroot/var/lib/dpkg/lock
+rm /tmp/debian-chroot/var/lib/dpkg/lock-frontend
+rm /tmp/debian-chroot/var/lib/apt/lists/lock
+## delete merged usr symlinks
+#rm /tmp/debian-chroot/libx32
+#rm /tmp/debian-chroot/lib64
+#rm /tmp/debian-chroot/lib32
+#rm /tmp/debian-chroot/sbin
+#rm /tmp/debian-chroot/bin
+#rm /tmp/debian-chroot/lib
+# in chrootless mode, there is more to remove
+rm /tmp/debian-chroot/var/lib/dpkg/arch
+rm /tmp/debian-chroot/var/lib/dpkg/triggers/Lock
+rm /tmp/debian-chroot/var/lib/dpkg/triggers/Unincorp
+rm /tmp/debian-chroot/var/lib/dpkg/status-old
+rm /tmp/debian-chroot/var/lib/dpkg/info/format
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
+rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
+# the rest should be empty directories that we can rmdir recursively
+find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir

tests/invalid-mirror

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+ret=0
+{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}/invalid || ret=$?
+rm /tmp/debian-chroot.tar
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi

tests/jessie-or-older

@@ -0,0 +1,39 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+adduser --gecos user --disabled-password user
+sysctl -w kernel.unprivileged_userns_clone=1
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+
+filter() {
+	./tarfilter \
+		--path-exclude=/usr/bin/uncompress \
+		--path-exclude=/var/cache/debconf/config.dat-old \
+		--path-exclude=/var/cache/debconf/templates.dat-old \
+		--path-exclude=/var/lib/dpkg/available \
+		--path-exclude=/var/lib/dpkg/cmethopt \
+		--path-exclude=/var/lib/dpkg/status-old \
+		--path-exclude=/var/lib/shells.state
+}
+
+# base for comparison without jessie-or-older hook
+{{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-root-normal.tar
+
+# root
+{{ CMD }} --mode=root --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-root.tar
+cmp /tmp/debian-chroot-root-normal.tar /tmp/debian-chroot-root.tar
+rm /tmp/debian-chroot-root.tar
+# unshare
+runuser -u user -- {{ CMD }} --mode=unshare --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-unshare.tar
+cmp /tmp/debian-chroot-root-normal.tar /tmp/debian-chroot-unshare.tar
+rm /tmp/debian-chroot-unshare.tar
+# fakechroot
+runuser -u user -- {{ CMD }} --mode=fakechroot --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-fakechroot.tar
+cmp /tmp/debian-chroot-root-normal.tar /tmp/debian-chroot-fakechroot.tar
+rm /tmp/debian-chroot-fakechroot.tar
+
+rm /tmp/debian-chroot-root-normal.tar

tests/keyring

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+rm /etc/apt/trusted.gpg.d/*.gpg
+{{ CMD }} --mode=root --variant=apt --keyring=/usr/share/keyrings/debian-archive-keyring.gpg --keyring=/usr/share/keyrings/ {{ DIST }} /tmp/debian-chroot "deb {{ MIRROR }} {{ DIST }} main"
+# make sure that no [signedby=...] managed to make it into the sources.list
+echo "deb {{ MIRROR }} {{ DIST }} main" | cmp /tmp/debian-chroot/etc/apt/sources.list -
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+rm -r /tmp/debian-chroot

tests/keyring-overwrites

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot; rmdir /tmp/emptydir; rm -f /tmp/emptyfile" EXIT INT TERM
+mkdir -p /tmp/emptydir
+touch /tmp/emptyfile
+# this overwrites the apt keyring options and should fail
+ret=0
+{{ CMD }} --mode=root --variant=apt --keyring=/tmp/emptydir --keyring=/tmp/emptyfile {{ DIST }} /tmp/debian-chroot "deb {{ MIRROR }} {{ DIST }} main" || ret=$?
+# make sure that no [signedby=...] managed to make it into the sources.list
+echo "deb {{ MIRROR }} {{ DIST }} main" | cmp /tmp/debian-chroot/etc/apt/sources.list -
+if [ "$ret" = 0 ]; then
+	echo expected failure but got exit $ret >&2
+	exit 1
+fi

tests/logfile

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+# we check the full log to also prevent debug printfs to accidentally make it into a commit
+{{ CMD }} --mode=root --variant=apt --logfile=/tmp/log {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+# omit the last line which should contain the runtime
+head --lines=-1 /tmp/log > /tmp/trimmed
+cat << LOG | diff -u - /tmp/trimmed
+I: chroot architecture {{ HOSTARCH }} is equal to the host's architecture
+I: automatically chosen format: directory
+I: running apt-get update...
+I: downloading packages with apt...
+I: extracting archives...
+I: installing essential packages...
+I: cleaning package lists and apt cache...
+LOG
+tail --lines=1 /tmp/log | grep '^I: success in .* seconds$'
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+rm -r /tmp/debian-chroot
+rm /tmp/log /tmp/trimmed

tests/man

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+
+# we redirect to /dev/null instead of using --quiet to not cause a broken pipe
+# when grep exits before mmdebstrap was able to write all its output
+{{ CMD }} --man | grep --fixed-strings 'mmdebstrap [OPTION...] [*SUITE* [*TARGET* [*MIRROR*...]]]' >/dev/null

tests/mirror-is-deb

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar "deb {{ MIRROR }} {{ DIST }} main"
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/mirror-is-real-file

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f /tmp/debian-chroot.tar /tmp/sources.list" EXIT INT TERM
+echo "deb {{ MIRROR }} {{ DIST }} main" > /tmp/sources.list
+{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar /tmp/sources.list
+tar -tf /tmp/debian-chroot.tar \
+	| sed 's#^./etc/apt/sources.list.d/0000sources.list$#./etc/apt/sources.list#' \
+	| sort | diff -u tar1.txt -

tests/mirror-is-stdin

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+echo "deb {{ MIRROR }} {{ DIST }} main" | {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar -
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/missing-dev-sys-proc-inside-the-chroot

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+adduser --gecos user --disabled-password user
+sysctl -w kernel.unprivileged_userns_clone=1
+runuser -u user -- {{ CMD }} --mode=unshare --variant=custom --include=dpkg,dash,diffutils,coreutils,libc-bin,sed {{ DIST }} /dev/null {{ MIRROR }}

tests/missing-device-nodes-outside-the-chroot

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+rm /dev/console
+adduser --gecos user --disabled-password user
+sysctl -w kernel.unprivileged_userns_clone=1
+runuser -u user -- {{ CMD }} --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+rm /tmp/debian-chroot.tar

tests/mount-is-missing

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+for p in /bin /usr/bin /sbin /usr/sbin; do
+	rm -f "$p/mount"
+done
+{{ CMD }} --mode=root --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+rm /tmp/debian-chroot.tar

tests/multiple-include

@@ -0,0 +1,26 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+{{ CMD }} --mode=root --variant=apt --include=doc-debian --include=tzdata {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+rm /tmp/debian-chroot/usr/share/doc-base/debian-*
+rm -r /tmp/debian-chroot/usr/share/doc/debian
+rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
+rm /tmp/debian-chroot/usr/share/lintian/overrides/tzdata
+rm /tmp/debian-chroot/etc/localtime
+rm /tmp/debian-chroot/etc/timezone
+rm /tmp/debian-chroot/usr/sbin/tzconfig
+rm -r /tmp/debian-chroot/usr/share/doc/tzdata
+rm -r /tmp/debian-chroot/usr/share/zoneinfo
+rm /tmp/debian-chroot/var/lib/apt/extended_states
+rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.list
+rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.md5sums
+rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.list
+rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.md5sums
+rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.config
+rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.postinst
+rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.postrm
+rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.templates
+rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.preinst
+rm /tmp/debian-chroot/var/lib/dpkg/info/tzdata.prerm
+tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/no-sbin-in-path

@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# If FAKECHROOT_CMD_SUBST sets up wrong substitutions, then binaries cannot be
+# found. For example if /usr/bin/chroot is listed in FAKECHROOT_CMD_SUBST but
+# /usr/sbin (the actual location of the chroot binary) is not in PATH, the
+# command fails
+
+set -eu
+export LC_ALL=C.UTF-8
+[ "{{ MODE }}" = "fakechroot" ]
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	if [ ! -e /mmdebstrap-testenv ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
+		exit 1
+	fi
+	adduser --gecos user --disabled-password user
+fi
+prefix=
+[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+$prefix env PATH=/usr/bin:/bin fakechroot fakeroot {{ CMD }} --mode=fakechroot --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/not-having-to-install-apt-in-include-because-a-hook-did-it-before

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+{{ CMD }} --mode={{ MODE }} --variant=essential --include=apt \
+	--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get update' \
+	--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$1" apt' \
+	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -