release 1.2.5

/etc/subgid contains a mapping of user names (not group names) to group
ids as defined in man subgid.

.mailmap

@@ -1,2 +1,4 @@
 Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
 Johannes Schauer Marin Rodrigues <josch@mister-muffin.de> <j.schauer@email.de>
+Helmut Grohne <helmut@subdivi.de> <helmut.grohne@intenta.de>
+Benjamin Drung <benjamin.drung@ionos.com> <benjamin.drung@cloud.ionos.com>

CHANGELOG.md

@@ -1,3 +1,71 @@
+1.2.5 (2023-01-04)
+------------------
+
+ - bugfix release
+
+1.2.4 (2022-12-23)
+------------------
+
+ - bugfix release
+ - add jessie-or-older extract hook
+
+1.2.3 (2022-11-16)
+------------------
+
+ - use Text::ParseWords::shellwords instead of spawning a new shell
+ - mount and unmount once, instead for each run_chroot() call
+
+1.2.2 (2022-10-27)
+------------------
+
+ - allow /etc/apt/trusted.gpg.d/ not to exist
+ - always create /var/lib/dpkg/arch to make foreign architecture chrootless
+   tarballs bit-by-bit identical
+ - write an empty /etc/machine-id instead of writing 'uninitialized'
+ - only print progress bars on interactive terminals that are wide enough
+
+1.2.1 (2022-09-08)
+------------------
+
+ - bugfix release
+
+1.2.0 (2022-09-05)
+------------------
+
+ - remove proot mode
+ - error out if stdout is an interactive terminal
+ - replace taridshift by tarfilter --idshift
+ - tarfilter: add --transform option
+ - multiple --skip options can be separated by comma or whitespace
+ - also cleanup the contents of /run
+ - support apt patterns and paths with commas and whitespace in --include
+ - hooks: store the values of the --include option in MMDEBSTRAP_INCLUDE
+ - add new --skip options: chroot/start-stop-daemon, chroot/policy-rc.d
+   chroot/mount, chroot/mount/dev, chroot/mount/proc, chroot/mount/sys,
+   cleanup/run
+
+1.1.0 (2022-07-26)
+----------------
+
+ - mount a new /dev/pts instance into the chroot to make posix_openpt work
+ - adjust merged-/usr hook to work the same way as debootstrap
+ - add no-merged-usr hook
+
+1.0.1 (2022-05-29)
+------------------
+
+ - bugfix release
+
+1.0.0 (2022-05-28)
+------------------
+
+ - all documented interfaces are now considered stable
+ - allow file:// mirrors
+ - /var/cache/apt/archives/ is now allowed to contain *.deb packages
+ - add file-mirror-automount hook-dir
+ - set $MMDEBSTRAP_VERBOSITY in hooks
+ - rewrite coverage with multiple individual and skippable shell scripts
+
 0.8.6 (2022-03-25)
 ------------------
 

README.md

@@ -34,7 +34,7 @@ Summary:
  - chroot with apt in 11 seconds
  - gzipped tarball with apt is 27M small
  - bit-by-bit reproducible output
- - unprivileged operation using Linux user namespaces, fakechroot or proot
+ - unprivileged operation using Linux user namespaces or fakechroot
  - can operate on filesystems mounted with nodev
  - foreign architecture chroots with qemu-user
  - variant installing only Essential:yes packages and dependencies
@@ -78,9 +78,9 @@ privileges to create a file (the chroot tarball) in one's home directory.
 Thus, mmdebstrap provides multiple options to create a chroot tarball with the
 right permissions **without superuser privileges**. This avoids a whole class
 of bugs like #921815. Depending on what is available, it uses either Linux user
-namespaces, fakechroot or proot.  Debootstrap supports fakechroot but will not
+namespaces or fakechroot. Debootstrap supports fakechroot but will not
 create a tarball with the right permissions by itself. Support for Linux user
-namespaces and proot is missing (see bugs #829134 and #698347, respectively).
+namespaces is missing (see #829134).
 
 When creating a chroot tarball with debootstrap, the temporary chroot directory
 cannot be on a filesystem that has been mounted with nodev. In unprivileged
@@ -137,6 +137,11 @@ By default, `coverage.sh` will skip running a single test which tries creating
 a Ubuntu Focal chroot. To not skip that test, run `coverage.sh` with the
 environment variable `ONLINE=yes`.
 
+If a test fails you can run individual tests by executing `coverage.py` with
+the test name and optionally limit it to a specific distribution like so:
+
+    CMD=./mmdebstrap ./coverage.py --dist unstable check-against-debootstrap-dist
+
 Bugs
 ====
 
@@ -147,11 +152,16 @@ Contributors
 ============
 
  - Johannes Schauer Marin Rodrigues (main author)
+ - Gioele Barabucci
  - Helmut Grohne
  - Benjamin Drung
- - Steve Dodd
+ - Jochen Sprickerhof
  - Josh Triplett
  - Konstantin Demin
+ - David Kalnischkies
+ - Joe Groocock
+ - Nicolas Vigier
+ - Raul Tambre
+ - Steve Dodd
  - Trent W. Buck
  - Vagrant Cascadian
- - Gioele Barabucci

coverage.py

@@ -0,0 +1,408 @@
+#!/usr/bin/env python3
+
+from debian.deb822 import Deb822, Release
+import email.utils
+import os
+import sys
+import shutil
+import subprocess
+import argparse
+import time
+from datetime import timedelta
+from collections import defaultdict
+from itertools import product
+
+have_qemu = os.getenv("HAVE_QEMU", "yes") == "yes"
+have_unshare = os.getenv("HAVE_UNSHARE", "yes") == "yes"
+have_binfmt = os.getenv("HAVE_BINFMT", "yes") == "yes"
+run_ma_same_tests = os.getenv("RUN_MA_SAME_TESTS", "yes") == "yes"
+cmd = os.getenv("CMD", "./mmdebstrap")
+
+default_dist = os.getenv("DEFAULT_DIST", "unstable")
+all_dists = ["oldstable", "stable", "testing", "unstable"]
+default_mode = "auto" if have_unshare else "root"
+all_modes = ["auto", "root", "unshare", "fakechroot", "chrootless"]
+default_variant = "apt"
+all_variants = [
+    "extract",
+    "custom",
+    "essential",
+    "apt",
+    "minbase",
+    "buildd",
+    "-",
+    "standard",
+]
+default_format = "auto"
+all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "null"]
+
+mirror = os.getenv("mirror", "http://127.0.0.1/debian")
+hostarch = subprocess.check_output(["dpkg", "--print-architecture"]).decode().strip()
+
+release_path = f"./shared/cache/debian/dists/{default_dist}/Release"
+if not os.path.exists(release_path):
+    print("path doesn't exist:", release_path, file=sys.stderr)
+    print("run ./make_mirror.sh first", file=sys.stderr)
+    exit(1)
+if os.getenv("SOURCE_DATE_EPOCH") is not None:
+    s_d_e = os.getenv("SOURCE_DATE_EPOCH")
+else:
+    with open(release_path) as f:
+        rel = Release(f)
+    s_d_e = str(email.utils.mktime_tz(email.utils.parsedate_tz(rel["Date"])))
+
+separator = (
+    "------------------------------------------------------------------------------"
+)
+
+
+def skip(condition, dist, mode, variant, fmt):
+    if not condition:
+        return ""
+    for line in condition.splitlines():
+        if not line:
+            continue
+        if eval(line):
+            return line.strip()
+    return ""
+
+
+def parse_config(confname):
+    config_dict = defaultdict(dict)
+    config_order = list()
+    all_vals = {
+        "Dists": all_dists,
+        "Modes": all_modes,
+        "Variants": all_variants,
+        "Formats": all_formats,
+    }
+    with open(confname) as f:
+        for test in Deb822.iter_paragraphs(f):
+            if "Test" not in test.keys():
+                print("Test without name", file=sys.stderr)
+                exit(1)
+            name = test["Test"]
+            config_order.append(name)
+            for k in test.keys():
+                v = test[k]
+                if k not in [
+                    "Test",
+                    "Dists",
+                    "Modes",
+                    "Variants",
+                    "Formats",
+                    "Skip-If",
+                    "Needs-QEMU",
+                    "Needs-Root",
+                ]:
+                    print(f"Unknown field name {k} in test {name}")
+                    exit(1)
+                if k in all_vals.keys():
+                    if v == "default":
+                        print(
+                            f"Setting {k} to default in Test {name} is redundant",
+                            file=sys.stderr,
+                        )
+                        exit(1)
+                    if v == "any":
+                        v = all_vals[k]
+                    else:
+                        # else, split the value by whitespace
+                        v = v.split()
+                        for i in v:
+                            if i not in all_vals[k]:
+                                print(
+                                    f"{i} is not a valid value for {k}", file=sys.stderr
+                                )
+                                exit(1)
+                config_dict[name][k] = v
+    return config_order, config_dict
+
+
+def format_failed(num, total, name, dist, mode, variant, fmt, config_dict):
+    ret = f"({num}/{total}) {name}"
+    if len(config_dict[name].get("Dists", [])) > 1:
+        ret += f" --dist={dist}"
+    if len(config_dict[name].get("Modes", [])) > 1:
+        ret += f" --mode={mode}"
+    if len(config_dict[name].get("Variants", [])) > 1:
+        ret += f" --variant={variant}"
+    if len(config_dict[name].get("Formats", [])) > 1:
+        ret += f" --format={fmt}"
+    return ret
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("test", nargs="*", help="only run these tests")
+    parser.add_argument(
+        "-x",
+        "--exitfirst",
+        action="store_const",
+        dest="maxfail",
+        const=1,
+        help="exit instantly on first error or failed test.",
+    )
+    parser.add_argument(
+        "--maxfail",
+        metavar="num",
+        action="store",
+        type=int,
+        dest="maxfail",
+        default=0,
+        help="exit after first num failures or errors.",
+    )
+    parser.add_argument(
+        "--mode",
+        metavar="mode",
+        help=f"only run tests with this mode (Default = {default_mode})",
+    )
+    parser.add_argument(
+        "--dist",
+        metavar="dist",
+        help=f"only run tests with this dist (Default = {default_dist})",
+    )
+    parser.add_argument(
+        "--variant",
+        metavar="variant",
+        help=f"only run tests with this variant (Default = {default_variant})",
+    )
+    parser.add_argument(
+        "--format",
+        metavar="format",
+        help=f"only run tests with this format (Default = {default_format})",
+    )
+    parser.add_argument(
+        "--skip", metavar="test", action="append", help="skip this test"
+    )
+    args = parser.parse_args()
+
+    # copy over files from git or as distributed
+    for (git, dist, target) in [
+        ("./mmdebstrap", "/usr/bin/mmdebstrap", "mmdebstrap"),
+        ("./tarfilter", "/usr/bin/mmtarfilter", "tarfilter"),
+        (
+            "./proxysolver",
+            "/usr/lib/apt/solvers/mmdebstrap-dump-solution",
+            "proxysolver",
+        ),
+        (
+            "./ldconfig.fakechroot",
+            "/usr/libexec/mmdebstrap/ldconfig.fakechroot",
+            "ldconfig.fakechroot",
+        ),
+    ]:
+        if os.path.exists(git):
+            shutil.copy(git, f"shared/{target}")
+        else:
+            shutil.copy(dist, f"shared/{target}")
+    # copy over hooks from git or as distributed
+    if os.path.exists("hooks"):
+        shutil.copytree("hooks", "shared/hooks", dirs_exist_ok=True)
+    else:
+        shutil.copytree(
+            "/usr/share/mmdebstrap/hooks", "shared/hooks", dirs_exist_ok=True
+        )
+
+    # parse coverage.txt
+    config_order, config_dict = parse_config("coverage.txt")
+
+    indirbutnotcovered = set(
+        [d for d in os.listdir("tests") if not d.startswith(".")]
+    ) - set(config_order)
+    if indirbutnotcovered:
+        print(
+            "test(s) missing from coverage.txt: %s"
+            % (", ".join(sorted(indirbutnotcovered))),
+            file=sys.stderr,
+        )
+        exit(1)
+    coveredbutnotindir = set(config_order) - set(
+        [d for d in os.listdir("tests") if not d.startswith(".")]
+    )
+    if coveredbutnotindir:
+        print(
+            "test(s) missing from ./tests: %s"
+            % (", ".join(sorted(coveredbutnotindir))),
+            file=sys.stderr,
+        )
+
+        exit(1)
+
+    # produce the list of tests using the cartesian product of all allowed
+    # dists, modes, variants and formats of a given test
+    tests = []
+    for name in config_order:
+        test = config_dict[name]
+        for dist, mode, variant, fmt in product(
+            test.get("Dists", [default_dist]),
+            test.get("Modes", [default_mode]),
+            test.get("Variants", [default_variant]),
+            test.get("Formats", [default_format]),
+        ):
+            skipreason = skip(test.get("Skip-If"), dist, mode, variant, fmt)
+            if skipreason:
+                tt = ("skip", skipreason)
+            elif have_qemu:
+                tt = "qemu"
+            elif test.get("Needs-QEMU", "false") == "true":
+                tt = ("skip", "test needs QEMU")
+            elif test.get("Needs-Root", "false") == "true":
+                tt = "sudo"
+            elif mode == "auto" and not have_unshare:
+                tt = "sudo"
+            elif mode == "root":
+                tt = "sudo"
+            elif mode == "unshare" and not have_unshare:
+                tt = ("skip", "test needs unshare")
+            else:
+                tt = "null"
+            tests.append((tt, name, dist, mode, variant, fmt))
+
+    torun = []
+    num_tests = len(tests)
+    if args.test:
+        # check if all given tests are either a valid name or a valid number
+        for test in args.test:
+            if test in [name for (_, name, _, _, _, _) in tests]:
+                continue
+            if not test.isdigit():
+                print(f"cannot find test named {test}", file=sys.stderr)
+                exit(1)
+            if int(test) >= len(tests) or int(test) <= 0 or str(int(test)) != test:
+                print(f"test number {test} doesn't exist", file=sys.stderr)
+                exit(1)
+
+        for i, (_, name, _, _, _, _) in enumerate(tests):
+            # if either the number or test name matches, then we use this test,
+            # otherwise we skip it
+            if name in args.test:
+                torun.append(i)
+            if str(i + 1) in args.test:
+                torun.append(i)
+        num_tests = len(torun)
+
+    starttime = time.time()
+    skipped = defaultdict(list)
+    failed = []
+    num_success = 0
+    num_finished = 0
+    for i, (test, name, dist, mode, variant, fmt) in enumerate(tests):
+        if torun and i not in torun:
+            continue
+        print(separator, file=sys.stderr)
+        print("(%d/%d) %s" % (i + 1, len(tests), name), file=sys.stderr)
+        print("dist: %s" % dist, file=sys.stderr)
+        print("mode: %s" % mode, file=sys.stderr)
+        print("variant: %s" % variant, file=sys.stderr)
+        print("format: %s" % fmt, file=sys.stderr)
+        if num_finished > 0:
+            currenttime = time.time()
+            timeleft = timedelta(
+                seconds=int(
+                    (num_tests - num_finished)
+                    * (currenttime - starttime)
+                    / num_finished
+                )
+            )
+            print("time left: %s" % timeleft, file=sys.stderr)
+        if failed:
+            print("failed: %d" % len(failed), file=sys.stderr)
+        num_finished += 1
+        with open("tests/" + name) as fin, open("shared/test.sh", "w") as fout:
+            for line in fin:
+                line = line.replace("{{ CMD }}", cmd)
+                line = line.replace("{{ SOURCE_DATE_EPOCH }}", s_d_e)
+                line = line.replace("{{ DIST }}", dist)
+                line = line.replace("{{ MIRROR }}", mirror)
+                line = line.replace("{{ MODE }}", mode)
+                line = line.replace("{{ VARIANT }}", variant)
+                line = line.replace("{{ FORMAT }}", fmt)
+                line = line.replace("{{ HOSTARCH }}", hostarch)
+                fout.write(line)
+        # ignore:
+        # SC2016 Expressions don't expand in single quotes, use double quotes for that.
+        # SC2050 This expression is constant. Did you forget the $ on a variable?
+        # SC2194 This word is constant. Did you forget the $ on a variable?
+        shellcheck = subprocess.run(
+            [
+                "shellcheck",
+                "--exclude=SC2050,SC2194,SC2016",
+                "-f",
+                "gcc",
+                "shared/test.sh",
+            ],
+            check=False,
+            stdout=subprocess.PIPE,
+        ).stdout.decode()
+        argv = None
+        match test:
+            case "qemu":
+                argv = ["./run_qemu.sh"]
+            case "sudo":
+                argv = ["./run_null.sh", "SUDO"]
+            case "null":
+                argv = ["./run_null.sh"]
+            case ("skip", reason):
+                skipped[reason].append(
+                    ("(%d/%d) %s" % (i + 1, len(tests), name), dist, mode, variant, fmt)
+                )
+                print(f"skipped because of {reason}", file=sys.stderr)
+                continue
+        print(separator, file=sys.stderr)
+        if args.skip and name in args.skip: