tests: skip debootstrap tests variant - on oldstable because of #917773

- only download Release files once and not by apt as well as with curl
   and thus avoid a mirror push happening between both downloads
 - no heuristic needed to place the file in their correct mirror
   location
 - no manual checksum checking
 - only throttle download speed when actually downloading and not when
   retrieving files from the cache
 - no translation of filenames between how the epoch colon is stored in
   files in /var/cache/apt/archives versus how it is stored in files on
   the mirrors
 - no special handling of stable update and security mirrors
 - implemented in Python instead of shell and thus an order of magnitude
   faster

CHANGELOG.md

@@ -1,3 +1,13 @@
+1.3.3 (2023-02-19)
+------------------
+
+ - testsuite improvements
+
+1.3.2 (2023-02-16)
+------------------
+
+ - unshare mode works in privileged docker containers
+
 1.3.1 (2023-01-20)
 ------------------
 

caching_proxy.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import time
+import http.client
+import http.server
+from io import StringIO
+import pathlib
+import urllib.parse
+
+oldcachedir = None
+newcachedir = None
+readonly = False
+
+
+class ProxyRequestHandler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        assert int(self.headers.get("Content-Length", 0)) == 0
+        assert self.headers["Host"]
+        pathprefix = "http://" + self.headers["Host"] + "/"
+        assert self.path.startswith(pathprefix)
+        sanitizedpath = urllib.parse.unquote(self.path.removeprefix(pathprefix))
+        oldpath = oldcachedir / sanitizedpath
+        newpath = newcachedir / sanitizedpath
+
+        if not readonly:
+            newpath.parent.mkdir(parents=True, exist_ok=True)
+
+        # just send back to client
+        if newpath.exists():
+            print(f"proxy cached: {self.path}", file=sys.stderr)
+            self.wfile.write(b"HTTP/1.1 200 OK\r\n")
+            self.send_header("Content-Length", newpath.stat().st_size)
+            self.end_headers()
+            with newpath.open(mode="rb") as new:
+                while True:
+                    buf = new.read(64 * 1024)  # same as shutil uses
+                    if not buf:
+                        break
+                    self.wfile.write(buf)
+            self.wfile.flush()
+            return
+
+        if readonly:
+            newpath = pathlib.Path("/dev/null")
+
+        # copy from oldpath to newpath and send back to client
+        # Only take files from the old cache if they are .deb files or Packages
+        # files in the by-hash directory as only those are unique by their path
+        # name. Other files like InRelease files have to be downloaded afresh.
+        if oldpath.exists() and (
+            oldpath.suffix == ".deb" or "by-hash" in oldpath.parts
+        ):
+            print(f"proxy cached: {self.path}", file=sys.stderr)
+            self.wfile.write(b"HTTP/1.1 200 OK\r\n")
+            self.send_header("Content-Length", oldpath.stat().st_size)
+            self.end_headers()
+            with oldpath.open(mode="rb") as old, newpath.open(mode="wb") as new:
+                while True:
+                    buf = old.read(64 * 1024)  # same as shutil uses
+                    if not buf:
+                        break
+                    self.wfile.write(buf)
+                    new.write(buf)
+            self.wfile.flush()
+            return
+
+        # download fresh copy
+        try:
+            print(f"\rproxy download: {self.path}", file=sys.stderr)
+            conn = http.client.HTTPConnection(self.headers["Host"], timeout=5)
+            conn.request("GET", self.path, None, dict(self.headers))
+            res = conn.getresponse()
+            assert (res.status, res.reason) == (200, "OK"), (res.status, res.reason)
+            self.wfile.write(b"HTTP/1.1 200 OK\r\n")
+            for k, v in res.getheaders():
+                # do not allow a persistent connection
+                if k == "connection":
+                    continue
+                self.send_header(k, v)
+            self.end_headers()
+            with newpath.open(mode="wb") as f:
+                while True:
+                    buf = res.read(64 * 1024)  # same as shutil uses
+                    if not buf:
+                        break
+                    self.wfile.write(buf)
+                    f.write(buf)
+                    time.sleep(64 / 1024)  # 1024 kB/s
+            self.wfile.flush()
+        except Exception as e:
+            self.send_error(502)
+
+
+def main():
+    global oldcachedir, newcachedir, readonly
+    if sys.argv[1] == "--readonly":
+        readonly = True
+        oldcachedir = pathlib.Path(sys.argv[2])
+        newcachedir = pathlib.Path(sys.argv[3])
+    else:
+        oldcachedir = pathlib.Path(sys.argv[1])
+        newcachedir = pathlib.Path(sys.argv[2])
+    print(f"starting caching proxy for {newcachedir}", file=sys.stderr)
+    httpd = http.server.ThreadingHTTPServer(
+        server_address=("", 8080), RequestHandlerClass=ProxyRequestHandler
+    )
+    httpd.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

coverage.py

@@ -13,14 +13,14 @@ from collections import defaultdict
 from itertools import product
 
 have_qemu = os.getenv("HAVE_QEMU", "yes") == "yes"
-have_unshare = os.getenv("HAVE_UNSHARE", "yes") == "yes"
 have_binfmt = os.getenv("HAVE_BINFMT", "yes") == "yes"
 run_ma_same_tests = os.getenv("RUN_MA_SAME_TESTS", "yes") == "yes"
+use_host_apt_config = os.getenv("USE_HOST_APT_CONFIG", "no") == "yes"
 cmd = os.getenv("CMD", "./mmdebstrap")
 
 default_dist = os.getenv("DEFAULT_DIST", "unstable")
 all_dists = ["oldstable", "stable", "testing", "unstable"]
-default_mode = "auto" if have_unshare else "root"
+default_mode = "auto"
 all_modes = ["auto", "root", "unshare", "fakechroot", "chrootless"]
 default_variant = "apt"
 all_variants = [
@@ -39,7 +39,7 @@ all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "null"]
 mirror = os.getenv("mirror", "http://127.0.0.1/debian")
 hostarch = subprocess.check_output(["dpkg", "--print-architecture"]).decode().strip()
 
-release_path = f"./shared/cache/debian/dists/{default_dist}/Release"
+release_path = f"./shared/cache/debian/dists/{default_dist}/InRelease"
 if not os.path.exists(release_path):
     print("path doesn't exist:", release_path, file=sys.stderr)
     print("run ./make_mirror.sh first", file=sys.stderr)
@@ -94,6 +94,7 @@ def parse_config(confname):
                     "Skip-If",
                     "Needs-QEMU",
                     "Needs-Root",
+                    "Needs-APT-Config",
                 ]:
                     print(f"Unknown field name {k} in test {name}")
                     exit(1)
@@ -202,7 +203,7 @@ def main():
     args = parser.parse_args()
 
     # copy over files from git or as distributed
-    for (git, dist, target) in [
+    for git, dist, target in [
         ("./mmdebstrap", "/usr/bin/mmdebstrap", "mmdebstrap"),
         ("./tarfilter", "/usr/bin/mmtarfilter", "tarfilter"),
         (
@@ -267,18 +268,18 @@ def main():
             skipreason = skip(test.get("Skip-If"), dist, mode, variant, fmt)
             if skipreason:
                 tt = ("skip", skipreason)
+            elif (
+                test.get("Needs-APT-Config", "false") == "true" and use_host_apt_config
+            ):
+                tt = ("skip", "test cannot use host apt config")
             elif have_qemu:
                 tt = "qemu"
             elif test.get("Needs-QEMU", "false") == "true":
                 tt = ("skip", "test needs QEMU")
             elif test.get("Needs-Root", "false") == "true":
                 tt = "sudo"
-            elif mode == "auto" and not have_unshare:
-                tt = "sudo"
             elif mode == "root":
                 tt = "sudo"
-            elif mode == "unshare" and not have_unshare:
-                tt = ("skip", "test needs unshare")
             else:
                 tt = "null"
             tests.append((tt, name, dist, mode, variant, fmt))
@@ -371,7 +372,9 @@ def main():
                 argv = ["./run_null.sh"]
             case ("skip", reason):
                 skipped[reason].append(
-                    ("(%d/%d) %s" % (i + 1, len(tests), name), dist, mode, variant, fmt)
+                    format_test(
+                        i + 1, len(tests), name, dist, mode, variant, fmt, config_dict
+                    )
                 )
                 print(f"skipped because of {reason}", file=sys.stderr)
                 continue

coverage.sh

@@ -22,8 +22,10 @@ if [ -e ./mmdebstrap ]; then
 	perlcritic --severity 4 --verbose 8 ./mmdebstrap
 fi
 
-[ -e ./tarfilter ] && black --check ./tarfilter
+for f in tarfilter coverage.py caching_proxy.py; do
-[ -e ./coverage.py ] && black --check ./coverage.py
+	[ -e "./$f" ] || continue
+	black --check "./$f"
+done
 
 shellcheck --exclude=SC2016 coverage.sh make_mirror.sh run_null.sh run_qemu.sh gpgvnoexpkeysig hooks/*/*.sh
 
@@ -51,21 +53,6 @@ if [ "$HAVE_QEMU" = "yes" ]; then
 	fi
 fi
 
-# check if all required debootstrap tarballs exist
-notfound=0
-for dist in oldstable stable testing unstable; do
-	for variant in minbase buildd -; do
-		if [ ! -e "shared/cache/debian-$dist-$variant.tar" ]; then
-			echo "shared/cache/debian-$dist-$variant.tar does not exist" >&2
-			notfound=1
-		fi
-	done
-done
-if [ "$notfound" -ne 0 ]; then
-	echo "not all required debootstrap tarballs are present" >&2
-	exit 1
-fi
-
 # choose the timestamp of the unstable Release file, so that we get
 # reproducible results for the same mirror timestamp
 SOURCE_DATE_EPOCH=$(date --date="$(grep-dctrl -s Date -n '' "$mirrordir/dists/$DEFAULT_DIST/Release")" +%s)
@@ -73,7 +60,6 @@ SOURCE_DATE_EPOCH=$(date --date="$(grep-dctrl -s Date -n '' "$mirrordir/dists/$D
 # for traditional sort order that uses native byte values
 export LC_ALL=C.UTF-8
 
-: "${HAVE_UNSHARE:=yes}"
 : "${HAVE_BINFMT:=yes}"
 
 # by default, use the mmdebstrap executable in the current directory together
@@ -81,7 +67,7 @@ export LC_ALL=C.UTF-8
 : "${CMD:=perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap}"
 mirror="http://127.0.0.1/debian"
 
-export HAVE_QEMU HAVE_UNSHARE HAVE_BINFMT RUN_MA_SAME_TESTS DEFAULT_DIST SOURCE_DATE_EPOCH CMD mirror
+export HAVE_QEMU HAVE_BINFMT RUN_MA_SAME_TESTS DEFAULT_DIST SOURCE_DATE_EPOCH CMD mirror
 
 ./coverage.py
 
@@ -99,8 +85,6 @@ cover -delete cover_db >&2
 END
 	if [ "$HAVE_QEMU" = "yes" ]; then
 		./run_qemu.sh
-	elif [ "$HAVE_UNSHARE" != "yes" ]; then
-		./run_null.sh SUDO
 	else
 		./run_null.sh
 	fi
@@ -110,4 +94,4 @@ END
 	echo
 fi
 
-rm shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/tarfilter shared/proxysolver
+rm -f shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/tarfilter shared/proxysolver

coverage.txt

@@ -1,10 +1,23 @@
+Test: debootstrap
+Dists: any
+Variants: minbase buildd -
+Needs-Root: true
+Needs-APT-Config: true
+Skip-If: variant == "-" and dist == "oldstable" #917773
+
 Test: check-against-debootstrap-dist
 Dists: any
 Variants: minbase buildd -
 Needs-Root: true
+Needs-APT-Config: true
+Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: as-debootstrap-unshare-wrapper
-Needs-QEMU: true
+Modes: unshare
+Needs-Root: true
+Variants: minbase -
+Needs-APT-Config: true
+Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: help
 
@@ -20,6 +33,7 @@ Needs-Root: true
 
 Test: dist-using-codename
 Dists: any
+Needs-APT-Config: true
 
 Test: fail-without-etc-subuid
 Needs-QEMU: true
@@ -29,12 +43,15 @@ Needs-QEMU: true
 
 Test: unshare-as-root-user-inside-chroot
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: root-mode-inside-chroot
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: root-mode-inside-unshare-chroot
-Needs-QEMU: true
+Modes: unshare
+Needs-APT-Config: true
 
 Test: root-without-cap-sys-admin
 Needs-Root: true
@@ -42,8 +59,21 @@ Needs-Root: true
 Test: mount-is-missing
 Needs-QEMU: true
 
+Test: mmdebstrap
+Needs-Root: true
+Modes: root
+Formats: tar squashfs ext2
+Variants: essential apt minbase buildd - standard
+Skip-If:
+ variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
+ variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
+ fmt == "squashfs" and dist == "oldstable" # squashfs-tools-ng is not available
+ fmt == "ext2" and dist == "oldstable" # genext2fs does not support SOURCE_DATE_EPOCH
+ mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
+ variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
+
 Test: check-for-bit-by-bit-identical-format-output
-Needs-QEMU: true
+Modes: unshare fakechroot
 Formats: tar squashfs ext2
 Variants: essential apt minbase buildd - standard
 Skip-If:
@@ -51,6 +81,8 @@ Skip-If:
  variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
  fmt == "squashfs" and dist == "oldstable" # squashfs-tools-ng is not available
  fmt == "ext2" and dist == "oldstable" # genext2fs does not support SOURCE_DATE_EPOCH
+ mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
+ variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
 Test: tarfilter-idshift
 Needs-QEMU: true
@@ -74,19 +106,21 @@ Test: missing-device-nodes-outside-the-chroot
 Needs-QEMU: true
 
 Test: missing-dev-sys-proc-inside-the-chroot
-Needs-QEMU: true
+Modes: unshare
+Variants: custom
 
 Test: chroot-directory-not-accessible-by-apt-user
 Needs-Root: true
 
 Test: cwd-directory-not-accessible-by-unshared-user
-Needs-QEMU: true
+Needs-Root: true
+Modes: unshare
 
 Test: create-gzip-compressed-tarball
-Needs-QEMU: true
 
 Test: custom-tmpdir
-Needs-QEMU: true
+Needs-Root: true
+Modes: unshare
 
 Test: xz-compressed-tarball
 
@@ -109,6 +143,7 @@ Test: read-from-stdin-write-to-stdout
 Test: supply-components-manually
 Modes: root
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: stable-default-mirror
 Needs-QEMU: true
@@ -133,19 +168,23 @@ Needs-QEMU: true
 Test: mirror-is-deb
 
 Test: mirror-is-real-file
+Needs-APT-Config: true
 
 Test: deb822-1-2
 Modes: root
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: deb822-2-2
 Modes: root
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: automatic-mirror-from-suite
 Needs-QEMU: true
 
 Test: invalid-mirror
+Needs-APT-Config: true
 
 Test: fail-installing-to-root
 Modes: root
@@ -167,12 +206,14 @@ Skip-If:
 
 Test: include-libmagic-mgc-arm64
 Needs-Root: true
+Needs-APT-Config: true
 Skip-If:
  hostarch != "amd64"
  not run_ma_same_tests
 
 Test: include-libmagic-mgc-arm64-with-multiple-arch-options
 Needs-Root: true
+Needs-APT-Config: true
 Skip-If:
  hostarch != "amd64"
  not run_ma_same_tests
@@ -185,6 +226,7 @@ Needs-QEMU: true
 
 Test: keyring-overwrites
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: signed-by-without-host-keys
 Needs-QEMU: true
@@ -194,6 +236,7 @@ Needs-QEMU: true
 
 Test: signed-by-with-host-keys
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: dpkgopt
 Needs-Root: true
@@ -227,13 +270,14 @@ Needs-Root: true
 
 Test: special-hooks-using-helpers
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: special-hooks-using-helpers-and-env-vars
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: special-hooks-with-mode-mode
 Modes: root unshare fakechroot
-Needs-QEMU: true
 
 Test: debootstrap-no-op-options
 Needs-Root: true
@@ -249,6 +293,7 @@ Needs-Root: true
 
 Test: logfile
 Needs-Root: true
+Needs-APT-Config: true
 
 Test: without-etc-resolv-conf-and-etc-hostname
 Needs-QEMU: true
@@ -274,19 +319,21 @@ Skip-If:
  variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
 
 Test: create-directory-dry-run
+Modes: root
 
 Test: create-tarball-dry-run
 Variants: any
 Modes: any
 
 Test: unpack-doc-debian
-Needs-QEMU: true
+Modes: root fakechroot
-Modes: any
 Variants: extract
+Needs-APT-Config: true
 
 Test: install-doc-debian
 Modes: chrootless
 Variants: custom
+Needs-APT-Config: true
 
 Test: chrootless
 Variants: essential
@@ -298,9 +345,9 @@ Skip-If:
 Test: chrootless-fakeroot
 Variants: essential
 Modes: chrootless
-Needs-QEMU: true
 Skip-If:
  dist in ["oldstable", "stable"]
+ hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
 
 Test: chrootless-foreign
 Variants: essential
@@ -314,12 +361,16 @@ Needs-QEMU: true
 Test: install-doc-debian-and-output-tarball
 Variants: custom
 Modes: chrootless
+Needs-APT-Config: true
 
 Test: install-doc-debian-and-test-hooks
 Variants: custom
 Modes: chrootless
+Needs-APT-Config: true
 
 Test: install-libmagic-mgc-on-arm64
+Variants: custom
+Modes: chrootless
 Skip-If:
  hostarch != "amd64"
  not have_binfmt
@@ -339,25 +390,26 @@ Modes: fakechroot
 
 Test: dev-ptmx
 Modes: root unshare
-Needs-QEMU: true
 
 Test: error-if-stdout-is-tty
 
 Test: variant-custom-timeout
 
 Test: include-deb-file
+Needs-APT-Config: true
 
 Test: unshare-include-deb
 Modes: unshare
-Needs-QEMU: true
 
 Test: pivot_root
 Modes: root unshare
-Needs-QEMU: true
+Needs-APT-Config: true
 
 Test: jessie-or-older
-Needs-QEMU: true
+Needs-Root: true
+Modes: root unshare fakechroot
 Variants: essential apt minbase
+Skip-If: mode == "fakechroot" and hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
 
 Test: apt-patterns
 
@@ -367,3 +419,8 @@ Test: empty-sources.list
 
 Test: merged-fakechroot-inside-unmerged-chroot
 Needs-Root: true
+Needs-APT-Config: true
+Skip-If: hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
+
+Test: auto-mode-as-normal-user
+Modes: auto

make_mirror.sh

@@ -92,76 +92,11 @@ deletecache() {
 }
 
 cleanup_newcachedir() {
+	kill "$PROXYPID" || :
 	echo "running cleanup_newcachedir"
 	deletecache "$newcachedir"
 }
 
-get_oldaptnames() {
-	if [ ! -e "$1/$2" ]; then
-		return
-	fi
-	xz -dc "$1/$2" \
-		| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename '' \
-		| paste -sd "    \n" \
-		| while read -r name ver arch fname; do
-			if [ ! -e "$1/$fname" ]; then
-				continue
-			fi
-			# apt stores deb files with the colon encoded as %3a while
-			# mirrors do not contain the epoch at all #645895
-			case "$ver" in *:*) ver="${ver%%:*}%3a${ver#*:}";; esac
-			aptname="$rootdir/var/cache/apt/archives/${name}_${ver}_${arch}.deb"
-			# we have to cp and not mv because other
-			# distributions might still need this file
-			# we have to cp and not symlink because apt
-			# doesn't recognize symlinks
-			cp --link "$1/$fname" "$aptname"
-			echo "$aptname"
-		done
-}
-
-get_newaptnames() {
-	if [ ! -e "$1/$2" ]; then
-		return
-	fi
-	# skip empty files by trying to uncompress the first byte of the payload
-	if [ "$(xz -dc "$1/$2" | head -c1 | wc -c)" -eq 0 ]; then
-		return
-	fi
-	xz -dc "$1/$2" \
-		| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename,SHA256 '' \
-		| paste -sd "     \n" \
-		| while read -r name ver arch fname hash; do
-			# sanity check for the hash because sometimes the
-			# archive switches the hash algorithm
-			if [ "${#hash}" -ne 64 ]; then
-				echo "expected hash length of 64 but got ${#hash} for: $hash" >&2
-				exit 1
-			fi
-			dir="${fname%/*}"
-			# apt stores deb files with the colon encoded as %3a while
-			# mirrors do not contain the epoch at all #645895
-			case "$ver" in *:*) ver="${ver%%:*}%3a${ver#*:}";; esac
-			aptname="$rootdir/var/cache/apt/archives/${name}_${ver}_${arch}.deb"
-			if [ -e "$aptname" ]; then
-				# make sure that we found the right file by checking its hash
-				echo "$hash  $aptname" | sha256sum --check >&2
-				mkdir -p "$1/$dir"
-				# since we move hardlinks around, the same hardlink might've been
-				# moved already into the same place by another distribution.
-				# mv(1) refuses to copy A to B if both are hardlinks of each other.
-				if [ -e "$aptname" ] && [ -e "$1/$fname" ] && [ "$(stat -c "%d %i" "$aptname")" = "$(stat -c "%d %i" "$1/$fname")" ]; then
-					# both files are already the same so we just need to
-					# delete the source
-					rm "$aptname"
-				else
-					mv "$aptname" "$1/$fname"
-				fi
-				echo "$aptname"
-			fi
-		done
-}
-
 cleanupapt() {
 	echo "running cleanupapt" >&2
 	if [ ! -e "$rootdir" ]; then
@@ -175,10 +110,11 @@ cleanupapt() {
 		"$rootdir/var/lib/dpkg/status" \
 		"$rootdir/var/lib/dpkg/lock-frontend" \
 		"$rootdir/var/lib/dpkg/lock" \
+		"$rootdir/var/lib/apt/lists/lock" \
 		"$rootdir/etc/apt/apt.conf" \
+		"$rootdir/etc/apt/sources.list.d/"* \
+		"$rootdir/etc/apt/preferences.d/"* \
 		"$rootdir/etc/apt/sources.list" \
-		"$rootdir/oldaptnames" \
-		"$rootdir/newaptnames" \
 		"$rootdir/var/cache/apt/archives/lock"; do
 		if [ ! -e "$f" ]; then
 			echo "does not exist: $f" >&2
@@ -227,34 +163,49 @@ Apt::Get::Download-Only true;
 Acquire::Languages "none";
 Dir::Etc::Trusted "/etc/apt/trusted.gpg";
 Dir::Etc::TrustedParts "/etc/apt/trusted.gpg.d";
-Acquire::http::Dl-Limit "1000";
+Acquire::http::Proxy "http://127.0.0.1:8080/";
-Acquire::https::Dl-Limit "1000";
-Acquire::Retries "5";
 END
 
 	: > "$rootdir/var/lib/dpkg/status"
 
-	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update
+	if [ "$dist" = "$DEFAULT_DIST" ] && [ "$nativearch" = "$HOSTARCH" ] && [ "$USE_HOST_APT_CONFIG" = "yes" ]; then
+		# we append sources and settings instead of overwriting after
+		# an empty line
+		for f in /etc/apt/sources.list /etc/apt/sources.list.d/*; do
+			[ -e "$f" ] || continue
+			[ -e "$rootdir/$f" ] && echo >> "$rootdir/$f"
+			# Filter out file:// repositories as they are added
+			# to each mmdebstrap call verbatim by
+			# debian/tests/copy_host_apt_config
+			# Also filter out all mirrors that are not of suite
+			# $DEFAULT_DIST, except experimental if the suite
+			# is unstable. This prevents packages from
+			# unstable entering a testing mirror.
+			if [ "$dist" = unstable ]; then
+				grep -v ' file://' "$f" \
+					| grep -E " (unstable|experimental) " \
+					>> "$rootdir/$f" || :
+			else
+				grep -v ' file://' "$f" \
+					| grep " $DEFAULT_DIST " \
+					>> "$rootdir/$f" || :
+			fi
+		done
+		for f in /etc/apt/preferences.d/*; do
+			[ -e "$f" ] || continue
+			[ -e "$rootdir/$f" ] && echo >> "$rootdir/$f"
+			cat "$f" >> "$rootdir/$f"
+		done
+	fi
 
-	# before downloading packages and before replacing the old Packages
+	echo "creating mirror for $dist" >&2
-	# file, copy all old *.deb packages from the mirror to
+	for f in /etc/apt/sources.list /etc/apt/sources.list.d/* /etc/apt/preferences.d/*; do
-	# /var/cache/apt/archives so that apt will not re-download *.deb
+		[ -e "$rootdir/$f" ] || continue
-	# packages that we already have
+		echo "contents of $f:" >&2
-	{
+		cat "$rootdir/$f" >&2
-		get_oldaptnames "$oldmirrordir" "dists/$dist/main/binary-$nativearch/Packages.xz"
+	done
-		case "$dist" in oldstable|stable)
+
-			get_oldaptnames "$oldmirrordir" "dists/$dist-updates/main/binary-$nativearch/Packages.xz"
+	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update
-			;;
-		esac
-		case "$dist" in
-			oldstable)
-				get_oldaptnames "$oldcachedir/debian-security" "dists/$dist/updates/main/binary-$nativearch/Packages.xz"
-				;;
-			stable)
-				get_oldaptnames "$oldcachedir/debian-security" "dists/$dist-security/main/binary-$nativearch/Packages.xz"
-				;;
-		esac
-	} | sort -u > "$rootdir/oldaptnames"
 
 	pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \
 		--format '$(FILENAME)' 'Created-By: Packages' "Architecture: $nativearch" \
@@ -276,73 +227,8 @@ END
 	# shellcheck disable=SC2086
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs
 
-	# to be able to also test gpg verification, we need to create a mirror
-	mkdir -p "$newmirrordir/dists/$dist/main/binary-$nativearch/"
-	curl --location "$mirror/dists/$dist/Release" > "$newmirrordir/dists/$dist/Release"
-	curl --location "$mirror/dists/$dist/Release.gpg" > "$newmirrordir/dists/$dist/Release.gpg"
-	curl --location "$mirror/dists/$dist/main/binary-$nativearch/Packages.xz" > "$newmirrordir/dists/$dist/main/binary-$nativearch/Packages.xz"
-	codename=$(awk '/^Codename: / { print $2; }' < "$newmirrordir/dists/$dist/Release")
-	[ -L "$newmirrordir/dists/$codename" ] || ln -s "$dist" "$newmirrordir/dists/$codename"
-	case "$dist" in oldstable|stable)
-		mkdir -p "$newmirrordir/dists/$dist-updates/main/binary-$nativearch/"
-		curl --location "$mirror/dists/$dist-updates/Release" > "$newmirrordir/dists/$dist-updates/Release"
-		curl --location "$mirror/dists/$dist-updates/Release.gpg" > "$newmirrordir/dists/$dist-updates/Release.gpg"
-		curl --location "$mirror/dists/$dist-updates/main/binary-$nativearch/Packages.xz" > "$newmirrordir/dists/$dist-updates/main/binary-$nativearch/Packages.xz"
-		[ -L "$newmirrordir/dists/$codename-updates" ] || ln -s "$dist-updates" "$newmirrordir/dists/$codename-updates"
-		;;
-	esac
-	case "$dist" in
-		oldstable)
-			mkdir -p "$newcachedir/debian-security/dists/$dist/updates/main/binary-$nativearch/"
-			curl --location "$security_mirror/dists/$dist/updates/Release" > "$newcachedir/debian-security/dists/$dist/updates/Release"
-			curl --location "$security_mirror/dists/$dist/updates/Release.gpg" > "$newcachedir/debian-security/dists/$dist/updates/Release.gpg"
-			curl --location "$security_mirror/dists/$dist/updates/main/binary-$nativearch/Packages.xz" > "$newcachedir/debian-security/dists/$dist/updates/main/binary-$nativearch/Packages.xz"
-			;;
-		stable)
-			mkdir -p "$newcachedir/debian-security/dists/$dist-security/main/binary-$nativearch/"
-			curl --location "$security_mirror/dists/$dist-security/Release" > "$newcachedir/debian-security/dists/$dist-security/Release"
-			curl --location "$security_mirror/dists/$dist-security/Release.gpg" > "$newcachedir/debian-security/dists/$dist-security/Release.gpg"
-			curl --location "$security_mirror/dists/$dist-security/main/binary-$nativearch/Packages.xz" > "$newcachedir/debian-security/dists/$dist-security/main/binary-$nativearch/Packages.xz"
-			[ -L "$newcachedir/debian-security/dists/$codename-security" ] || ln -s "$dist-security" "$newcachedir/debian-security/dists/$codename-security"
-			;;
-	esac
-
-	# the deb files downloaded by apt must be moved to their right locations in the
-	# pool directory
-	#
-	# Instead of parsing the Packages file, we could also attempt to move the deb
-	# files ourselves to the appropriate pool directories. But that approach
-	# requires re-creating the heuristic by which the directory is chosen, requires
-	# stripping the epoch from the filename and will break once mirrors change.
-	# This way, it doesn't matter where the mirror ends up storing the package.
-	{
-		get_newaptnames "$newmirrordir" "dists/$dist/main/binary-$nativearch/Packages.xz";
-		case "$dist" in oldstable|stable)
-			get_newaptnames "$newmirrordir" "dists/$dist-updates/main/binary-$nativearch/Packages.xz"
-			;;
-		esac
-		case "$dist" in
-			oldstable)
-				get_newaptnames "$newcachedir/debian-security" "dists/$dist/updates/main/binary-$nativearch/Packages.xz"
-				;;
-			stable)
-				get_newaptnames "$newcachedir/debian-security" "dists/$dist-security/main/binary-$nativearch/Packages.xz"
-				;;
-		esac
-	} | sort -u > "$rootdir/newaptnames"
-
 	rm "$rootdir/var/cache/apt/archives/lock"
 	rmdir "$rootdir/var/cache/apt/archives/partial"
-	# remove all packages that were in the old Packages file but not in the
-	# new one anymore
-	comm -23 "$rootdir/oldaptnames" "$rootdir/newaptnames" | xargs --delimiter="\n" --no-run-if-empty rm
-	# now the apt cache should be empty
-	if [ -n "$(ls -1qA "$rootdir/var/cache/apt/archives/")" ]; then
-		echo "$rootdir/var/cache/apt/archives not empty:"
-		ls -la "$rootdir/var/cache/apt/archives/"
-		exit 1
-	fi
-
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --option Dir::Etc::SourceList=/dev/null update
 	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get clean
 
@@ -397,13 +283,16 @@ security_mirror="http://security.debian.org/debian-security"
 components=main
 
 : "${DEFAULT_DIST:=unstable}"
+: "${ONLY_DEFAULT_DIST:=no}"
+: "${ONLY_HOSTARCH:=no}"
 : "${HAVE_QEMU:=yes}"
 : "${RUN_MA_SAME_TESTS:=yes}"
 # by default, use the mmdebstrap executable in the current directory
 : "${CMD:=./mmdebstrap}"
+: "${USE_HOST_APT_CONFIG:=no}"
 
-if [ -e "$oldmirrordir/dists/$DEFAULT_DIST/Release" ]; then
+if [ -e "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
-	http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/Release" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/Release")
+	http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/InRelease")
 	case "$http_code" in
 		200) ;; # need update
 		304) echo up-to-date; exit 0;;
@@ -411,6 +300,19 @@ if [ -e "$oldmirrordir/dists/$DEFAULT_DIST/Release" ]; then
 	esac
 fi
 
+./caching_proxy.py "$oldcachedir" "$newcachedir" &
+PROXYPID=$!
+
+for i in $(seq 10); do
+	curl --proxy "http://127.0.0.1:8080/" --silent -o /dev/null "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" && break
+	sleep 1
+done
+if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
+	echo "failed to start proxy" >&2
+	kill $PROXYPID
+	exit 1
+fi
+
 trap "cleanup_newcachedir" EXIT INT TERM
 
 mkdir -p "$newcachedir"
@@ -424,12 +326,23 @@ elif [ "$HOSTARCH" = arm64 ]; then
 	arches="$arches amd64 armhf"
 fi
 
-for nativearch in $arches; do
+# we need the split_inline_sig() function
-	for dist in oldstable stable testing unstable; do
+# shellcheck disable=SC1091
+. /usr/share/debootstrap/functions
+
+for dist in oldstable stable testing unstable; do
+	for nativearch in $arches; do
 		# non-host architectures are only downloaded for $DEFAULT_DIST
 		if [ "$nativearch" != "$HOSTARCH" ] && [ "$DEFAULT_DIST" != "$dist" ]; then
 			continue
 		fi
+		# if ONLY_DEFAULT_DIST is set, only download DEFAULT_DIST
+		if [ "$ONLY_DEFAULT_DIST" = "yes" ] && [ "$DEFAULT_DIST" != "$dist" ]; then
+			continue
+		fi
+		if [ "$ONLY_HOSTARCH" = "yes" ] && [ "$nativearch" != "$HOSTARCH" ]; then
+			continue
+		fi
 		# we need a first pass without updates and security patches
 		# because otherwise, old package versions needed by
 		# debootstrap will not get included
@@ -453,8 +366,20 @@ END
 				;;
 		esac
 	done
+	codename=$(awk '/^Codename: / { print $2; }' < "$newmirrordir/dists/$dist/InRelease")
+	ln -s "$dist" "$newmirrordir/dists/$codename"
+
+	# split the InRelease file into Release and Release.gpg not because apt
+	# or debootstrap need it that way but because grep-dctrl does
+	split_inline_sig \
+		"$newmirrordir/dists/$dist/InRelease" \
+		"$newmirrordir/dists/$dist/Release" \
+		"$newmirrordir/dists/$dist/Release.gpg"
+	touch --reference="$newmirrordir/dists/$dist/InRelease" "$newmirrordir/dists/$dist/Release" "$newmirrordir/dists/$dist/Release.gpg"
 done
 
+kill $PROXYPID
+
 # Create some symlinks so that we can trick apt into accepting multiple apt
 # lines that point to the same repository but look different. This is to
 # avoid the warning:
@@ -500,6 +425,25 @@ if [ "$HAVE_QEMU" = "yes" ]; then
 			;;
 	esac
 
+	# we use the caching proxy again when building the qemu image
+	#  - we can re-use the packages that were already downloaded earlier
+	#  - we make sure that the qemu image uses the same Release file even
+	#    if a mirror push happened between now and earlier
+	#  - we avoid polluting the mirror with the additional packages by
+	#    using --readonly
+	./caching_proxy.py --readonly "$oldcachedir" "$newcachedir" &
+	PROXYPID=$!
+
+	for i in $(seq 10); do
+		curl --proxy "http://127.0.0.1:8080/" --silent -o /dev/null "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" && break
+		sleep 1
+	done
+	if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
+		echo "failed to start proxy" >&2
+		kill $PROXYPID
+		exit 1
+	fi
+
 	# We must not use any --dpkgopt here because any dpkg options still
 	# leak into the chroot with chrootless mode.
 	# We do not use our own package cache here because
@@ -541,11 +485,12 @@ if [ "$HAVE_QEMU" = "yes" ]; then
 		arches=$HOSTARCH
 	fi
 	$CMD --variant=apt --architectures="$arches" --include="$pkgs" \
-		--aptopt='Acquire::http::Dl-Limit "1000"' \
+		--setup-hook='echo "Acquire::http::Proxy \"http://127.0.0.1:8080/\";" > "$1/etc/apt/apt.conf.d/00proxy"' \
-		--aptopt='Acquire::https::Dl-Limit "1000"' \
+		--customize-hook='rm "$1/etc/apt/apt.conf.d/00proxy"' \
-		--aptopt='Acquire::Retries "5"' \
 		"$DEFAULT_DIST" - "$mirror" > "$tmpdir/debian-chroot.tar"
 
+	kill $PROXYPID
+
 	cat << END > "$tmpdir/mmdebstrap.service"
 [Unit]
 Description=mmdebstrap worker script
@@ -673,37 +618,6 @@ END
 	trap "cleanup_newcachedir" EXIT INT TERM
 fi
 
-mirror="http://127.0.0.1/debian"
-for dist in oldstable stable testing unstable; do
-	for variant in minbase buildd -; do
-		echo "running debootstrap --variant=$variant $dist \${TEMPDIR} $mirror"
-		cat << END > shared/test.sh
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-export SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
-echo "SOURCE_DATE_EPOCH=\$SOURCE_DATE_EPOCH"
-tmpdir="\$(mktemp -d)"
-chmod 755 "\$tmpdir"
-case "$dist" in
-	oldstable|stable)
-		debootstrap --no-merged-usr --variant=$variant $dist "\$tmpdir" $mirror
-		;;
-	*)
-		debootstrap --merged-usr --variant=$variant $dist "\$tmpdir" $mirror
-		;;
-esac
-tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system --xattrs -C "\$tmpdir" -c . > "$newcache/debian-$dist-$variant.tar"
-rm -r "\$tmpdir"
-END
-		if [ "$HAVE_QEMU" = "yes" ]; then
-			cachedir=$newcachedir ./run_qemu.sh
-		else
-			./run_null.sh SUDO
-		fi
-	done
-done
-
 if [ "$HAVE_QEMU" = "yes" ]; then
 	# now replace the minihttpd config with one that serves the new repository
 	guestfish -a "$newcachedir/debian-$DEFAULT_DIST.qcow" -i <<EOF

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '1.3.1';
+our $VERSION = '1.3.3';
 
 use English;
 use Getopt::Long;
@@ -304,7 +304,8 @@ sub shellescape {
 }
 
 sub test_unshare_userns {
-    my $verbose = shift;
+    my $verbose      = shift;
+    my $unshare_fail = shift;
     if ($EFFECTIVE_USER_ID == 0) {
         my $msg = "cannot unshare user namespace when executing as root";
         if ($verbose) {
@@ -345,7 +346,11 @@ sub test_unshare_userns {
         if (($? >> 8) == 127) {
             my $msg = "cannot find newuidmap";
             if ($verbose) {
-                warning $msg;
+                if ($unshare_fail) {
+                    error $msg;
+                } else {
+                    warning $msg;
+                }
             } else {
                 debug $msg;
             }
@@ -1336,11 +1341,11 @@ sub setup_mounts {
                   . " /sys directory is missing in the target");
         } elsif ((any { $_ eq $options->{mode} } ('root', 'unshare'))
             && !-e "/sys") {
-            warning("skipping bind-mounting /sys because"
+            warning("skipping mounting /sys because"
                   . " /sys does not exist on the outside");
         } elsif ((any { $_ eq $options->{mode} } ('root', 'unshare'))
             && !-d "/sys") {
-            warning("skipping bind-mounting /sys because"
+            warning("skipping mounting /sys because"
                   . " /sys on the outside is not a directory");
         } elsif ($options->{mode} eq 'root') {
             # we don't know whether we run in root mode inside an unshared
@@ -1411,18 +1416,19 @@ sub setup_mounts {
                   . " /proc directory is missing in the target");
         } elsif ((any { $_ eq $options->{mode} } ('root', 'unshare'))
             && !-e "/proc") {
-            warning("skipping bind-mounting /proc because"
+            warning("skipping mounting /proc because"
                   . " /proc does not exist on the outside");
         } elsif ((any { $_ eq $options->{mode} } ('root', 'unshare'))
             && !-d "/proc") {
-            warning("skipping bind-mounting /proc because"
+            warning("skipping mounting /proc because"
                   . " /proc on the outside is not a directory");
-        } elsif ($options->{mode} eq 'root') {
+        } elsif (any { $_ eq $options->{mode} } ('root', 'unshare')) {
             # we don't know whether we run in root mode inside an unshared
             # user namespace or as real root so we first try the real mount and
             # then fall back to mounting in a way that works in unshared
             if (
-                0 == system(
+                $options->{mode} eq 'root'
+                && 0 == system(
                     'mount', '-t', 'proc', '-o', 'ro', 'proc',
                     "$options->{root}/proc"
                 )
@@ -1451,22 +1457,35 @@ sub setup_mounts {
                     0 == system('umount', '--no-mtab', "$options->{root}/proc")
                       or warning("umount /proc failed: $?");
                 };
+            } elsif (
+                # if mounting proc failed, try bind-mounting it read-only as a
+                # last resort
+                0 == system(
+                    'mount', '-o',
+                    'rbind', '/proc',
+                    "$options->{root}/proc"
+                )
+            ) {
+                warning("since mounting /proc normally failed, /proc is now "
+                      . "bind-mounted instead");
+                # to make sure that changes (like unmounting) to the
+                # bind-mounted /proc do not affect the outside /proc, change
+                # all the bind-mounts under /proc to be a slave mount.
+                if (
+                    0 != system('mount', '--make-rslave',
+                        "$options->{root}/proc")) {
+                    warning("mount --make-rslave /proc failed");
+                }
+                push @cleanup_tasks, sub {
+                    # since we cannot write to /etc/mtab we need --no-mtab
+                    0 == system(
+                        'umount', '--no-mtab',
+                        '--lazy', "$options->{root}/proc"
+                    ) or warning("umount /proc failed: $?");
+                };
             } else {
                 error "mount /proc failed: $?";
             }
-        } elsif ($options->{mode} eq 'unshare') {
-            # naturally we have to clean up after ourselves in sudo mode where
-            # we do a real mount. But we also need to unmount in unshare mode
-            # because otherwise, even with the --one-file-system tar option,
-            # the permissions of the mount source will be stored and not the
-            # mount target (the directory)
-            push @cleanup_tasks, sub {
-                # since we cannot write to /etc/mtab we need --no-mtab
-                0 == system('umount', '--no-mtab', "$options->{root}/proc")
-                  or warning("umount /proc failed: $?");
-            };
-            0 == system('mount', '-t', 'proc', 'proc', "$options->{root}/proc")
-              or error "mount /proc failed: $?";
         } elsif (any { $_ eq $options->{mode} } ('fakechroot', 'chrootless')) {
             # we cannot mount in fakechroot mode
         } else {
@@ -4711,7 +4730,7 @@ sub main() {
         );
         close $fh;
         if (    $? == 0
-            and $content =~ /^apt (\d+\.\d+\.\d+)\w* \(\S+\)$/am) {
+            and $content =~ /^apt (\d+\.\d+\.\d+)\S* \(\S+\)$/am) {
             $aptversion = version->new($1);
         }
         if ($aptversion < "2.3.14") {
@@ -4797,7 +4816,7 @@ sub main() {
         }
         # ...or we are not root and then we need to be able to unshare the user
         # namespace.
-        if ($EFFECTIVE_USER_ID != 0 && !test_unshare_userns(1)) {
+        if ($EFFECTIVE_USER_ID != 0 && !test_unshare_userns(1, 1)) {
             my $procfile = '/proc/sys/kernel/unprivileged_userns_clone';
             open(my $fh, '<', $procfile)
               or error "failed to open $procfile: $!";
@@ -6014,12 +6033,16 @@ sub main() {
     close $nblkwriter;
     if (!$options->{dryrun} && $format eq 'ext2') {
         $numblocks = <$nblkreader>;
-        if (!defined $numblocks) {
+        if (defined $numblocks) {
+            chomp $numblocks;
+        } else {
             # This can happen if the setup process died early and thus closes
             # the pipe from the other and. The EOF is turned into undef.
-            error "failed to read required number of blocks";
+            # we cannot die here because that would skip the cleanup task
+            warning "failed to read required number of blocks";
+            $exitstatus = 1;
+            $numblocks  = -1;
         }
-        chomp $numblocks;
     }
     close $nblkreader;
 
@@ -6348,13 +6371,26 @@ Example: Minimizing the number of packages installed from experimental
 
 =item B<--keyring>=I<file>|I<directory>
 
-Change the default keyring to use by apt. By default, F</etc/apt/trusted.gpg>
+Change the default keyring to use by apt during the initial setup. This is
-and F</etc/apt/trusted.gpg.d> are used. Depending on whether a file or
+similar to setting B<Dir::Etc::Trusted> and B<Dir::Etc::TrustedParts> using
-directory is passed to this option, the former and latter default can be
+B<--aptopt> except that the latter setting will be permanently stored in the
-changed, respectively.  Since apt only supports a single keyring file and
+chroot while the keyrings passed via <--keyring> will only be visible to apt as
-directory, respectively, you can B<not> use this option to pass multiple files
+run by B<mmdebstrap>. Do not use B<--keyring> if apt inside the chroot needs to
-and/or directories. Using the C<--keyring> argument in the following way is
+know about your keys after the initial chroot creation by B<mmdebstrap>. This
-equal to keeping the default:
+option is mainly intended for users who use B<mmdebstrap> as a B<deboostrap>
+drop-in replacement. As such, it is probably not what you want to use if you
+use B<mmdebstrap> with more than a single mirror unless you pass it a directory
+containing all the keyrings you need.
+
+By default, the local setting of B<Dir::Etc::Trusted> and
+B<Dir::Etc::TrustedParts> are used to choose the keyring used by apt as run by
+B<mmdebstrap>. These two locations are set to F</etc/apt/trusted.gpg> and
+F</etc/apt/trusted.gpg.d> by default. Depending on whether a file or directory
+is passed to this option, the former and latter default can be changed,
+respectively.  Since apt only supports a single keyring file and directory,
+respectively, you can B<not> use this option to pass multiple files and/or
+directories. Using the C<--keyring> argument in the following way is equal to
+keeping the default:
 
     --keyring=/etc/apt/trusted.gpg --keyring=/etc/apt/trusted.gpg.d
 
@@ -6363,6 +6399,10 @@ specifying the mirror like this:
 
     mmdebstrap mysuite out.tar "deb [signed-by=/path/to/key.gpg] http://..."
 
+Another reason to use C<signed-by> instead of B<--keyring> is if apt inside the
+chroot needs to know by what key the repository is signed even after the
+initial chroot creation.
+
 The C<signed-by> option will automatically be added to the final
 C<sources.list> if the keyring required for the selected I<SUITE> is not yet
 trusted by apt. Automatically adding the C<signed-by> option in these cases
@@ -6374,6 +6414,13 @@ installed, then you can create a Ubuntu Bionic chroot on Debian like this:
 The resulting chroot will have a C<source.list> with a C<signed-by> option
 pointing to F</usr/share/keyrings/ubuntu-archive-keyring.gpg>.
 
+You do not need to use B<--keyring> or C<signed-by> if you placed the keys that
+apt needs to know about into F</etc/apt/trusted.gpg.d> in the B<--setup-hook>
+(which is before C<apt update> runs), for example by using the <copy-in>
+special hook. You also need to copy your keys into the chroot explicitly if the
+key you passed via C<signed-by> points to a location that is not otherwise
+populated during chroot creation (for example by installing a keyring package).
+
 =item B<--dpkgopt>=I<option>|I<file>
 
 Pass arbitrary I<option>s to dpkg. Will be permanently added to
@@ -6434,16 +6481,16 @@ comma or whitespace.
 
 =item B<--components>=I<comp1>[,I<comp2>,...]
 
-Comma or whitespace separated list of components like main, contrib and
+Comma or whitespace separated list of components like main, contrib, non-free
-non-free which will be used for all URI-only I<MIRROR> arguments. The option
+and non-free-firmware which will be used for all URI-only I<MIRROR> arguments.
-can be specified multiple times and the components are concatenated in the
+The option can be specified multiple times and the components are concatenated
-order in which they are given on the command line. If later list items are
+in the order in which they are given on the command line. If later list items
-repeated, then they get dropped so that the resulting component list is free
+are repeated, then they get dropped so that the resulting component list is
-of duplicates. So the following are equivalent:
+free of duplicates. So the following are equivalent:
 
-    --components="main contrib non-free"
+    --components="main contrib non-free non-free-firmware"
-    --components=main,contrib,non-free
+    --components=main,contrib,non-free,non-free-firmware
-    --comp=main --comp="contrib non-free" --comp="main,non-free"
+    --comp=main --comp="contrib non-free" --comp="main,non-free-firmware"
 
 =item B<--architectures>=I<native>[,I<foreign1>,...]
 
@@ -6635,14 +6682,30 @@ needs to be able to mount and thus requires C<SYS_CAP_ADMIN>.
 
 =item B<unshare>
 
-This mode uses Linux user namespaces to allow unprivileged use of chroot and
+When used as a normal (not root) user, this mode uses Linux user namespaces to
-creation of files that appear to be owned by the superuser inside the unshared
+allow unprivileged use of chroot and creation of files that appear to be owned
-namespace. A tarball created in this mode should be bit-by-bit identical to a
+by the superuser inside the unshared namespace. A tarball created in this mode
-tarball created with the B<root> mode.
+will be bit-by-bit identical to a tarball created with the B<root> mode. With
+this mode, the only binaries that will run as the root user will be
+B<newuidmap(1)> and B<newgidmap(1)> via their setuid bit. Running those
+successfully requires F</etc/subuid> and F</etc/subgid> to have an entry for
+your username. This entry was usually created by B<adduser(8)> already.
+
+The unshared user will not automatically have access to the same files as you
+do. This is intentional and an additional security against unintended changes
+to your files that could theoretically result from running B<mmdebstrap> and
+package maintainer scripts. To copy files in and out of the chroot, either use
+globally readable or writable directories or use special hooks like B<copy-in>
+and B<copy-out>.
+
+Besides the user namespace, the mount, pid (process ids), uts (hostname) and
+ipc namespaces will be unshared as well. See the man pages of B<namespaces(7)>
+and B<unshare(2)> as well as the manual pages they are linking to.
 
 A directory chroot created with this mode will end up with wrong ownership
-information. For correct ownership information, the directory must be accessed
+information (seen from outside the unshared user namespace). For correct
-from a user namespace with the right subuid/subgid offset, like so:
+ownership information, the directory must be accessed from a user namespace
+with the right subuid/subgid offset, like so:
 
     $ lxc-usernsexec -- lxc-unshare -s 'MOUNT|PID|UTSNAME|IPC' -- \
     > /usr/sbin/chroot ./debian-rootfs /bin/bash

tarfilter

@@ -178,14 +178,14 @@ Lastly, shift user id and group id of each entry by the value given by the
         skip = False
         if not hasattr(args, "pathfilter"):
             return False
-        for (t, r) in args.pathfilter:
+        for t, r in args.pathfilter:
             if r.match(member.name[1:]) is not None:
                 if t == "path_include":
                     skip = False
                 else:
                     skip = True
         if skip and (member.isdir() or member.issym()):
-            for (t, r) in args.pathfilter:
+            for t, r in args.pathfilter:
                 if t != "path_include":
                     continue
                 prefix = prefix_prog.sub(r"\1", r.pattern)
@@ -198,7 +198,7 @@ Lastly, shift user id and group id of each entry by the value given by the
         if not hasattr(args, "paxfilter"):
             return False
         skip = False
-        for (t, r) in args.paxfilter:
+        for t, r in args.paxfilter:
             if r.match(header) is None:
                 continue
             if t == "pax_include":

tests/as-debootstrap-unshare-wrapper

@@ -2,18 +2,58 @@
 set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-if [ ! -e /mmdebstrap-testenv ]; then
+
-	echo "this test modifies the system and should only be run inside a container" >&2
+prefix=
-	exit 1
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+	fi
+	prefix="runuser -u ${SUDO_USER:-user} --"
+fi
+
+# debootstrap uses apt-config to figure out whether the system running it has
+# any proxies configured and then runs the binary to set the http_proxy
+# environment variable. This will fail if debootstrap is run in a linux user
+# namespace because auto-apt-proxy will see /tmp/.auto-apt-proxy-0 as being
+# owned by the user "nobody" and group "nogroup" and fail with:
+#   insecure cache dir /tmp/.auto-apt-proxy-0. Must be owned by UID 0 and have permissions 700
+# We cannot overwrite a configuration item using the APT_CONFIG environment
+# variable, so instead we use it to set the Dir configuration option
+# to /dev/null to force all apt settings to their defaults.
+# There is currently no better way to disable this behavior. See also:
+# https://bugs.debian.org/1031105
+# https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/90
+AUTOPROXY=
+eval "$(apt-config shell AUTOPROXY Acquire::http::Proxy-Auto-Detect)"
+if [ -n "$AUTOPROXY" ] && [ -x "$AUTOPROXY" ] && [ -e /tmp/.auto-apt-proxy-0 ]; then
+	TMP_APT_CONFIG=$(mktemp)
+	echo "Dir \"/dev/null\";" > "$TMP_APT_CONFIG"
+	chmod 644 "$TMP_APT_CONFIG"
+fi
+
+# debootstrap runs mount -t proc proc /proc which doesn't work in an unshared
+# namespace on privileged docker (like salsaci), so mount /proc manually
+# https://bugs.debian.org/1031222
+# https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/91
+$prefix {{ CMD }} --variant=custom --mode={{ MODE }} \
+	--setup-hook='env '"${AUTOPROXY:+APT_CONFIG='$TMP_APT_CONFIG'}"' container=lxc debootstrap --variant={{ VARIANT }} unstable "$1" {{ MIRROR }}' \
+	--setup-hook='mount -o rbind /proc "$1/proc"' \
+	--setup-hook='chroot "$1" dpkg-reconfigure systemd || true' \
+	--setup-hook='umount --lazy "$1/proc"' \
+	- /tmp/debian-mm.tar {{ MIRROR }}
+if [ -n "$AUTOPROXY" ] && [ -x "$AUTOPROXY" ] && [ -e /tmp/.auto-apt-proxy-0 ]; then
+	rm "$TMP_APT_CONFIG"
 fi
-useradd --home-dir /home/user --create-home user
-runuser -u user -- {{ CMD }} --variant=custom --mode=unshare --setup-hook='env container=lxc debootstrap unstable "$1" {{ MIRROR }}' - /tmp/debian-mm.tar {{ MIRROR }}
 
 mkdir /tmp/debian-mm
 tar --xattrs --xattrs-include='*' -C /tmp/debian-mm -xf /tmp/debian-mm.tar
 
 mkdir /tmp/debian-debootstrap
-tar --xattrs --xattrs-include='*' -C /tmp/debian-debootstrap -xf "cache/debian-unstable--.tar"
+tar --xattrs --xattrs-include='*' -C /tmp/debian-debootstrap -xf "cache/debian-unstable-{{ VARIANT }}.tar"
 
 # diff cannot compare device nodes, so we use tar to do that for us and then
 # delete the directory
@@ -38,18 +78,23 @@ find /tmp/debian-debootstrap/run/ -mindepth 1 -maxdepth 1 ! -name lock -print0 |
 
 # debootstrap doesn't clean apt
 rm /tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_main_binary-{{ HOSTARCH }}_Packages \
+	/tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_InRelease \
 	/tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_Release \
 	/tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_Release.gpg
 
-rm /tmp/debian-debootstrap/etc/machine-id /tmp/debian-mm/etc/machine-id
+if [ -e /tmp/debian-debootstrap/etc/machine-id ]; then
+	rm /tmp/debian-debootstrap/etc/machine-id /tmp/debian-mm/etc/machine-id
+fi
 rm /tmp/debian-mm/var/cache/apt/archives/lock
 rm /tmp/debian-mm/var/lib/apt/lists/lock
 rm /tmp/debian-mm/var/lib/dpkg/arch
 
-# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
 # also needed for users that are created by systemd-sysusers before systemd 252
 # https://github.com/systemd/systemd/pull/24534
 for f in shadow shadow-; do
+	if [ ! -e /tmp/debian-debootstrap/etc/$f ]; then
+		continue
+	fi
 	if ! cmp /tmp/debian-debootstrap/etc/$f /tmp/debian-mm/etc/$f >&2; then
 		echo patching /etc/$f >&2
 		awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-mm/etc/$f > /tmp/debian-mm/etc/$f.bak

tests/auto-mode-as-normal-user

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+
+trap "rm -f /tmp/debian-chroot.tar.gz" EXIT INT TERM
+
+[ {{ MODE }} = "auto" ]
+
+prefix=
+if [ "$(id -u)" -eq 0 ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+	fi
+	prefix="runuser -u ${SUDO_USER:-user} --"
+fi
+
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar.gz {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar.gz | sort | diff -u tar1.txt -

tests/check-against-debootstrap-dist

@@ -96,6 +96,7 @@ fi
 find /tmp/debian-{{ DIST }}-debootstrap/run/ -mindepth 1 -maxdepth 1 ! -name lock -print0 | xargs --no-run-if-empty -0 rm -r
 # debootstrap doesn't clean apt
 rm /tmp/debian-{{ DIST }}-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_{{ DIST }}_main_binary-{{ HOSTARCH }}_Packages \
+	/tmp/debian-{{ DIST }}-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_{{ DIST }}_InRelease \
 	/tmp/debian-{{ DIST }}-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_{{ DIST }}_Release \
 	/tmp/debian-{{ DIST }}-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_{{ DIST }}_Release.gpg
 
@@ -182,23 +183,6 @@ if [ "{{ VARIANT }}" = "-" ]; then
 	esac
 fi
 
-# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
-case {{ DIST }} in oldstable|stable)
-for f in shadow shadow-; do
-	if [ ! -e /tmp/debian-{{ DIST }}-mm/etc/$f ]; then
-		continue
-	fi
-	if ! cmp /tmp/debian-{{ DIST }}-debootstrap/etc/$f /tmp/debian-{{ DIST }}-mm/etc/$f >&2; then
-		echo patching /etc/$f on {{ DIST }} {{ VARIANT }} >&2
-		awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-{{ DIST }}-mm/etc/$f > /tmp/debian-{{ DIST }}-mm/etc/$f.bak
-		cat /tmp/debian-{{ DIST }}-mm/etc/$f.bak > /tmp/debian-{{ DIST }}-mm/etc/$f
-		rm /tmp/debian-{{ DIST }}-mm/etc/$f.bak
-	else
-		echo no difference for /etc/$f on {{ DIST }} {{ VARIANT }} >&2
-	fi
-done;;
-esac
-
 # check if the file content differs
 diff --unified --no-dereference --recursive /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm >&2
 

tests/check-for-bit-by-bit-identical-format-output

@@ -1,36 +1,21 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
-	echo "this test modifies the system and should only be run inside a container" >&2
-	exit 1
-fi
-useradd --home-dir /home/user --create-home user
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-{{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-root.{{ FORMAT }} {{ MIRROR }}
+
-if [ "{{ FORMAT }}" = tar ]; then
+trap "rm -f /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }}" EXIT INT TERM
-	printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 /tmp/debian-chroot-root.tar -
+
-elif [ "{{ FORMAT }}" = squashfs ]; then
+if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
-	printf 'hsqs' | cmp --bytes=4 /tmp/debian-chroot-root.squashfs -
+	if [ ! -e /mmdebstrap-testenv ]; then
-elif [ "{{ FORMAT }}" = ext2 ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
-	printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 /tmp/debian-chroot-root.ext2 -
+		exit 1
-else
+	fi
-	echo "unknown format: {{ FORMAT }}" >&2
+	useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
-	exit 1
 fi
-runuser -u user -- {{ CMD }} --mode=unshare --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-unshare.{{ FORMAT }} {{ MIRROR }}
+runuser -u "${SUDO_USER:-user}" -- {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} {{ MIRROR }}
-cmp /tmp/debian-chroot-root.{{ FORMAT }} /tmp/debian-chroot-unshare.{{ FORMAT }}
+cmp ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} \
-rm /tmp/debian-chroot-unshare.{{ FORMAT }}
+	|| diffoscope ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }}
-case {{ VARIANT }} in essential|apt|minbase|buildd)
+
-	# variants important and standard differ because permissions drwxr-sr-x
-	# and extended attributes of ./var/log/journal/ cannot be preserved
-	# in fakechroot mode
-	runuser -u user -- {{ CMD }} --mode=fakechroot --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-fakechroot.{{ FORMAT }} {{ MIRROR }}
-	cmp /tmp/debian-chroot-root.{{ FORMAT }} /tmp/debian-chroot-fakechroot.{{ FORMAT }}
-	rm /tmp/debian-chroot-fakechroot.{{ FORMAT }}
-	;;
-esac
 # we cannot test chrootless mode here, because mmdebstrap relies on the
 # usrmerge package to set up merged-/usr and that doesn't work in chrootless
 # mode
-rm /tmp/debian-chroot-root.{{ FORMAT }}

tests/chrootless

@@ -11,6 +11,6 @@ for INCLUDE in '' 'apt' 'apt,build-essential' 'systemd-sysv'; do
 			${INCLUDE:+--include="$INCLUDE"} \
 			{{ DIST }} "/tmp/$MODE.tar" {{ MIRROR }}
 	done
-	cmp /tmp/root.tar /tmp/chrootless.tar
+	cmp /tmp/root.tar /tmp/chrootless.tar || diffoscope /tmp/root.tar /tmp/chrootless.tar
 	rm /tmp/chrootless.tar /tmp/root.tar
 done

tests/chrootless-fakeroot

@@ -3,15 +3,21 @@ set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
-	if [ ! -e /mmdebstrap-testenv ]; then
+[ {{ MODE }} = chrootless ]
-		echo "this test modifies the system and should only be run inside a container" >&2
+
-		exit 1
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 	fi
-	useradd --home-dir /home/user --create-home user
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
+
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
 # we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
 # DPKG_ROOT
 # permissions drwxr-sr-x and extended attributes of ./var/log/journal/ cannot
@@ -21,9 +27,9 @@ for INCLUDE in '' 'apt' 'apt,build-essential' 'systemd-sysv'; do
 		--customize-hook='if [ -d "$1"/var/log/journal ]; then rmdir "$1"/var/log/journal; mkdir --mode=2755 "$1"/var/log/journal; chroot "$1" chown root:systemd-journal /var/log/journal; fi' \
 		${INCLUDE:+--include="$INCLUDE"} \
 		{{ DIST }} /tmp/root.tar {{ MIRROR }}
-	$prefix fakeroot {{ CMD }} --mode=chrootless --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
+	$prefix fakeroot {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
 		${INCLUDE:+--include="$INCLUDE"} \
 		{{ DIST }} /tmp/chrootless.tar {{ MIRROR }}
-	cmp /tmp/root.tar /tmp/chrootless.tar
+	cmp /tmp/root.tar /tmp/chrootless.tar || diffoscope /tmp/root.tar /tmp/chrootless.tar
 	rm /tmp/chrootless.tar /tmp/root.tar
 done

tests/chrootless-foreign

@@ -63,6 +63,6 @@ for INCLUDE in '' 'apt' 'systemd-sysv'; do
 			> "/tmp/$tar.tar.tmp"
 		mv "/tmp/$tar.tar.tmp" "/tmp/$tar.tar"
 	done
-	cmp /tmp/root.tar /tmp/chrootless.tar
+	cmp /tmp/root.tar /tmp/chrootless.tar || diffoscope /tmp/root.tar /tmp/chrootless.tar
 	rm /tmp/chrootless.tar /tmp/root.tar
 done

tests/create-arm64-tarball

@@ -1,15 +1,19 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
-	if [ ! -e /mmdebstrap-testenv ]; then
+prefix=
-		echo "this test modifies the system and should only be run inside a container" >&2
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
-		exit 1
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 	fi
-	useradd --home-dir /home/user --create-home user
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
+
-[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
 [ "{{ MODE }}" = "fakechroot" ] && prefix="$prefix fakechroot fakeroot"
 $prefix {{ CMD }} --mode={{ MODE }} --variant=apt --architectures=arm64 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 # we ignore differences between architectures by ignoring some files

tests/create-directory

@@ -1,7 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
 {{ CMD }} --mode=root --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 chroot /tmp/debian-chroot dpkg-query --showformat '${binary:Package}\n' --show > pkglist.txt
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort > tar1.txt
-rm -r /tmp/debian-chroot

tests/create-gzip-compressed-tarball

@@ -1,12 +1,20 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
+
-	echo "this test modifies the system and should only be run inside a container" >&2
+prefix=
-	exit 1
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+	fi
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-useradd --home-dir /home/user --create-home user
+
-runuser -u user -- {{ CMD }} --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot.tar.gz {{ MIRROR }}
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar.gz {{ MIRROR }}
 printf '\037\213\010' | cmp --bytes=3 /tmp/debian-chroot.tar.gz -
 tar -tf /tmp/debian-chroot.tar.gz | sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar.gz

tests/create-tarball-dry-run

@@ -8,15 +8,14 @@ export LC_ALL=C.UTF-8
 prefix=
 include=,
 if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != root ] && [ "{{ MODE }}" != auto ]; then
-	# this must be qemu
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
-	if ! id -u user >/dev/null 2>&1; then
 		if [ ! -e /mmdebstrap-testenv ]; then
 			echo "this test modifies the system and should only be run inside a container" >&2
 			exit 1
 		fi
-		useradd --home-dir /home/user --create-home user
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 	fi
-	prefix="runuser -u user --"
+	prefix="runuser -u ${SUDO_USER:-user} --"
 	if [ "{{ VARIANT }}" = extract ] || [ "{{ VARIANT }}" = custom ]; then
 		include="$(tr '\n' ',' < pkglist.txt)"
 	fi

tests/custom-tmpdir

@@ -1,25 +1,33 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
+
-	echo "this test modifies the system and should only be run inside a container" >&2
+[ "$(id -u)" -eq 0 ]
-	exit 1
+[ {{ MODE }} = "unshare" ]
+
+if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+	if [ ! -e /mmdebstrap-testenv ]; then
+		echo "this test modifies the system and should only be run inside a container" >&2
+		exit 1
+	fi
+	useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 fi
+prefix="runuser -u ${SUDO_USER:-user} --"
+
 # https://www.etalabs.net/sh_tricks.html
 quote () { printf %s\\n "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" ; }
-useradd --home-dir /home/user --create-home user
+homedir=$($prefix sh -c 'cd && pwd')
-homedir=$(runuser -u user -- sh -c 'cd && pwd')
 # apt:test/integration/test-apt-key
 TMPDIR_ADD="This is fü\$\$ing cràzy, \$(apt -v)\$!"
-runuser -u user -- mkdir "$homedir/$TMPDIR_ADD"
+$prefix mkdir "$homedir/$TMPDIR_ADD"
 # make sure the unshared user can traverse into the TMPDIR
 chmod 711 "$homedir"
 # set permissions and sticky bit like the real /tmp
 chmod 1777 "$homedir/$TMPDIR_ADD"
-runuser -u user -- env TMPDIR="$homedir/$TMPDIR_ADD" {{ CMD }} --mode=unshare --variant=apt \
+$prefix env TMPDIR="$homedir/$TMPDIR_ADD" {{ CMD }} --mode={{ MODE }} --variant=apt \
 	--setup-hook='case "$1" in '"$(quote "$homedir/$TMPDIR_ADD/mmdebstrap.")"'??????????) exit 0;; *) echo "$1"; exit 1;; esac' \
 	 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
 # use rmdir as a quick check that nothing is remaining in TMPDIR
-runuser -u user -- rmdir "$homedir/$TMPDIR_ADD"
+$prefix rmdir "$homedir/$TMPDIR_ADD"
 rm /tmp/debian-chroot.tar

tests/cwd-directory-not-accessible-by-unshared-user

@@ -1,21 +1,30 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
+
-	echo "this test modifies the system and should only be run inside a container" >&2
+[ "$(id -u)" -eq 0 ]
-	exit 1
+[ {{ MODE }} = "unshare" ]
+
+if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+       if [ ! -e /mmdebstrap-testenv ]; then
+               echo "this test modifies the system and should only be run inside a container" >&2
+               exit 1
+       fi
+       useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 fi
-useradd --home-dir /home/user --create-home user
+prefix="runuser -u ${SUDO_USER:-user} --"
+
 mkdir /tmp/debian-chroot
 chmod 700 /tmp/debian-chroot
-chown user:user /tmp/debian-chroot
+chown "${SUDO_USER:-user}:${SUDO_USER:-user}" /tmp/debian-chroot
+set -- env --chdir=/tmp/debian-chroot
 if [ "{{ CMD }}" = "./mmdebstrap" ]; then
-	set -- "$(realpath --canonicalize-existing ./mmdebstrap)"
+	set -- "$@" "$(realpath --canonicalize-existing ./mmdebstrap)"
 elif [ "{{ CMD }}" = "perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap" ]; then
-	set -- perl -MDevel::Cover=-silent,-nogcov "$(realpath --canonicalize-existing ./mmdebstrap)"
+	set -- "$@" perl -MDevel::Cover=-silent,-nogcov "$(realpath --canonicalize-existing ./mmdebstrap)"
 else
-	set -- {{ CMD }}
+	set -- "$@" {{ CMD }}
 fi
-env --chdir=/tmp/debian-chroot runuser -u user -- "$@" --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+$prefix "$@" --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar

tests/debootstrap

@@ -0,0 +1,17 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+
+tmpdir="$(mktemp -d)"
+chmod 755 "$tmpdir"
+case "{{ DIST }}" in
+	oldstable|stable)
+		debootstrap --no-merged-usr --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
+		;;
+	*)
+		debootstrap --merged-usr --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
+		;;
+esac
+tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system --xattrs -C "$tmpdir" -c . > "./cache/debian-{{ DIST }}-{{ VARIANT }}.tar"
+rm -r "$tmpdir"

tests/dev-ptmx

@@ -7,15 +7,17 @@ if [ {{ MODE }} != unshare ] && [ {{ MODE }} != root ]; then
 	exit 1
 fi
 
-if [ ! -e /mmdebstrap-testenv ]; then
-	echo "this test modifies the system and should only be run inside a container" >&2
-	exit 1
-fi
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	useradd --home-dir /home/user --create-home user
-fi
 prefix=
-[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+	fi
+	prefix="runuser -u ${SUDO_USER:-user} --"
+fi
 
 # this mimics what apt does in apt-pkg/deb/dpkgpm.cc/pkgDPkgPM::StartPtyMagic()
 cat > /tmp/test.c << 'END'
@@ -123,12 +125,13 @@ script -qfec "$prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
 	--customize-hook='chroot \"\$1\" runuser -u user -- python3 -c \"import pty; print(pty.openpty())\"' \
 	--customize-hook='chroot \"\$1\" script -c \"echo foobar\"' \
 	--customize-hook='chroot \"\$1\" runuser -u user -- env --chdir=/home/user script -c \"echo foobar\"' \
-	--customize-hook='chroot \"\$1\" apt-get install --yes doc-debian 2>&1 | tee /tmp/log' \
+	--customize-hook='chroot \"\$1\" apt-get install --yes doc-debian 2>&1 | tee \"\$1\"/tmp/log' \
 	--customize-hook=\"copy-in /tmp/test.c /tmp\" \
 	--customize-hook='chroot \"\$1\" gcc /tmp/test.c -o /tmp/test' \
 	--customize-hook='chroot \"\$1\" /tmp/test' \
 	--customize-hook='chroot \"\$1\" runuser -u user -- /tmp/test' \
 	--customize-hook='rm \"\$1\"/tmp/test \"\$1\"/tmp/test.c' \
+	--customize-hook=\"copy-out /tmp/log /tmp\" \
 	{{ DIST }} /dev/null {{ MIRROR }}" /dev/null
 
 fail=0

tests/dist-using-codename

@@ -4,8 +4,10 @@
 
 set -eu
 export LC_ALL=C.UTF-8
-trap "rm -f Release; rm -rf /tmp/debian-chroot" EXIT INT TERM
+trap "rm -f InRelease; rm -rf /tmp/debian-chroot.tar /tmp/expected" EXIT INT TERM
-/usr/lib/apt/apt-helper download-file "{{ MIRROR }}/dists/{{ DIST }}/Release" Release
+/usr/lib/apt/apt-helper download-file "{{ MIRROR }}/dists/{{ DIST }}/InRelease" InRelease
-codename=$(awk '/^Codename: / { print $2; }' Release)
+codename=$(awk '/^Codename: / { print $2; }' InRelease)
-{{ CMD }} --mode={{ MODE }} --variant=apt "$codename" /tmp/debian-chroot {{ MIRROR }}
+{{ CMD }} --mode={{ MODE }} --variant=apt "$codename" /tmp/debian-chroot.tar {{ MIRROR }}
-echo "deb {{ MIRROR }} $codename main" | diff -u - /tmp/debian-chroot/etc/apt/sources.list
+echo "deb {{ MIRROR }} $codename main" > /tmp/expected
+tar --to-stdout --extract --file /tmp/debian-chroot.tar ./etc/apt/sources.list \
+	| diff -u /tmp/expected -

tests/essential-hook

@@ -8,7 +8,7 @@ echo tzdata tzdata/Zones/Europe select Berlin | chroot "$1" debconf-set-selectio
 SCRIPT
 chmod +x /tmp/essential.sh
 {{ CMD }} --mode=root --variant=apt --include=tzdata --essential-hook='echo tzdata tzdata/Areas select Europe | chroot "$1" debconf-set-selections' --essential-hook=/tmp/essential.sh {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-echo Europe/Berlin | cmp /tmp/debian-chroot/etc/timezone
+[ "$(readlink /tmp/debian-chroot/etc/localtime)" = "/usr/share/zoneinfo/Europe/Berlin" ]
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort \
 	| grep -v '^./etc/localtime' \
 	| grep -v '^./etc/timezone' \

tests/fail-installing-to-existing-file

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap "rm -f /tmp/exists" EXIT INT TERM
+
 touch /tmp/exists
 ret=0
 {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/exists {{ MIRROR }} || ret=$?

tests/fail-with-path-with-quotes

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap 'rm -rf /tmp/quoted\"path' EXIT INT TERM
+
 ret=0
 {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/quoted\"path {{ MIRROR }} || ret=$?
 if [ "$ret" = 0 ]; then

tests/include-deb-file

@@ -3,6 +3,8 @@
 set -eu
 export LC_ALL=C.UTF-8
 
+trap "rm -rf /tmp/dummypkg.deb /tmp/dummypkg" EXIT INT TERM
+
 # instead of obtaining a .deb from our cache, we create a new package because
 # otherwise apt might decide to download the package with the same name and
 # version from the cache instead of using the local .deb

tests/install-busybox-based-sub-essential-system

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
 pkgs=base-files,base-passwd,busybox,debianutils,dpkg,libc-bin,mawk,tar
 # busybox --install -s will install symbolic links into the rootfs, leaving
 # existing files untouched. It has to run after extraction (otherwise there is
@@ -31,4 +34,3 @@ chroot /tmp/debian-chroot echo foobar \
 	| chroot /tmp/debian-chroot tee /dev/null \
 	| chroot /tmp/debian-chroot sed 's/foobar/blubber/' \
 	| chroot /tmp/debian-chroot grep blubber >/dev/null
-rm -r /tmp/debian-chroot

tests/install-doc-debian

@@ -1,16 +1,25 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
-	if [ ! -e /mmdebstrap-testenv ]; then
+[ {{ VARIANT }} = "custom" ]
-		echo "this test modifies the system and should only be run inside a container" >&2
+[ {{ MODE }} = "chrootless" ]
-		exit 1
+
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 	fi
-	useradd --home-dir /home/user --create-home user
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
+
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --include=doc-debian {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-$prefix {{ CMD }} --mode=chrootless --variant=custom --include=doc-debian {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 tar -C /tmp/debian-chroot --owner=0 --group=0 --numeric-owner --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/debian-chroot.tar .
 tar tvf /tmp/debian-chroot.tar > doc-debian.tar.list
 rm /tmp/debian-chroot.tar

tests/install-doc-debian-and-output-tarball

@@ -2,15 +2,22 @@
 set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
-	if [ ! -e /mmdebstrap-testenv ]; then
+[ {{ VARIANT }} = "custom" ]
-		echo "this test modifies the system and should only be run inside a container" >&2
+[ {{ MODE }} = "chrootless" ]
-		exit 1
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 	fi
-	useradd --home-dir /home/user --create-home user
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
+
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --include=doc-debian {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-$prefix {{ CMD }} --mode=chrootless --variant=custom --include=doc-debian {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar tvf /tmp/debian-chroot.tar | grep -v ' ./dev' | diff -u doc-debian.tar.list -
 rm /tmp/debian-chroot.tar

tests/install-doc-debian-and-test-hooks

@@ -2,16 +2,25 @@
 set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
-	if [ ! -e /mmdebstrap-testenv ]; then
+[ {{ VARIANT }} = "custom" ]
-		echo "this test modifies the system and should only be run inside a container" >&2
+[ {{ MODE }} = "chrootless" ]
-		exit 1
+
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 	fi
-	useradd --home-dir /home/user --create-home user
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
+
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+$prefix {{ CMD }} --mode={{ MODE }} --skip=cleanup/tmp --variant={{ VARIANT }} --include=doc-debian --setup-hook='touch "$1/tmp/setup"' --customize-hook='touch "$1/tmp/customize"' {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-$prefix {{ CMD }} --mode=chrootless --skip=cleanup/tmp --variant=custom --include=doc-debian --setup-hook='touch "$1/tmp/setup"' --customize-hook='touch "$1/tmp/customize"' {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 rm /tmp/debian-chroot/tmp/setup
 rm /tmp/debian-chroot/tmp/customize
 tar -C /tmp/debian-chroot --owner=0 --group=0 --numeric-owner --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/debian-chroot.tar .

tests/install-libmagic-mgc-on-arm64

@@ -1,16 +1,23 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
-	if [ ! -e /mmdebstrap-testenv ]; then
+[ {{ VARIANT }} = "custom" ]
-		echo "this test modifies the system and should only be run inside a container" >&2
+[ {{ MODE }} = "chrootless" ]
-		exit 1
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 	fi
-	useradd --home-dir /home/user --create-home user
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
+
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --architectures=arm64 --include=libmagic-mgc {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-$prefix {{ CMD }} --mode=chrootless --variant=custom --architectures=arm64 --include=libmagic-mgc {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 # delete contents of libmagic-mgc
 rm /tmp/debian-chroot/usr/lib/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/README.Debian

tests/jessie-or-older

@@ -1,15 +1,30 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
-	echo "this test modifies the system and should only be run inside a container" >&2
-	exit 1
-fi
-useradd --home-dir /home/user --create-home user
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
+trap "rm -f /tmp/debian-chroot-{{ MODE }}.tar /tmp/debian-chroot-root-normal.tar" EXIT INT TERM
+
+[ "$(id -u)" -eq 0 ]
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+	fi
+	prefix="runuser -u ${SUDO_USER:-user} --"
+fi
+
+MMTARFILTER=
+[ -x /usr/bin/mmtarfilter ] && MMTARFILTER=/usr/bin/mmtarfilter
+[ -x ./tarfilter ] && MMTARFILTER=./tarfilter
+
 filter() {
-	./tarfilter \
+	"$MMTARFILTER" \
 		--path-exclude=/usr/bin/uncompress \
 		--path-exclude=/var/cache/debconf/config.dat-old \
 		--path-exclude=/var/cache/debconf/templates.dat-old \
@@ -20,19 +35,7 @@ filter() {
 }
 
 # base for comparison without jessie-or-older hook
-{{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-root-normal.tar
+{{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} - {{ MIRROR }} > /tmp/debian-chroot-root-normal.tar
-
-# root
-{{ CMD }} --mode=root --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-root.tar
-cmp /tmp/debian-chroot-root-normal.tar /tmp/debian-chroot-root.tar
-rm /tmp/debian-chroot-root.tar
-# unshare
-runuser -u user -- {{ CMD }} --mode=unshare --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-unshare.tar
-cmp /tmp/debian-chroot-root-normal.tar /tmp/debian-chroot-unshare.tar
-rm /tmp/debian-chroot-unshare.tar
-# fakechroot
-runuser -u user -- {{ CMD }} --mode=fakechroot --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-fakechroot.tar
-cmp /tmp/debian-chroot-root-normal.tar /tmp/debian-chroot-fakechroot.tar
-rm /tmp/debian-chroot-fakechroot.tar
 
-rm /tmp/debian-chroot-root-normal.tar
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --hook-dir=./hooks/jessie-or-older {{ DIST }} - {{ MIRROR }} | filter > /tmp/debian-chroot-{{ MODE }}.tar
+filter < /tmp/debian-chroot-root-normal.tar | cmp - /tmp/debian-chroot-{{ MODE }}.tar

tests/logfile

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
+
+trap "rm -rf /tmp/debian-chroot /tmp/log /tmp/trimmed" EXIT INT TERM
+
 # we check the full log to also prevent debug printfs to accidentally make it into a commit
 {{ CMD }} --mode=root --variant=apt --logfile=/tmp/log {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 # omit the last line which should contain the runtime
@@ -17,5 +20,3 @@ I: cleaning package lists and apt cache...
 LOG
 tail --lines=1 /tmp/log | grep '^I: success in .* seconds$'
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
-rm -r /tmp/debian-chroot
-rm /tmp/log /tmp/trimmed

tests/merged-fakechroot-inside-unmerged-chroot

@@ -46,4 +46,4 @@ chmod +x script.sh
 	--customize-hook=./script.sh \
 	--customize-hook="copy-out /tmp/chroot-fakechroot.tar /tmp" \
 	{{ DIST }} /dev/null {{ MIRROR }}
-cmp /tmp/chroot-fakechroot.tar /tmp/chroot-root.tar
+cmp /tmp/chroot-fakechroot.tar /tmp/chroot-root.tar || diffoscope /tmp/chroot-fakechroot.tar /tmp/chroot-root.tar

tests/missing-dev-sys-proc-inside-the-chroot

@@ -1,9 +1,20 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
+
-	echo "this test modifies the system and should only be run inside a container" >&2
+[ {{ MODE }} = "unshare" ]
-	exit 1
+[ {{ VARIANT }} = "custom" ]
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+	fi
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-useradd --home-dir /home/user --create-home user
+
-runuser -u user -- {{ CMD }} --mode=unshare --variant=custom --include=dpkg,dash,diffutils,coreutils,libc-bin,sed {{ DIST }} /dev/null {{ MIRROR }}
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --include=dpkg,dash,diffutils,coreutils,libc-bin,sed {{ DIST }} /dev/null {{ MIRROR }}

tests/mmdebstrap

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+
+[ "$(id -u)" -eq 0 ]
+[ {{ MODE }} = "root" ]
+case {{ FORMAT }} in tar|squashfs|ext2) : ;; *) exit 1;; esac
+
+{{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} {{ MIRROR }}
+if [ "{{ FORMAT }}" = tar ]; then
+	printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar -
+elif [ "{{ FORMAT }}" = squashfs ]; then
+	printf 'hsqs' | cmp --bytes=4 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.squashfs -
+elif [ "{{ FORMAT }}" = ext2 ]; then
+	printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext2 -
+else
+	echo "unknown format: {{ FORMAT }}" >&2
+	exit 1
+fi

tests/no-sbin-in-path

@@ -7,16 +7,22 @@
 
 set -eu
 export LC_ALL=C.UTF-8
-[ "{{ MODE }}" = "fakechroot" ]
+
 trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
-	if [ ! -e /mmdebstrap-testenv ]; then
+[ "{{ MODE }}" = "fakechroot" ]
-		echo "this test modifies the system and should only be run inside a container" >&2
+
-		exit 1
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 	fi
-	useradd --home-dir /home/user --create-home user
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
+
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
+$prefix env PATH=/usr/bin:/bin fakechroot fakeroot {{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-$prefix env PATH=/usr/bin:/bin fakechroot fakeroot {{ CMD }} --mode=fakechroot --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/pivot_root

@@ -4,21 +4,17 @@ export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 trap "rm -f /tmp/chroot1.tar /tmp/chroot2.tar /tmp/chroot3.tar /tmp/mmdebstrap" EXIT INT TERM
 
-if [ ! -e /mmdebstrap-testenv ]; then
-	echo "this test modifies the system and should only be run inside a container" >&2
-	exit 1
-fi
-
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	useradd --home-dir /home/user --create-home user
-fi
-
 prefix=
-[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
-
+	if ! id "${SUDO_USER:-user}" 2>/dev/null; then
-MMDEBSTRAP=
+		if [ ! -e /mmdebstrap-testenv ]; then
-[ -e /usr/bin/mmdebstrap ] && MMDEBSTRAP=/usr/bin/mmdebstrap
+			echo "this test modifies the system and should only be run inside a container" >&2
-[ -e ./mmdebstrap ] && MMDEBSTRAP=./mmdebstrap
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+	fi
+	prefix="runuser -u ${SUDO_USER:-user} --"
+fi
 
 $prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
 	--include=mount \
@@ -28,28 +24,31 @@ if [ {{ MODE }} = "unshare" ]; then
 	# calling pivot_root in root mode does not work for mysterious reasons:
 	# pivot_root: failed to change root from `.' to `mnt': Invalid argument
 	$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount \
-		--customize-hook="upload $MMDEBSTRAP /$MMDEBSTRAP" \
+		--customize-hook='mkdir -p "$1/mnt" "$1/oldroot"' \
-		--customize-hook='chmod +x "$1"/'"$MMDEBSTRAP" \
+		--customize-hook='[ ! -e /usr/bin/mmdebstrap ] || cp -aT /usr/bin/mmdebstrap "$1/usr/bin/mmdebstrap"' \
-		--customize-hook='mount -o rbind "$1" /mnt && cd /mnt && /sbin/pivot_root . mnt' \
+		--customize-hook='[ ! -e ./mmdebstrap ] || cp -aT ./mmdebstrap "$1/mnt/mmdebstrap"' \
+		--customize-hook='mount -o rbind "$1" /mnt && cd /mnt && /sbin/pivot_root . oldroot' \
 		--customize-hook='unshare -U echo nested unprivileged unshare' \
-		--customize-hook="/$MMDEBSTRAP"' --mode=unshare --variant=apt --include=mount {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
+		--customize-hook='env --chdir=/mnt {{ CMD }} --mode=unshare --variant=apt --include=mount {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
 		--customize-hook='copy-out /tmp/chroot3.tar /tmp' \
-		--customize-hook='rm "$1/'"$MMDEBSTRAP"'"' \
+		--customize-hook='rm -f "/usr/bin/mmdebstrap" "/mnt/mmdebstrap"' \
-		--customize-hook='umount -l mnt sys' \
+		--customize-hook='umount -l oldroot sys' \
+		--customize-hook='rmdir /oldroot' \
 		{{ DIST }} /tmp/chroot2.tar {{ MIRROR }}
 
-		cmp /tmp/chroot1.tar /tmp/chroot2.tar
+		cmp /tmp/chroot1.tar /tmp/chroot2.tar || diffoscope /tmp/chroot1.tar /tmp/chroot2.tar
-		cmp /tmp/chroot1.tar /tmp/chroot3.tar
+		cmp /tmp/chroot1.tar /tmp/chroot3.tar || diffoscope /tmp/chroot1.tar /tmp/chroot3.tar
 		rm /tmp/chroot2.tar /tmp/chroot3.tar
 fi
 
 $prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=mount \
-	--customize-hook="upload $MMDEBSTRAP /$MMDEBSTRAP" \
+	--customize-hook='mkdir -p "$1/mnt"' \
-	--customize-hook='chmod +x "$1"/'"$MMDEBSTRAP" \
+	--customize-hook='[ ! -e /usr/bin/mmdebstrap ] || cp -aT /usr/bin/mmdebstrap "$1/usr/bin/mmdebstrap"' \
-	--chrooted-customize-hook="/$MMDEBSTRAP"' --mode=unshare --variant=apt --include=mount {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
+	--customize-hook='[ ! -e ./mmdebstrap ] || cp -aT ./mmdebstrap "$1/mnt/mmdebstrap"' \
+	--chrooted-customize-hook='env --chdir=/mnt {{ CMD }} --mode=unshare --variant=apt --include=mount {{ DIST }} /tmp/chroot3.tar {{ MIRROR }}' \
 	--customize-hook='copy-out /tmp/chroot3.tar /tmp' \
-	--customize-hook='rm "$1/'"$MMDEBSTRAP"'"' \
+	--customize-hook='rm -f "$1/usr/bin/mmdebstrap" "$1/mnt/mmdebstrap"' \
 	{{ DIST }} /tmp/chroot2.tar {{ MIRROR }}
 
-cmp /tmp/chroot1.tar /tmp/chroot2.tar
+cmp /tmp/chroot1.tar /tmp/chroot2.tar || diffoscope /tmp/chroot1.tar /tmp/chroot2.tar
-cmp /tmp/chroot1.tar /tmp/chroot3.tar
+cmp /tmp/chroot1.tar /tmp/chroot3.tar || diffoscope /tmp/chroot1.tar /tmp/chroot3.tar

tests/root-mode-inside-chroot

@@ -6,9 +6,12 @@
 set -eu
 export LC_ALL=C.UTF-8
 [ "$(whoami)" = "root" ]
+
+trap "rm -f /tmp/debian-chroot.tar script.sh" EXIT INT TERM
+
 cat << 'SCRIPT' > script.sh
 #!/bin/sh
-set -eu
+set -exu
 rootfs="$1"
 mkdir -p "$rootfs/mnt"
 [ -e /usr/bin/mmdebstrap ] && cp -aT /usr/bin/mmdebstrap "$rootfs/usr/bin/mmdebstrap"
@@ -23,4 +26,3 @@ chmod +x script.sh
 	--customize-hook="download /tmp/debian-chroot.tar /tmp/debian-chroot.tar" \
 	{{ DIST }} /dev/null {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
-rm /tmp/debian-chroot.tar script.sh

tests/root-mode-inside-unshare-chroot

@@ -5,13 +5,22 @@
 
 set -eu
 export LC_ALL=C.UTF-8
-if [ ! -e /mmdebstrap-testenv ]; then
+
-	echo "this test modifies the system and should only be run inside a container" >&2
+[ {{ MODE }} = "unshare" ]
-	exit 1
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+	fi
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-[ "$(whoami)" = "root" ]
+
-useradd --home-dir /home/user --create-home user
+cat << 'SCRIPT' > /tmp/script.sh
-cat << 'SCRIPT' > script.sh
 #!/bin/sh
 set -eu
 rootfs="$1"
@@ -22,10 +31,10 @@ chroot "$rootfs" env --chdir=/mnt \
 	{{ CMD }} --mode=root --variant=apt \
 	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 SCRIPT
-chmod +x script.sh
+chmod +x /tmp/script.sh
-runuser -u user -- {{ CMD }} --mode=unshare --variant=apt --include=perl,mount \
+$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --include=perl,mount \
-	--customize-hook=./script.sh \
+	--customize-hook=/tmp/script.sh \
 	--customize-hook="download /tmp/debian-chroot.tar /tmp/debian-chroot.tar" \
 	{{ DIST }} /dev/null {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
-rm /tmp/debian-chroot.tar script.sh
+rm /tmp/debian-chroot.tar /tmp/script.sh

tests/special-hooks-with-mode-mode

@@ -1,15 +1,19 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
-	if [ ! -e /mmdebstrap-testenv ]; then
+prefix=
-		echo "this test modifies the system and should only be run inside a container" >&2
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
-		exit 1
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 	fi
-	useradd --home-dir /home/user --create-home user
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
+
-[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
 [ "{{ MODE }}" = "fakechroot" ] && prefix="$prefix fakechroot fakeroot"
 symlinktarget=/real
 [ "{{ MODE }}" = "fakechroot" ] && symlinktarget='$1/real'

tests/unpack-doc-debian

@@ -1,17 +1,25 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+
-	if [ ! -e /mmdebstrap-testenv ]; then
+trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
-		echo "this test modifies the system and should only be run inside a container" >&2
+
-		exit 1
+[ {{ VARIANT }} = extract ]
+
+prefix=
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 	fi
-	useradd --home-dir /home/user --create-home user
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-prefix=
+
-[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
 [ "{{ MODE }}" = "fakechroot" ] && prefix="$prefix fakechroot fakeroot"
-$prefix {{ CMD }} --mode={{ MODE }} --variant=extract --include=doc-debian {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} --include=doc-debian {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 # delete contents of doc-debian
 rm /tmp/debian-chroot/usr/share/doc-base/debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian

tests/unshare-include-deb

@@ -5,14 +5,19 @@ export LC_ALL=C.UTF-8
 
 [ "{{ MODE }}" = unshare ]
 
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+trap "rm -rf /tmp/dummypkg.deb /tmp/dummypkg" EXIT INT TERM
-	if [ ! -e /mmdebstrap-testenv ]; then
+
-		echo "this test modifies the system and should only be run inside a container" >&2
+prefix=
-		exit 1
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 	fi
-	useradd --home-dir /home/user --create-home user
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-[ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"
 
 # instead of obtaining a .deb from our cache, we create a new package because
 # otherwise apt might decide to download the package with the same name and
@@ -31,9 +36,8 @@ Description: dummypkg
 END
 dpkg-deb --build "/tmp/dummypkg" "/tmp/dummypkg.deb"
 
-# make the .deb only redable by user which will exclude the unshared user
+# make the .deb only redable by its owner which will exclude the unshared user
 chmod 600 /tmp/dummypkg.deb
-chown user /tmp/dummypkg.deb
 
 ret=0
 $prefix {{ CMD }} --variant=apt --mode={{ MODE }} --include="/tmp/dummypkg.deb" \