coverage.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
-from debian.deb822 import Deb822, Release
+from debian.deb822 import Deb822
-import email.utils
 import os
 import sys
 import shutil
@@ -15,7 +14,6 @@ have_qemu = os.getenv("HAVE_QEMU", "yes") == "yes"
 have_unshare = os.getenv("HAVE_UNSHARE", "yes") == "yes"
 have_binfmt = os.getenv("HAVE_BINFMT", "yes") == "yes"
 run_ma_same_tests = os.getenv("RUN_MA_SAME_TESTS", "yes") == "yes"
-cmd = os.getenv("CMD", "./mmdebstrap")
 
 default_dist = os.getenv("DEFAULT_DIST", "unstable")
 all_dists = ["oldstable", "stable", "testing", "unstable"]
@@ -35,21 +33,11 @@ all_variants = [
 default_format = "auto"
 all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "null"]
 
+only_dists = []
+
 mirror = os.getenv("mirror", "http://127.0.0.1/debian")
 hostarch = subprocess.check_output(["dpkg", "--print-architecture"]).decode().strip()
 
-release_path = f"./shared/cache/debian/dists/{default_dist}/Release"
-if not os.path.exists(release_path):
-    print("path doesn't exist:", release_path, file=sys.stderr)
-    print("run ./make_mirror.sh first", file=sys.stderr)
-    exit(1)
-if os.getenv("SOURCE_DATE_EPOCH") is not None:
-    s_d_e = os.getenv("SOURCE_DATE_EPOCH")
-else:
-    with open(release_path) as f:
-        rel = Release(f)
-    s_d_e = str(email.utils.mktime_tz(email.utils.parsedate_tz(rel["Date"])))
-
 separator = (
     "------------------------------------------------------------------------------"
 )
@@ -87,7 +75,6 @@ def main():
         help="exit after first num failures or errors.",
     )
     parser.add_argument("--mode", metavar="mode", help="only run tests with this mode")
-    parser.add_argument("--dist", metavar="dist", help="only run tests with this dist")
     args = parser.parse_args()
 
     # copy over files from git or as distributed
@@ -118,10 +105,16 @@ def main():
             "/usr/share/mmdebstrap/hooks", "shared/hooks", dirs_exist_ok=True
         )
 
+    onlyrun = None
+    if len(sys.argv) > 1:
+        onlyrun = sys.argv[1]
+
     tests = []
     with open("coverage.txt") as f:
         for test in Deb822.iter_paragraphs(f):
             name = test["Test"]
+            if args.test and name not in args.test:
+                continue
             dists = test.get("Dists", default_dist)
             if dists == "any":
                 dists = all_dists
@@ -151,7 +144,11 @@ def main():
             else:
                 formats = formats.split()
             for dist in dists:
+                if only_dists and dist not in only_dists:
+                    continue
                 for mode in modes:
+                    if args.mode and args.mode != mode:
+                        continue
                     for variant in variants:
                         for fmt in formats:
                             skipreason = skip(
@@ -175,60 +172,27 @@ def main():
                                 tt = "null"
                             tests.append((tt, name, dist, mode, variant, fmt))
 
-    torun = []
-    num_tests = len(tests)
-    if args.test:
-        # check if all given tests are either a valid name or a valid number
-        for test in args.test:
-            if test in [name for (_, name, _, _, _, _) in tests]:
-                continue
-            if not test.isdigit():
-                print(f"cannot find test named {test}", file=sys.stderr)
-                exit(1)
-            if int(test) >= len(tests) or int(test) <= 0 or str(int(test)) != test:
-                print(f"test number {test} doesn't exist", file=sys.stderr)
-                exit(1)
-
-        for i, (_, name, _, _, _, _) in enumerate(tests):
-            # if either the number or test name matches, then we use this test,
-            # otherwise we skip it
-            if name in args.test:
-                torun.append(i)
-            if str(i + 1) in args.test:
-                torun.append(i)
-        num_tests = len(torun)
-
     starttime = time.time()
     skipped = defaultdict(list)
     failed = []
     num_success = 0
-    num_finished = 0
     for i, (test, name, dist, mode, variant, fmt) in enumerate(tests):
-        if torun and i not in torun:
-            continue
         print(separator, file=sys.stderr)
         print("(%d/%d) %s" % (i + 1, len(tests), name), file=sys.stderr)
         print("dist: %s" % dist, file=sys.stderr)
         print("mode: %s" % mode, file=sys.stderr)
         print("variant: %s" % variant, file=sys.stderr)
         print("format: %s" % fmt, file=sys.stderr)
-        if num_finished > 0:
+        if i > 0:
             currenttime = time.time()
             timeleft = timedelta(
-                seconds=int(
+                seconds=int((len(tests) - i) * (currenttime - starttime) / i)
-                    (num_tests - num_finished)
-                    * (currenttime - starttime)
-                    / num_finished
-                )
             )
             print("time left: %s" % timeleft, file=sys.stderr)
-        if failed:
-            print("failed: %d" % len(failed))
-        num_finished += 1
         with open("tests/" + name) as fin, open("shared/test.sh", "w") as fout:
             for line in fin:
-                line = line.replace("{{ CMD }}", cmd)
+                for e in ["CMD", "SOURCE_DATE_EPOCH"]:
-                line = line.replace("{{ SOURCE_DATE_EPOCH }}", s_d_e)
+                    line = line.replace("{{ " + e + " }}", os.getenv(e))
                 line = line.replace("{{ DIST }}", dist)
                 line = line.replace("{{ MIRROR }}", mirror)
                 line = line.replace("{{ MODE }}", mode)
@@ -251,18 +215,11 @@ def main():
                 print(f"skipped because of {reason}", file=sys.stderr)
                 continue
         print(separator, file=sys.stderr)
-        if args.dist and args.dist != dist:
-            print(f"skipping because of --dist={args.dist}", file=sys.stderr)
-            continue
-        if args.mode and args.mode != mode:
-            print(f"skipping because of --mode={args.mode}", file=sys.stderr)
-            continue
         proc = subprocess.Popen(argv)
         try:
             proc.wait()
         except KeyboardInterrupt:
-            proc.terminate()
+            proc.kill()
-            proc.wait()
             break
         print(separator, file=sys.stderr)
         if proc.returncode != 0:
@@ -280,7 +237,7 @@ def main():
         file=sys.stderr,
     )
     if skipped:
-        print("skipped %d:" % sum([len(v) for v in skipped.values()]), file=sys.stderr)
+        print("skipped %d:" % len(skipped), file=sys.stderr)
         for reason, l in skipped.items():
             print(f"skipped because of {reason}:", file=sys.stderr)
             for t in l:

coverage.txt

@@ -271,8 +271,6 @@ Skip-If:
  variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
  variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
 
-Test: create-directory-dry-run
-
 Test: create-tarball-dry-run
 Variants: any
 Modes: any
@@ -318,7 +316,3 @@ Skip-If:
 
 Test: no-sbin-in-path
 Modes: fakechroot
-
-Test: dev-ptmx
-Modes: root unshare
-Needs-QEMU: true

hooks/file-mirror-automount/setup00.sh

@@ -8,8 +8,8 @@ fi
 
 rootdir="$1"
 
-env APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get indextargets --no-release-info --format '$(REPO_URI)' \
+env APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get indextargets --no-release-info \
-	| sed -ne 's/^file:\/\+//p' \
+	| sed -ne 's/^Repo-URI: file:\/\+//p' \
 	| sort -u \
 	| while read path; do
 		mkdir -p "$rootdir/run/mmdebstrap"

make_mirror.sh

@@ -660,7 +660,6 @@ for dist in oldstable stable testing unstable; do
 set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
-echo "SOURCE_DATE_EPOCH=\$SOURCE_DATE_EPOCH"
 tmpdir="\$(mktemp -d)"
 chmod 755 "\$tmpdir"
 debootstrap --no-merged-usr --variant=$variant $dist "\$tmpdir" $mirror

mmdebstrap

@@ -1091,20 +1091,6 @@ sub run_chroot {
                             );
                             next;
                         }
-                        if ($fname eq "ptmx") {
-                            # We must not bind-mount ptmx from the outside or
-                            # otherwise posix_openpt() will fail. Instead
-                            # /dev/ptmx must refer to /dev/pts/ptmx either by
-                            # symlink or by bind-mounting. We choose a symlink.
-                            symlink '/dev/pts/ptmx',
-                              "$options->{root}/dev/ptmx"
-                              or error "cannot create /dev/pts/ptmx symlink";
-                            push @cleanup_tasks, sub {
-                                unlink "$options->{root}/dev/ptmx"
-                                  or error "unlink /dev/ptmx";
-                            };
-                            next;
-                        }
                         if (!-e "/dev/$fname") {
                             warning("skipping creation of ./dev/$fname because"
                                   . " /dev/$fname does not exist"
@@ -1147,13 +1133,13 @@ sub run_chroot {
                               . " /dev directory is missing in the target");
                         next;
                     }
-                    if (!-e "/dev/$fname" && $fname ne "pts/") {
+                    if (!-e "/dev/$fname") {
                         warning("skipping creation of ./dev/$fname because"
                               . " /dev/$fname does not exist"
                               . " on the outside");
                         next;
                     }
-                    if (!-d "/dev/$fname" && $fname ne "pts/") {
+                    if (!-d "/dev/$fname") {
                         warning("skipping creation of ./dev/$fname because"
                               . " /dev/$fname on the outside is not a"
                               . " directory");
@@ -1201,32 +1187,9 @@ sub run_chroot {
                             "$options->{root}/dev/$fname")
                           or warning("umount ./dev/$fname failed: $?");
                     };
-                    if ($fname eq "pts/") {
-                        # We cannot just bind-mount /dev/pts from the host as
-                        # doing so will make posix_openpt() fail. Instead, we
-                        # need to mount a new devpts.
-                        # We need ptmxmode=666 because /dev/ptmx is a symlink
-                        # to /dev/pts/ptmx and without it posix_openpt() will
-                        # fail if we are not the root user.
-                        # See also:
-                        # kernel.org/doc/Documentation/filesystems/devpts.txt
-                        # salsa.debian.org/debian/schroot/-/merge_requests/2
-                        # https://bugs.debian.org/856877
-                        # https://bugs.debian.org/817236
-                        0 == system(
-                            'mount',
-                            '-t',
-                            'devpts',
-                            'none',
-                            "$options->{root}/dev/pts",
-                            '-o',
-                            'noexec,nosuid,uid=5,mode=620,ptmxmode=666'
-                        ) or error "mount /dev/pts failed";
-                    } else {
                     0 == system('mount', '-o', 'bind', "/dev/$fname",
                         "$options->{root}/dev/$fname")
                       or error "mount ./dev/$fname failed: $?";
-                    }
                 } else {
                     error "unsupported type: $type";
                 }
@@ -4933,8 +4896,8 @@ sub main() {
                 }
                 # initialize gpg trustdb with empty one
                 {
-                    0 == system(@gpgcmd, '--update-trustdb')
+                    `@gpgcmd --update-trustdb >/dev/null 2>/dev/null`;
-                      or error "gpg failed to initialize trustdb:: $?";
+                    $? == 0 or error "gpg failed to initialize trustdb: $?";
                 }
                 # find all the fingerprints of the keys apt currently
                 # knows about

run_qemu.sh

@@ -31,7 +31,7 @@ qemu-img create -f qcow2 -b "$(realpath $cachedir)/debian-$DEFAULT_DIST.qcow" -F
 # to connect to serial use:
 #   minicom -D 'unix#/tmp/ttyS0'
 ret=0
-timeout --foreground 20m qemu-system-x86_64 \
+timeout 20m qemu-system-x86_64 \
 	-cpu host \
 	-no-user-config \
 	-M accel=kvm:tcg -m 4G -nographic \

tests/arm64-without-qemu-support

@@ -5,7 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-echo 0 > /proc/sys/fs/binfmt_misc/qemu-aarch64
+apt-get remove --yes qemu-user-static binfmt-support qemu-user
 ret=0
 {{ CMD }} --mode={{ MODE }} --variant=apt --architectures=arm64 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }} || ret=$?
 if [ "$ret" = 0 ]; then

tests/check-against-debootstrap-dist

@@ -3,8 +3,6 @@ set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
-echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
-
 # we create the apt user ourselves or otherwise its uid/gid will differ
 # compared to the one chosen in debootstrap because of different installation
 # order in comparison to the systemd users

tests/create-directory-dry-run

@@ -1,11 +1,8 @@
+cat << END > shared/test.sh
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-{{ CMD }} --mode={{ MODE }} --dry-run --variant=apt \
+{{ CMD }} --mode={{ MODE }} --dry-run --variant=apt --setup-hook="exit 1" --essential-hook="exit 1" --customize-hook="exit 1" {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
-	--setup-hook="exit 1" \
-	--essential-hook="exit 1" \
-	--customize-hook="exit 1" \
-	{{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 rm /tmp/debian-chroot/dev/console
 rm /tmp/debian-chroot/dev/fd
 rm /tmp/debian-chroot/dev/full
@@ -26,3 +23,21 @@ rm /tmp/debian-chroot/var/lib/apt/lists/lock
 rm /tmp/debian-chroot/var/lib/dpkg/status
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir
+END
+if [ "$HAVE_QEMU" = "yes" ]; then
+	./run_qemu.sh
+	runtests=$((runtests+1))
+elif [ "{{ MODE }}" = "root" ]; then
+	./run_null.sh SUDO
+	runtests=$((runtests+1))
+else
+	./run_null.sh
+	runtests=$((runtests+1))
+fi
+
+# test all --dry-run variants
+
+# we are testing all variants here because with 0.7.5 we had a bug:
+# mmdebstrap sid /dev/null --simulate ==> E: cannot read /var/cache/apt/archives/
+for variant in extract custom essential apt minbase buildd important standard; do
+	for mode in root unshare fakechroot proot chrootless; do

tests/custom-tmpdir

@@ -5,18 +5,14 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-# https://www.etalabs.net/sh_tricks.html
-quote () { printf %s\\n "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" ; }
 adduser --gecos user --disabled-password user
 sysctl -w kernel.unprivileged_userns_clone=1
 homedir=$(runuser -u user -- sh -c 'cd && pwd')
-# apt:test/integration/test-apt-key
+runuser -u user -- mkdir "$homedir/tmp"
-TMPDIR_ADD="This is fü\$\$ing cràzy, \$(apt -v)\$!"
+runuser -u user -- env TMPDIR="$homedir/tmp" {{ CMD }} --mode=unshare --variant=apt \
-runuser -u user -- mkdir "$homedir/$TMPDIR_ADD"
+	--setup-hook='case "$1" in "'"$homedir/tmp/mmdebstrap."'"??????????) exit 0;; *) exit 1;; esac' \
-runuser -u user -- env TMPDIR="$homedir/$TMPDIR_ADD" {{ CMD }} --mode=unshare --variant=apt \
-	--setup-hook='case "$1" in '"$(quote "$homedir/$TMPDIR_ADD/mmdebstrap.")"'??????????) exit 0;; *) echo "$1"; exit 1;; esac' \
 	 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
 # use rmdir as a quick check that nothing is remaining in TMPDIR
-runuser -u user -- rmdir "$homedir/$TMPDIR_ADD"
+runuser -u user -- rmdir "$homedir/tmp"
 rm /tmp/debian-chroot.tar

tests/dev-ptmx

@@ -1,145 +0,0 @@
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-
-if [ {{ MODE }} != unshare ] && [ {{ MODE }} != root ]; then
-	echo "test requires root or unshare mode" >&2
-	exit 1
-fi
-
-if [ ! -e /mmdebstrap-testenv ]; then
-	echo "this test modifies the system and should only be run inside a container" >&2
-	exit 1
-fi
-if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	adduser --gecos user --disabled-password user
-fi
-prefix=
-[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
-
-# this mimics what apt does in apt-pkg/deb/dpkgpm.cc/pkgDPkgPM::StartPtyMagic()
-cat > /tmp/test.c << 'END'
-#define _GNU_SOURCE
-
-#include <stdlib.h>
-#include <fcntl.h>
-#include <termios.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <sys/ioctl.h>
-#include <signal.h>
-
-int main() {
-	int ret;
-	int fd = posix_openpt(O_RDWR | O_NOCTTY);
-	if (fd < 0) {
-		perror("posix_openpt");
-		return 1;
-	}
-	char buf[64]; // 64 is used by apt
-	ret = ptsname_r(fd, buf, sizeof(buf));
-	if (ret != 0) {
-		perror("ptsname_r");
-		return 1;
-	}
-	ret = grantpt(fd);
-	if (ret == -1) {
-		perror("grantpt");
-		return 1;
-	}
-	struct termios origtt;
-	ret = tcgetattr(STDIN_FILENO, &origtt);
-	if (ret != 0) {
-		perror("tcgetattr1");
-		return 1;
-	}
-	struct termios tt;
-	ret = tcgetattr(STDOUT_FILENO, &tt);
-	if (ret != 0) {
-		perror("tcgetattr2");
-		return 1;
-	}
-	struct winsize win;
-	ret = ioctl(STDOUT_FILENO, TIOCGWINSZ, &win);
-	if (ret < 0) {
-		perror("ioctl stdout TIOCGWINSZ");
-		return 1;
-	}
-	ret = ioctl(fd, TIOCSWINSZ, &win);
-	if (ret < 0) {
-		perror("ioctl fd TIOCGWINSZ");
-		return 1;
-	}
-	ret = tcsetattr(fd, TCSANOW, &tt);
-	if (ret != 0) {
-		perror("tcsetattr1");
-		return 1;
-	}
-	cfmakeraw(&tt);
-	tt.c_lflag &= ~ECHO;
-	tt.c_lflag |= ISIG;
-	sigset_t sigmask;
-	sigset_t sigmask_old;
-	ret = sigemptyset(&sigmask);
-	if (ret != 0) {
-		perror("sigemptyset");
-		return 1;
-	}
-	ret = sigaddset(&sigmask, SIGTTOU);
-	if (ret != 0) {
-		perror("sigaddset");
-		return 1;
-	}
-	ret = sigprocmask(SIG_BLOCK,&sigmask, &sigmask_old);
-	if (ret != 0) {
-		perror("sigprocmask1");
-		return 1;
-	}
-	ret = tcsetattr(STDIN_FILENO, TCSAFLUSH, &tt);
-	if (ret != 0) {
-		perror("tcsetattr2");
-		return 1;
-	}
-	ret = sigprocmask(SIG_BLOCK,&sigmask_old, NULL);
-	if (ret != 0) {
-		perror("sigprocmask2");
-		return 1;
-	}
-	ret = tcsetattr(STDIN_FILENO, TCSAFLUSH, &origtt);
-	if (ret != 0) {
-		perror("tcsetattr3");
-		return 1;
-	}
-	return 0;
-}
-END
-
-# use script to create a fake tty
-# run all tests as root and as a normal user (the latter requires ptmxmode=666)
-script -qfc "$prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
-	--include=gcc,libc6-dev,python3 \
-	--customize-hook='chroot \"\$1\" adduser --gecos user --disabled-password user' \
-	--customize-hook='chroot \"\$1\" python3 -c \"import pty; print(pty.openpty())\"' \
-	--customize-hook='chroot \"\$1\" runuser -u user -- python3 -c \"import pty; print(pty.openpty())\"' \
-	--customize-hook='chroot \"\$1\" script -c \"echo foobar\"' \
-	--customize-hook='chroot \"\$1\" runuser -u user -- env --chdir=/home/user script -c \"echo foobar\"' \
-	--customize-hook='chroot \"\$1\" apt-get install --yes doc-debian 2>&1 | tee /tmp/log' \
-	--customize-hook=\"copy-in /tmp/test.c /tmp\" \
-	--customize-hook='chroot \"\$1\" gcc /tmp/test.c -o /tmp/test' \
-	--customize-hook='chroot \"\$1\" /tmp/test' \
-	--customize-hook='chroot \"\$1\" runuser -u user -- /tmp/test' \
-	--customize-hook='rm \"\$1\"/tmp/test \"\$1\"/tmp/test.c' \
-	{{ DIST }} /dev/null {{ MIRROR }}" /dev/null
-
-fail=0
-grep '^E:' /tmp/log && fail=1
-grep 'Can not write log' /tmp/log && fail=1
-grep 'posix_openpt' /tmp/log && fail=1
-grep 'No such file or directory' /tmp/log && fail=1
-if [ $fail -eq 1 ]; then
-	echo "apt failed to write log:" >&2
-	cat /tmp/log >&2
-	exit 1
-fi
-
-rm /tmp/test.c /tmp/log