Mount a new instance of /dev/pts in the chroot

Before, we bind-mounted /dev/ptmx and /dev/pts from the host into the
chroot. This will make posix_openpt() fail with 'No such file or
directory'.  The ability to create pseudo terminals is important for apt
(which will throw a warning otherwise) or running script(1) or source
package testsuites like for src:util-linux. This functionality is
restored by mounting a new devpts instance to /dev/pts and making
/dev/ptmx a symlink to /dev/pts/ptmx. Mounting with ptmxmode=666 is
required such that also non-root users in unshare mode are able to
create pseudo terminals. See also:

https://www.kernel.org/doc/Documentation/filesystems/devpts.txt
https://salsa.debian.org/debian/schroot/-/merge_requests/2
https://bugs.debian.org/856877
https://bugs.debian.org/817236

coverage.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-from debian.deb822 import Deb822
+from debian.deb822 import Deb822, Release
+import email.utils
 import os
 import sys
 import shutil
@@ -14,6 +15,7 @@ have_qemu = os.getenv("HAVE_QEMU", "yes") == "yes"
 have_unshare = os.getenv("HAVE_UNSHARE", "yes") == "yes"
 have_binfmt = os.getenv("HAVE_BINFMT", "yes") == "yes"
 run_ma_same_tests = os.getenv("RUN_MA_SAME_TESTS", "yes") == "yes"
+cmd = os.getenv("CMD", "./mmdebstrap")
 
 default_dist = os.getenv("DEFAULT_DIST", "unstable")
 all_dists = ["oldstable", "stable", "testing", "unstable"]
@@ -33,11 +35,21 @@ all_variants = [
 default_format = "auto"
 all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "null"]
 
-only_dists = []
-
 mirror = os.getenv("mirror", "http://127.0.0.1/debian")
 hostarch = subprocess.check_output(["dpkg", "--print-architecture"]).decode().strip()
 
+release_path = f"./shared/cache/debian/dists/{default_dist}/Release"
+if not os.path.exists(release_path):
+    print("path doesn't exist:", release_path, file=sys.stderr)
+    print("run ./make_mirror.sh first", file=sys.stderr)
+    exit(1)
+if os.getenv("SOURCE_DATE_EPOCH") is not None:
+    s_d_e = os.getenv("SOURCE_DATE_EPOCH")
+else:
+    with open(release_path) as f:
+        rel = Release(f)
+    s_d_e = str(email.utils.mktime_tz(email.utils.parsedate_tz(rel["Date"])))
+
 separator = (
     "------------------------------------------------------------------------------"
 )
@@ -75,6 +87,7 @@ def main():
         help="exit after first num failures or errors.",
     )
     parser.add_argument("--mode", metavar="mode", help="only run tests with this mode")
+    parser.add_argument("--dist", metavar="dist", help="only run tests with this dist")
     args = parser.parse_args()
 
     # copy over files from git or as distributed
@@ -105,16 +118,10 @@ def main():
             "/usr/share/mmdebstrap/hooks", "shared/hooks", dirs_exist_ok=True
         )
 
-    onlyrun = None
-    if len(sys.argv) > 1:
-        onlyrun = sys.argv[1]
-
     tests = []
     with open("coverage.txt") as f:
         for test in Deb822.iter_paragraphs(f):
             name = test["Test"]
-            if args.test and name not in args.test:
-                continue
             dists = test.get("Dists", default_dist)
             if dists == "any":
                 dists = all_dists
@@ -144,11 +151,7 @@ def main():
             else:
                 formats = formats.split()
             for dist in dists:
-                if only_dists and dist not in only_dists:
-                    continue
                 for mode in modes:
-                    if args.mode and args.mode != mode:
-                        continue
                     for variant in variants:
                         for fmt in formats:
                             skipreason = skip(
@@ -172,27 +175,60 @@ def main():
                                 tt = "null"
                             tests.append((tt, name, dist, mode, variant, fmt))
 
+    torun = []
+    num_tests = len(tests)
+    if args.test:
+        # check if all given tests are either a valid name or a valid number
+        for test in args.test:
+            if test in [name for (_, name, _, _, _, _) in tests]:
+                continue
+            if not test.isdigit():
+                print(f"cannot find test named {test}", file=sys.stderr)
+                exit(1)
+            if int(test) >= len(tests) or int(test) <= 0 or str(int(test)) != test:
+                print(f"test number {test} doesn't exist", file=sys.stderr)
+                exit(1)
+
+        for i, (_, name, _, _, _, _) in enumerate(tests):
+            # if either the number or test name matches, then we use this test,
+            # otherwise we skip it
+            if name in args.test:
+                torun.append(i)
+            if str(i + 1) in args.test:
+                torun.append(i)
+        num_tests = len(torun)
+
     starttime = time.time()
     skipped = defaultdict(list)
     failed = []
     num_success = 0
+    num_finished = 0
     for i, (test, name, dist, mode, variant, fmt) in enumerate(tests):
+        if torun and i not in torun:
+            continue
         print(separator, file=sys.stderr)
         print("(%d/%d) %s" % (i + 1, len(tests), name), file=sys.stderr)
         print("dist: %s" % dist, file=sys.stderr)
         print("mode: %s" % mode, file=sys.stderr)
         print("variant: %s" % variant, file=sys.stderr)
         print("format: %s" % fmt, file=sys.stderr)
-        if i > 0:
+        if num_finished > 0:
             currenttime = time.time()
             timeleft = timedelta(
-                seconds=int((len(tests) - i) * (currenttime - starttime) / i)
+                seconds=int(
+                    (num_tests - num_finished)
+                    * (currenttime - starttime)
+                    / num_finished
+                )
             )
             print("time left: %s" % timeleft, file=sys.stderr)
+        if failed:
+            print("failed: %d" % len(failed))
+        num_finished += 1
         with open("tests/" + name) as fin, open("shared/test.sh", "w") as fout:
             for line in fin:
-                for e in ["CMD", "SOURCE_DATE_EPOCH"]:
-                    line = line.replace("{{ " + e + " }}", os.getenv(e))
+                line = line.replace("{{ CMD }}", cmd)
+                line = line.replace("{{ SOURCE_DATE_EPOCH }}", s_d_e)
                 line = line.replace("{{ DIST }}", dist)
                 line = line.replace("{{ MIRROR }}", mirror)
                 line = line.replace("{{ MODE }}", mode)
@@ -215,11 +251,18 @@ def main():
                 print(f"skipped because of {reason}", file=sys.stderr)
                 continue
         print(separator, file=sys.stderr)
+        if args.dist and args.dist != dist:
+            print(f"skipping because of --dist={args.dist}", file=sys.stderr)
+            continue
+        if args.mode and args.mode != mode:
+            print(f"skipping because of --mode={args.mode}", file=sys.stderr)
+            continue
         proc = subprocess.Popen(argv)
         try:
             proc.wait()
         except KeyboardInterrupt:
-            proc.kill()
+            proc.terminate()
+            proc.wait()
             break
         print(separator, file=sys.stderr)
         if proc.returncode != 0:
@@ -237,7 +280,7 @@ def main():
         file=sys.stderr,
     )
     if skipped:
-        print("skipped %d:" % len(skipped), file=sys.stderr)
+        print("skipped %d:" % sum([len(v) for v in skipped.values()]), file=sys.stderr)
         for reason, l in skipped.items():
             print(f"skipped because of {reason}:", file=sys.stderr)
             for t in l:

coverage.txt

@@ -271,6 +271,8 @@ Skip-If:
  variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
  variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
 
+Test: create-directory-dry-run
+
 Test: create-tarball-dry-run
 Variants: any
 Modes: any
@@ -316,3 +318,7 @@ Skip-If:
 
 Test: no-sbin-in-path
 Modes: fakechroot
+
+Test: dev-ptmx
+Modes: root unshare
+Needs-QEMU: true

hooks/file-mirror-automount/setup00.sh

@@ -8,8 +8,8 @@ fi
 
 rootdir="$1"
 
-env APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get indextargets --no-release-info \
-	| sed -ne 's/^Repo-URI: file:\/\+//p' \
+env APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get indextargets --no-release-info --format '$(REPO_URI)' \
+	| sed -ne 's/^file:\/\+//p' \
 	| sort -u \
 	| while read path; do
 		mkdir -p "$rootdir/run/mmdebstrap"

make_mirror.sh

@@ -660,6 +660,7 @@ for dist in oldstable stable testing unstable; do
 set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
+echo "SOURCE_DATE_EPOCH=\$SOURCE_DATE_EPOCH"
 tmpdir="\$(mktemp -d)"
 chmod 755 "\$tmpdir"
 debootstrap --no-merged-usr --variant=$variant $dist "\$tmpdir" $mirror

mmdebstrap

@@ -1091,6 +1091,20 @@ sub run_chroot {
                             );
                             next;
                         }
+                        if ($fname eq "ptmx") {
+                            # We must not bind-mount ptmx from the outside or
+                            # otherwise posix_openpt() will fail. Instead
+                            # /dev/ptmx must refer to /dev/pts/ptmx either by
+                            # symlink or by bind-mounting. We choose a symlink.
+                            symlink '/dev/pts/ptmx',
+                              "$options->{root}/dev/ptmx"
+                              or error "cannot create /dev/pts/ptmx symlink";
+                            push @cleanup_tasks, sub {
+                                unlink "$options->{root}/dev/ptmx"
+                                  or error "unlink /dev/ptmx";
+                            };
+                            next;
+                        }
                         if (!-e "/dev/$fname") {
                             warning("skipping creation of ./dev/$fname because"
                                   . " /dev/$fname does not exist"
@@ -1133,13 +1147,13 @@ sub run_chroot {
                               . " /dev directory is missing in the target");
                         next;
                     }
-                    if (!-e "/dev/$fname") {
+                    if (!-e "/dev/$fname" && $fname ne "pts/") {
                         warning("skipping creation of ./dev/$fname because"
                               . " /dev/$fname does not exist"
                               . " on the outside");
                         next;
                     }
-                    if (!-d "/dev/$fname") {
+                    if (!-d "/dev/$fname" && $fname ne "pts/") {
                         warning("skipping creation of ./dev/$fname because"
                               . " /dev/$fname on the outside is not a"
                               . " directory");
@@ -1187,9 +1201,32 @@ sub run_chroot {
                             "$options->{root}/dev/$fname")
                           or warning("umount ./dev/$fname failed: $?");
                     };
-                    0 == system('mount', '-o', 'bind', "/dev/$fname",
-                        "$options->{root}/dev/$fname")
-                      or error "mount ./dev/$fname failed: $?";
+                    if ($fname eq "pts/") {
+                        # We cannot just bind-mount /dev/pts from the host as
+                        # doing so will make posix_openpt() fail. Instead, we
+                        # need to mount a new devpts.
+                        # We need ptmxmode=666 because /dev/ptmx is a symlink
+                        # to /dev/pts/ptmx and without it posix_openpt() will
+                        # fail if we are not the root user.
+                        # See also:
+                        # kernel.org/doc/Documentation/filesystems/devpts.txt
+                        # salsa.debian.org/debian/schroot/-/merge_requests/2
+                        # https://bugs.debian.org/856877
+                        # https://bugs.debian.org/817236
+                        0 == system(
+                            'mount',
+                            '-t',
+                            'devpts',
+                            'none',
+                            "$options->{root}/dev/pts",
+                            '-o',
+                            'noexec,nosuid,uid=5,mode=620,ptmxmode=666'
+                        ) or error "mount /dev/pts failed";
+                    } else {
+                        0 == system('mount', '-o', 'bind', "/dev/$fname",
+                            "$options->{root}/dev/$fname")
+                          or error "mount ./dev/$fname failed: $?";
+                    }
                 } else {
                     error "unsupported type: $type";
                 }
@@ -4896,8 +4933,8 @@ sub main() {
                 }
                 # initialize gpg trustdb with empty one
                 {
-                    `@gpgcmd --update-trustdb >/dev/null 2>/dev/null`;
-                    $? == 0 or error "gpg failed to initialize trustdb: $?";
+                    0 == system(@gpgcmd, '--update-trustdb')
+                      or error "gpg failed to initialize trustdb:: $?";
                 }
                 # find all the fingerprints of the keys apt currently
                 # knows about

run_qemu.sh

@@ -31,7 +31,7 @@ qemu-img create -f qcow2 -b "$(realpath $cachedir)/debian-$DEFAULT_DIST.qcow" -F
 # to connect to serial use:
 #   minicom -D 'unix#/tmp/ttyS0'
 ret=0
-timeout 20m qemu-system-x86_64 \
+timeout --foreground 20m qemu-system-x86_64 \
 	-cpu host \
 	-no-user-config \
 	-M accel=kvm:tcg -m 4G -nographic \

tests/arm64-without-qemu-support

@@ -5,7 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-apt-get remove --yes qemu-user-static binfmt-support qemu-user
+echo 0 > /proc/sys/fs/binfmt_misc/qemu-aarch64
 ret=0
 {{ CMD }} --mode={{ MODE }} --variant=apt --architectures=arm64 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }} || ret=$?
 if [ "$ret" = 0 ]; then

tests/check-against-debootstrap-dist

@@ -3,6 +3,8 @@ set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
+echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
+
 # we create the apt user ourselves or otherwise its uid/gid will differ
 # compared to the one chosen in debootstrap because of different installation
 # order in comparison to the systemd users

tests/create-directory-dry-run

@@ -1,8 +1,11 @@
-cat << END > shared/test.sh
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-{{ CMD }} --mode={{ MODE }} --dry-run --variant=apt --setup-hook="exit 1" --essential-hook="exit 1" --customize-hook="exit 1" {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+{{ CMD }} --mode={{ MODE }} --dry-run --variant=apt \
+	--setup-hook="exit 1" \
+	--essential-hook="exit 1" \
+	--customize-hook="exit 1" \
+	{{ DIST }} /tmp/debian-chroot {{ MIRROR }}
 rm /tmp/debian-chroot/dev/console
 rm /tmp/debian-chroot/dev/fd
 rm /tmp/debian-chroot/dev/full
@@ -23,21 +26,3 @@ rm /tmp/debian-chroot/var/lib/apt/lists/lock
 rm /tmp/debian-chroot/var/lib/dpkg/status
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir
-END
-if [ "$HAVE_QEMU" = "yes" ]; then
-	./run_qemu.sh
-	runtests=$((runtests+1))
-elif [ "{{ MODE }}" = "root" ]; then
-	./run_null.sh SUDO
-	runtests=$((runtests+1))
-else
-	./run_null.sh
-	runtests=$((runtests+1))
-fi
-
-# test all --dry-run variants
-
-# we are testing all variants here because with 0.7.5 we had a bug:
-# mmdebstrap sid /dev/null --simulate ==> E: cannot read /var/cache/apt/archives/
-for variant in extract custom essential apt minbase buildd important standard; do
-	for mode in root unshare fakechroot proot chrootless; do

tests/custom-tmpdir

@@ -5,14 +5,18 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
+# https://www.etalabs.net/sh_tricks.html
+quote () { printf %s\\n "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" ; }
 adduser --gecos user --disabled-password user
 sysctl -w kernel.unprivileged_userns_clone=1
 homedir=$(runuser -u user -- sh -c 'cd && pwd')
-runuser -u user -- mkdir "$homedir/tmp"
-runuser -u user -- env TMPDIR="$homedir/tmp" {{ CMD }} --mode=unshare --variant=apt \
-	--setup-hook='case "$1" in "'"$homedir/tmp/mmdebstrap."'"??????????) exit 0;; *) exit 1;; esac' \
+# apt:test/integration/test-apt-key
+TMPDIR_ADD="This is fü\$\$ing cràzy, \$(apt -v)\$!"
+runuser -u user -- mkdir "$homedir/$TMPDIR_ADD"
+runuser -u user -- env TMPDIR="$homedir/$TMPDIR_ADD" {{ CMD }} --mode=unshare --variant=apt \
+	--setup-hook='case "$1" in '"$(quote "$homedir/$TMPDIR_ADD/mmdebstrap.")"'??????????) exit 0;; *) echo "$1"; exit 1;; esac' \
 	 {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
 # use rmdir as a quick check that nothing is remaining in TMPDIR
-runuser -u user -- rmdir "$homedir/tmp"
+runuser -u user -- rmdir "$homedir/$TMPDIR_ADD"
 rm /tmp/debian-chroot.tar

tests/dev-ptmx

@@ -0,0 +1,145 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+
+if [ {{ MODE }} != unshare ] && [ {{ MODE }} != root ]; then
+	echo "test requires root or unshare mode" >&2
+	exit 1
+fi
+
+if [ ! -e /mmdebstrap-testenv ]; then
+	echo "this test modifies the system and should only be run inside a container" >&2
+	exit 1
+fi
+if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
+	adduser --gecos user --disabled-password user
+fi
+prefix=
+[ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
+
+# this mimics what apt does in apt-pkg/deb/dpkgpm.cc/pkgDPkgPM::StartPtyMagic()
+cat > /tmp/test.c << 'END'
+#define _GNU_SOURCE
+
+#include <stdlib.h>
+#include <fcntl.h>
+#include <termios.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <signal.h>
+
+int main() {
+	int ret;
+	int fd = posix_openpt(O_RDWR | O_NOCTTY);
+	if (fd < 0) {
+		perror("posix_openpt");
+		return 1;
+	}
+	char buf[64]; // 64 is used by apt
+	ret = ptsname_r(fd, buf, sizeof(buf));
+	if (ret != 0) {
+		perror("ptsname_r");
+		return 1;
+	}
+	ret = grantpt(fd);
+	if (ret == -1) {
+		perror("grantpt");
+		return 1;
+	}
+	struct termios origtt;
+	ret = tcgetattr(STDIN_FILENO, &origtt);
+	if (ret != 0) {
+		perror("tcgetattr1");
+		return 1;
+	}
+	struct termios tt;
+	ret = tcgetattr(STDOUT_FILENO, &tt);
+	if (ret != 0) {
+		perror("tcgetattr2");
+		return 1;
+	}
+	struct winsize win;
+	ret = ioctl(STDOUT_FILENO, TIOCGWINSZ, &win);
+	if (ret < 0) {
+		perror("ioctl stdout TIOCGWINSZ");
+		return 1;
+	}
+	ret = ioctl(fd, TIOCSWINSZ, &win);
+	if (ret < 0) {
+		perror("ioctl fd TIOCGWINSZ");
+		return 1;
+	}
+	ret = tcsetattr(fd, TCSANOW, &tt);
+	if (ret != 0) {
+		perror("tcsetattr1");
+		return 1;
+	}
+	cfmakeraw(&tt);
+	tt.c_lflag &= ~ECHO;
+	tt.c_lflag |= ISIG;
+	sigset_t sigmask;
+	sigset_t sigmask_old;
+	ret = sigemptyset(&sigmask);
+	if (ret != 0) {
+		perror("sigemptyset");
+		return 1;
+	}
+	ret = sigaddset(&sigmask, SIGTTOU);
+	if (ret != 0) {
+		perror("sigaddset");
+		return 1;
+	}
+	ret = sigprocmask(SIG_BLOCK,&sigmask, &sigmask_old);
+	if (ret != 0) {
+		perror("sigprocmask1");
+		return 1;
+	}
+	ret = tcsetattr(STDIN_FILENO, TCSAFLUSH, &tt);
+	if (ret != 0) {
+		perror("tcsetattr2");
+		return 1;
+	}
+	ret = sigprocmask(SIG_BLOCK,&sigmask_old, NULL);
+	if (ret != 0) {
+		perror("sigprocmask2");
+		return 1;
+	}
+	ret = tcsetattr(STDIN_FILENO, TCSAFLUSH, &origtt);
+	if (ret != 0) {
+		perror("tcsetattr3");
+		return 1;
+	}
+	return 0;
+}
+END
+
+# use script to create a fake tty
+# run all tests as root and as a normal user (the latter requires ptmxmode=666)
+script -qfc "$prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
+	--include=gcc,libc6-dev,python3 \
+	--customize-hook='chroot \"\$1\" adduser --gecos user --disabled-password user' \
+	--customize-hook='chroot \"\$1\" python3 -c \"import pty; print(pty.openpty())\"' \
+	--customize-hook='chroot \"\$1\" runuser -u user -- python3 -c \"import pty; print(pty.openpty())\"' \
+	--customize-hook='chroot \"\$1\" script -c \"echo foobar\"' \
+	--customize-hook='chroot \"\$1\" runuser -u user -- env --chdir=/home/user script -c \"echo foobar\"' \
+	--customize-hook='chroot \"\$1\" apt-get install --yes doc-debian 2>&1 | tee /tmp/log' \
+	--customize-hook=\"copy-in /tmp/test.c /tmp\" \
+	--customize-hook='chroot \"\$1\" gcc /tmp/test.c -o /tmp/test' \
+	--customize-hook='chroot \"\$1\" /tmp/test' \
+	--customize-hook='chroot \"\$1\" runuser -u user -- /tmp/test' \
+	--customize-hook='rm \"\$1\"/tmp/test \"\$1\"/tmp/test.c' \
+	{{ DIST }} /dev/null {{ MIRROR }}" /dev/null
+
+fail=0
+grep '^E:' /tmp/log && fail=1
+grep 'Can not write log' /tmp/log && fail=1
+grep 'posix_openpt' /tmp/log && fail=1
+grep 'No such file or directory' /tmp/log && fail=1
+if [ $fail -eq 1 ]; then
+	echo "apt failed to write log:" >&2
+	cat /tmp/log >&2
+	exit 1
+fi
+
+rm /tmp/test.c /tmp/log