tests: do not run sysctl -w kernel.unprivileged_userns_clone=1 as its the default value since linux 5.10.1 (Dec 2020)

Thanks: Helmut Grohne for finding this issue and help interating this

.mailmap

@@ -1,4 +1,6 @@
 Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
 Johannes Schauer Marin Rodrigues <josch@mister-muffin.de> <j.schauer@email.de>
+Johannes Schauer Marin Rodrigues <josch@mister-muffin.de> <josch@debian.org>
+Johannes Schauer Marin Rodrigues <josch@mister-muffin.de> <Johannes Schauer Marin Rodrigues josch@debian.org>
 Helmut Grohne <helmut@subdivi.de> <helmut.grohne@intenta.de>
 Benjamin Drung <benjamin.drung@ionos.com> <benjamin.drung@cloud.ionos.com>

CHANGELOG.md

@@ -1,3 +1,8 @@
+1.3.1 (2023-01-20)
+------------------
+
+ - bugfix release
+
 1.3.0 (2023-01-16)
 ------------------
 

coverage.py

@@ -132,6 +132,30 @@ def format_test(num, total, name, dist, mode, variant, fmt, config_dict):
     return ret
 
 
+def print_time_per_test(time_per_test, name="test"):
+    print(
+        f"average time per {name}:",
+        sum(time_per_test.values(), start=timedelta()) / len(time_per_test),
+        file=sys.stderr,
+    )
+    print(
+        f"median time per {name}:",
+        sorted(time_per_test.values())[len(time_per_test) // 2],
+        file=sys.stderr,
+    )
+    head_tail_num = 10
+    print(f"{head_tail_num} fastests {name}s:", file=sys.stderr)
+    for k, v in sorted(time_per_test.items(), key=lambda i: i[1])[
+        : min(head_tail_num, len(time_per_test))
+    ]:
+        print(f"    {k}: {v}", file=sys.stderr)
+    print(f"{head_tail_num} slowest {name}s:", file=sys.stderr)
+    for k, v in sorted(time_per_test.items(), key=lambda i: i[1], reverse=True)[
+        : min(head_tail_num, len(time_per_test))
+    ]:
+        print(f"    {k}: {v}", file=sys.stderr)
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("test", nargs="*", help="only run these tests")
@@ -288,6 +312,7 @@ def main():
     num_success = 0
     num_finished = 0
     time_per_test = {}
+    acc_time_per_test = defaultdict(list)
     for i, (test, name, dist, mode, variant, fmt) in enumerate(tests):
         if torun and i not in torun:
             continue
@@ -380,6 +405,7 @@ def main():
             i + 1, len(tests), name, dist, mode, variant, fmt, config_dict
         )
         time_per_test[formated_test_name] = walltime
+        acc_time_per_test[name].append(walltime)
         print(separator, file=sys.stderr)
         print(f"duration: {walltime}", file=sys.stderr)
         if proc.returncode != 0 or shellcheck != "":
@@ -409,27 +435,15 @@ def main():
             print(f, file=sys.stderr)
         exit(1)
     if len(time_per_test) > 1:
-        print(
-            "average time per test:",
-            sum(time_per_test.values(), start=timedelta()) / len(time_per_test),
-            file=sys.stderr,
+        print_time_per_test(time_per_test)
+    if len(acc_time_per_test) > 1:
+        print_time_per_test(
+            {
+                f"{len(v)}x {k}": sum(v, start=timedelta())
+                for k, v in acc_time_per_test.items()
+            },
+            "accumulated test",
         )
-        print(
-            "median time per test:",
-            sorted(time_per_test.values())[len(time_per_test) // 2],
-            file=sys.stderr,
-        )
-        head_tail_num = 10
-        print(f"{head_tail_num} fastests tests:", file=sys.stderr)
-        for k, v in sorted(time_per_test.items(), key=lambda i: i[1])[
-            : min(head_tail_num, len(time_per_test))
-        ]:
-            print(f"    {k}: {v}", file=sys.stderr)
-        print(f"{head_tail_num} slowest tests:", file=sys.stderr)
-        for k, v in sorted(time_per_test.items(), key=lambda i: i[1], reverse=True)[
-            : min(head_tail_num, len(time_per_test))
-        ]:
-            print(f"    {k}: {v}", file=sys.stderr)
 
 
 if __name__ == "__main__":

coverage.txt

@@ -364,3 +364,6 @@ Test: apt-patterns
 Test: apt-patterns-custom
 
 Test: empty-sources.list
+
+Test: merged-fakechroot-inside-unmerged-chroot
+Needs-Root: true

hooks/maybe-jessie-or-older/extract00.sh

@@ -2,15 +2,24 @@
 
 set -eu
 
-# if dpkg is new enough, do nothing
-# we cannot ask dpkg-query about the version because dpkg is only extracted
-# but not installed at this point
-dpkg_ver="$(chroot "$1" dpkg --version | grep --extended-regexp --only-matching '[0-9]+\.[0-9.]+')"
-if dpkg --compare-versions "$dpkg_ver" ge 1.17.11; then
-	echo "dpkg version $dpkg_ver is >= 1.17.11 -- not running jessie-or-older extract00 hook" >&2
+# we need to check the version of dpkg
+# since at this point packages are just extracted but not installed, we cannot use dpkg-query
+# since we want to support chrootless, we cannot run dpkg --version inside the chroot
+# to avoid this hook depending on dpkg-dev being installed, we do not parse the extracted changelog with dpkg-parsechangelog
+# we also want to avoid parsing the changelog because /usr/share/doc might've been added to dpkg --path-exclude
+# instead, we just ask apt about the latest version of dpkg it knows of
+# this should only fail in situations where there are multiple versions of dpkg in different suites
+ver=$(env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions dpkg 2>/dev/null | sed -ne 's/^Version: \(.*\)$/\1/p' || printf '')
+if [ -z "$ver" ]; then
+	echo "no package called dpkg can be installed -- not running jessie-or-older extract00 hook" >&2
+	exit 0
+fi
+
+if dpkg --compare-versions "$ver" ge 1.17.11; then
+	echo "dpkg version $ver is >= 1.17.11 -- not running jessie-or-older extract00 hook" >&2
 	exit 0
 else
-	echo "dpkg version $dpkg_ver is << 1.17.11 -- running jessie-or-older extract00 hook" >&2
+	echo "dpkg version $ver is << 1.17.11 -- running jessie-or-older extract00 hook" >&2
 fi
 
 # resolve the script path using several methods in order:

hooks/maybe-jessie-or-older/extract01.sh

@@ -2,18 +2,44 @@
 
 set -eu
 
-# If the init package has not been extracted, then it is not part of the
-# Essential:yes set and we do not need this workaround. This holds true for the
-# init package version 1.34 and later. Instead of asking apt about the init
-# version (which might not be the same version that was picked to be installed)
-# we check for the presence of the init package by checking whether
-# /usr/share/doc/init/copyright exists.
-
-if [ -e "$1/usr/share/doc/init/copyright" ]; then
-	echo "the init package is not Essential:yes -- not running jessie-or-older extract01 hook" >&2
+# The jessie-or-older extract01 hook has to be run up to the point where the
+# Essential:yes field was removed from the init package (with
+# init-system-helpers 1.34). Since the essential packages have only been
+# extracted but not installed, we cannot use dpkg-query to find out its
+# version. Since /usr/share/doc might be missing due to dpkg --path-exclude, we
+# also cannot check whether /usr/share/doc/init/copyright exists. There also
+# was a time (before init-system-helpers 1.20) where there was no init package
+# at all where we also want to apply this hook. So we just ask apt about the
+# candidate version for init-system-helpers. This should only fail in
+# situations where there are multiple versions of init-system-helpers in
+# different suites.
+ver=$(env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions init-system-helpers 2>/dev/null | sed -ne 's/^Version: \(.*\)$/\1/p' || printf '')
+if [ -z "$ver" ]; then
+	# there is no package called init-system-helpers, so either:
+	#  - this is so old that init-system-helpers didn't exist yet
+	#  - we are in a future where init-system-helpers doesn't exist anymore
+	#  - something strange is going on
+	# we should only call the hook in the first case
+	ver=$(env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions base-files 2>/dev/null | sed -ne 's/^Version: \(.*\)$/\1/p' || printf '')
+	if [ -z "$ver" ]; then
+		echo "neither init-system-helpers nor base-files can be installed -- not running jessie-or-older extract01 hook" >&2
 		exit 0
+	fi
+
+	# Jessie is Debian 8
+	if dpkg --compare-versions "$ver" ge 8; then
+		echo "there is no init-system-helpers but base-files version $ver is >= 8 -- not running jessie-or-older extract01 hook" >&2
+		exit 0
+	else
+		echo "there is no init-system-helpers but base-files version $ver is << 8 -- running jessie-or-older extract01 hook" >&2
+	fi
 else
-	echo "the init package is Essential:yes -- running jessie-or-older extract01 hook" >&2
+	if dpkg --compare-versions "$ver" ge 1.34; then
+		echo "init-system-helpers version $ver is >= 1.34 -- not running jessie-or-older extract01 hook" >&2
+		exit 0
+	else
+		echo "init-system-helpers version $ver is << 1.34 -- running jessie-or-older extract01 hook" >&2
+	fi
 fi
 
 # resolve the script path using several methods in order:

hooks/maybe-merged-usr/essential00.sh

@@ -2,13 +2,24 @@
 
 set -eu
 
-# if the usr-is-merged package cannot be installed with apt, do nothing
-if ! env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
-	echo "no package called usr-is-merged found -- not running merged-usr essential hook" >&2
+ver=$(dpkg-query --root="$1" -f '${db:Status-Status} ${Source} ${Version}' --show usr-is-merged 2>/dev/null || printf '')
+case "$ver" in
+	'')
+		echo "no package called usr-is-merged is installed -- not running merged-usr essential hook" >&2
 		exit 0
-else
-	echo "package usr-is-merged found -- running merged-usr essential hook" >&2
-fi
+		;;
+	'installed mmdebstrap-dummy-usr-is-merged 1')
+		echo "dummy usr-is-merged package installed -- running merged-usr essential hook" >&2
+		;;
+	'installed usrmerge '*)
+		echo "usr-is-merged package from src:usrmerge installed -- not running merged-usr essential hook" >&2
+		exit 0
+		;;
+	*)
+		echo "unexpected situation for package usr-is-merged: $ver" >&2
+		exit 1
+		;;
+esac
 
 # resolve the script path using several methods in order:
 #  1. using dirname -- "$0"

ldconfig.fakechroot

@@ -106,10 +106,24 @@ def main():
     for d in get_libdirs(chroot, [chroot / "etc" / "ld.so.conf"]):
         make_relative(d)
 
+    rootarg = chroot
+    argv = sys.argv[1:]
+    for arg in sys.argv[1:]:
+        if arg == "-r":
+            rootarg = None
+        elif rootarg is None:
+            argpath = Path(arg)
+            if argpath.is_absolute():
+                rootarg = chroot / argpath.relative_to("/")
+            else:
+                rootarg = Path.cwd() / argpath
+    if rootarg is None:
+        rootarg = chroot
+
     # we add any additional arguments before "-r" such that any other "-r"
     # option will be overwritten by the one we set
     subprocess.check_call(
-        [chroot / "sbin" / "ldconfig"] + sys.argv[1:] + ["-r", chroot]
+        [chroot / "sbin" / "ldconfig"] + sys.argv[1:] + ["-r", rootarg]
     )
 
 

make_mirror.sh

@@ -264,7 +264,7 @@ END
 			--or --field=Priority important --or --field=Priority standard \
 			\))
 
-	pkgs="$pkgs build-essential busybox gpg eatmydata"
+	pkgs="$pkgs build-essential busybox gpg eatmydata fakechroot fakeroot"
 
 	# we need usr-is-merged to simulate debootstrap behaviour for all dists
 	# starting from Debian 12 (Bullseye)

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '1.3.0';
+our $VERSION = '1.3.1';
 
 use English;
 use Getopt::Long;
@@ -963,6 +963,16 @@ sub run_dpkg_progress {
 sub run_apt_progress {
     my $options = shift;
     my @debs    = @{ $options->{PKGS} // [] };
+
+    if ($verbosity_level >= 3) {
+        my @apt_debug_opts = qw(
+          -oDebug::pkgProblemResolver=true
+          -oDebug::pkgDepCache::Marker=1
+          -oDebug::pkgDepCache::AutoInstall=1
+        );
+        push @{ $options->{ARGV} }, @apt_debug_opts;
+    }
+
     my $get_exec = sub {
         my @prefix = ();
         my @opts   = ();
@@ -1031,15 +1041,6 @@ sub run_apt_download_progress {
         info "downloading packages with apt...";
     }
 
-    if ($verbosity_level >= 3) {
-        my @apt_debug_opts = qw(
-          -oDebug::pkgProblemResolver=true
-          -oDebug::pkgDepCache::Marker=1
-          -oDebug::pkgDepCache::AutoInstall=1
-        );
-        push @{ $options->{APT_ARGV} }, @apt_debug_opts;
-    }
-
     pipe my $rfh, my $wfh;
     my $pid = open my $fh, '-|' // error "fork() failed: $!";
     if ($pid == 0) {
@@ -2251,10 +2252,21 @@ sub run_setup() {
         # root mode when the path of the chroot is not in a world-readable
         # location.
         my $partial = '/var/lib/apt/lists/partial';
-        if (
-            system('/usr/lib/apt/apt-helper', 'drop-privs', '--', 'test',
-                '-r', "$options->{root}$partial") != 0
-        ) {
+        my @testcmd = (
+            '/usr/lib/apt/apt-helper', 'drop-privs', '--', 'test',
+            '-r', "$options->{root}$partial"
+        );
+        my $pid = fork() // error "fork() failed: $!";
+        if ($pid == 0) {
+            open(STDOUT, '>', '/dev/null')
+              or error "cannot open /dev/null for writing: $!";
+            open(STDERR, '>', '/dev/null')
+              or error "cannot open /dev/null for writing: $!";
+            exec { $testcmd[0] } @testcmd
+              or error("cannot exec " . (join " ", @testcmd) . ": $!");
+        }
+        waitpid $pid, 0;
+        if ($? != 0) {
             warning "Download is performed unsandboxed as root as file"
               . " $options->{root}$partial couldn't be accessed by user _apt";
             open my $fh, '>>', $tmpfile
@@ -2602,20 +2614,30 @@ sub run_prepare {
                 $subst{ldconfig}
                   = '/usr/libexec/mmdebstrap/ldconfig.fakechroot';
             }
-            my @fakechrootsubst = ();
+            my %mergedusrmap = (
+                "/bin"      => "/usr/bin",
+                "/sbin"     => "/usr/sbin",
+                "/usr/bin/" => "/bin",
+                "/usr/sbin" => "/sbin"
+            );
+            my %fakechrootsubst;
             foreach my $d (split ':', $ENV{PATH}) {
-                foreach my $k (sort keys %subst) {
-                    if (-e "$d/$k") {
-                        push @fakechrootsubst, "$d/$k=$subst{$k}";
-                    }
+                foreach my $k (sort %subst) {
+                    my $mapped_path = $mergedusrmap{$d} // $d;
+                    next if !-e "$d/$k" && !-e "$mapped_path/$k";
+                    $fakechrootsubst{"$d/$k=$subst{$k}"}           = 1;
+                    $fakechrootsubst{"$mapped_path/$k=$subst{$k}"} = 1;
                 }
             }
             if (defined $ENV{FAKECHROOT_CMD_SUBST}
                 && $ENV{FAKECHROOT_CMD_SUBST} ne "") {
-                push @fakechrootsubst, split /:/, $ENV{FAKECHROOT_CMD_SUBST};
+                foreach my $e (split /:/, $ENV{FAKECHROOT_CMD_SUBST}) {
+                    $fakechrootsubst{$e} = 1;
+                }
             }
             ## no critic (Variables::RequireLocalizedPunctuationVars)
-            $ENV{FAKECHROOT_CMD_SUBST} = join ':', @fakechrootsubst;
+            $ENV{FAKECHROOT_CMD_SUBST} = join ':',
+              (sort keys %fakechrootsubst);
         }
         if (defined $ENV{FAKECHROOT_EXCLUDE_PATH}
             && $ENV{FAKECHROOT_EXCLUDE_PATH} ne "") {
@@ -6704,7 +6726,14 @@ C<Essential:yes> packages.
 
 =item B<apt>
 
-The B<essential> set plus apt.
+The B<essential> set plus apt. This variant uses the fact that B<apt> treats
+itself as essential and thus running C<apt-get dist-upgrade> without any
+packages installed will install the B<essential> set plus B<apt>. If you just
+want B<essential> and B<apt>, then this variant is faster than using the
+B<essential> variant and adding B<apt> via C<--include> because all packages
+get installed at once. The downside of this variant is, that if it should
+happen that an B<essential> package is not installable, then it will just get
+ignored without throwing an error.
 
 =item B<required>, B<minbase>
 
@@ -7452,6 +7481,15 @@ F</etc/unsupported-skip-usrmerge-conversion> inside the chroot and install the
 B<usr-is-merged> package to avoid the installation of the B<usrmerge> package
 and its dependencies.
 
+If you are using B<mmdebstrap> in a setup where you do not know upfront whether
+the chroot you are creating should be merged-/usr or not and you want to avoid
+installation of the B<usrmerge> package and it's dependencies, you can use:
+
+    --hook-dir=/usr/share/mmdebstrap/hooks/maybe-merged-usr
+
+That hook will use the availability of the B<usr-is-merged> package to decide
+whether to call the B<merged-usr> hook or not.
+
 =head1 COMPRESSION
 
 B<mmdebstrap> will choose a suitable compressor for the output tarball

run_null.sh

@@ -17,14 +17,24 @@ while [ "$#" -gt 0 ]; do
 	shift
 done
 
-# subshell so that we can cd without effecting the rest
-(
-	set +e
-	cd ./shared;
-	$SUDO sh -x ./test.sh;
-	echo $?;
-) 2>&1 | tee shared/output.txt
-if [ "$(cat shared/exitstatus.txt)" -ne 0 ]; then
+# - Run command with fds 3 and 4 closed so that whatever test.sh does it
+#   cannot interfere with these.
+# - Both stdin and stderr of test.sh are written to stdout
+# - Write exit status of test.sh to fd 3
+# - Write stdout to shared/output.txt as well as to fd 4
+# - Redirect fd 3 to stdout
+# - Read fd 3 and let the group exit with that value
+# - Redirect fd 4 to stdout
+ret=0
+{ { { {
+        ret=0;
+        ( exec 3>&- 4>&-; env --chdir=./shared $SUDO sh -x ./test.sh 2>&1) || ret=$?;
+        echo $ret >&3;
+      } | tee shared/output.txt >&4;
+    } 3>&1;
+  } | { read -r xs; exit "$xs"; }
+} 4>&1 || ret=$?
+if [ "$ret" -ne 0 ]; then
 	echo "test.sh failed"
 	exit 1
 fi

tests/as-debootstrap-unshare-wrapper

@@ -6,8 +6,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-sysctl -w kernel.unprivileged_userns_clone=1
-adduser --gecos user --disabled-password user
+useradd --home-dir /home/user --create-home user
 runuser -u user -- {{ CMD }} --variant=custom --mode=unshare --setup-hook='env container=lxc debootstrap unstable "$1" {{ MIRROR }}' - /tmp/debian-mm.tar {{ MIRROR }}
 
 mkdir /tmp/debian-mm

tests/auto-mode-without-unshare-capabilities

@@ -5,8 +5,10 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=0
+useradd --home-dir /home/user --create-home user
+if [ -e /proc/sys/kernel/unprivileged_userns_clone ] && [ "$(sysctl -n kernel.unprivileged_userns_clone)" = "1" ]; then
+	sysctl -w kernel.unprivileged_userns_clone=0
+fi
 runuser -u user -- {{ CMD }} --mode=auto --variant=apt {{ DIST }} /tmp/debian-chroot.tar.gz {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar.gz | sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar.gz

tests/check-for-bit-by-bit-identical-format-output

@@ -5,8 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+useradd --home-dir /home/user --create-home user
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 {{ CMD }} --mode=root --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-root.{{ FORMAT }} {{ MIRROR }}
 if [ "{{ FORMAT }}" = tar ]; then

tests/chrootless-fakeroot

@@ -8,7 +8,7 @@ if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
 		echo "this test modifies the system and should only be run inside a container" >&2
 		exit 1
 	fi
-	adduser --gecos user --disabled-password user
+	useradd --home-dir /home/user --create-home user
 fi
 prefix=
 [ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"

tests/create-arm64-tarball

@@ -6,14 +6,7 @@ if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
 		echo "this test modifies the system and should only be run inside a container" >&2
 		exit 1
 	fi
-	adduser --gecos user --disabled-password user
-fi
-if [ "{{ MODE }}" = unshare ]; then
-	if [ ! -e /mmdebstrap-testenv ]; then
-		echo "this test modifies the system and should only be run inside a container" >&2
-		exit 1
-	fi
-	sysctl -w kernel.unprivileged_userns_clone=1
+	useradd --home-dir /home/user --create-home user
 fi
 prefix=
 [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"

tests/create-gzip-compressed-tarball

@@ -5,8 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+useradd --home-dir /home/user --create-home user
 runuser -u user -- {{ CMD }} --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot.tar.gz {{ MIRROR }}
 printf '\037\213\010' | cmp --bytes=3 /tmp/debian-chroot.tar.gz -
 tar -tf /tmp/debian-chroot.tar.gz | sort | diff -u tar1.txt -

tests/create-tarball-dry-run

@@ -14,14 +14,7 @@ if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != root ] && [ "{{ MODE }}" != auto ];
 			echo "this test modifies the system and should only be run inside a container" >&2
 			exit 1
 		fi
-		adduser --gecos user --disabled-password user
-	fi
-	if [ "{{ MODE }}" = unshare ]; then
-		if [ ! -e /mmdebstrap-testenv ]; then
-			echo "this test modifies the system and should only be run inside a container" >&2
-			exit 1
-		fi
-		sysctl -w kernel.unprivileged_userns_clone=1
+		useradd --home-dir /home/user --create-home user
 	fi
 	prefix="runuser -u user --"
 	if [ "{{ VARIANT }}" = extract ] || [ "{{ VARIANT }}" = custom ]; then

tests/custom-tmpdir

@@ -7,8 +7,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 fi
 # https://www.etalabs.net/sh_tricks.html
 quote () { printf %s\\n "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" ; }
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+useradd --home-dir /home/user --create-home user
 homedir=$(runuser -u user -- sh -c 'cd && pwd')
 # apt:test/integration/test-apt-key
 TMPDIR_ADD="This is fü\$\$ing cràzy, \$(apt -v)\$!"

tests/cwd-directory-not-accessible-by-unshared-user

@@ -5,8 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+useradd --home-dir /home/user --create-home user
 mkdir /tmp/debian-chroot
 chmod 700 /tmp/debian-chroot
 chown user:user /tmp/debian-chroot

tests/dev-ptmx

@@ -12,7 +12,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	exit 1
 fi
 if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	adduser --gecos user --disabled-password user
+	useradd --home-dir /home/user --create-home user
 fi
 prefix=
 [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"
@@ -116,9 +116,9 @@ END
 
 # use script to create a fake tty
 # run all tests as root and as a normal user (the latter requires ptmxmode=666)
-script -qfc "$prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
-	--include=gcc,libc6-dev,python3,adduser \
-	--customize-hook='chroot \"\$1\" adduser --gecos user --disabled-password user' \
+script -qfec "$prefix {{ CMD }} --mode={{ MODE }} --variant=apt \
+	--include=gcc,libc6-dev,python3,passwd \
+	--customize-hook='chroot \"\$1\" useradd --home-dir /home/user --create-home user' \
 	--customize-hook='chroot \"\$1\" python3 -c \"import pty; print(pty.openpty())\"' \
 	--customize-hook='chroot \"\$1\" runuser -u user -- python3 -c \"import pty; print(pty.openpty())\"' \
 	--customize-hook='chroot \"\$1\" script -c \"echo foobar\"' \

tests/fail-without-etc-subuid

@@ -5,8 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+useradd --home-dir /home/user --create-home user
 rm /etc/subuid
 ret=0
 runuser -u user -- {{ CMD }} --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot {{ MIRROR }} || ret=$?

tests/fail-without-username-in-etc-subuid

@@ -5,8 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+useradd --home-dir /home/user --create-home user
 awk -F: '$1!="user"' /etc/subuid > /etc/subuid.tmp
 mv /etc/subuid.tmp /etc/subuid
 ret=0

tests/file-mirror-automount-hook

@@ -6,7 +6,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	exit 1
 fi
 if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	adduser --gecos user --disabled-password user
+	useradd --home-dir /home/user --create-home user
 fi
 prefix=
 [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"

tests/install-doc-debian

@@ -6,7 +6,7 @@ if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
 		echo "this test modifies the system and should only be run inside a container" >&2
 		exit 1
 	fi
-	adduser --gecos user --disabled-password user
+	useradd --home-dir /home/user --create-home user
 fi
 prefix=
 [ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"

tests/install-doc-debian-and-output-tarball

@@ -7,7 +7,7 @@ if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
 		echo "this test modifies the system and should only be run inside a container" >&2
 		exit 1
 	fi
-	adduser --gecos user --disabled-password user
+	useradd --home-dir /home/user --create-home user
 fi
 prefix=
 [ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"

tests/install-doc-debian-and-test-hooks

@@ -7,7 +7,7 @@ if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
 		echo "this test modifies the system and should only be run inside a container" >&2
 		exit 1
 	fi
-	adduser --gecos user --disabled-password user
+	useradd --home-dir /home/user --create-home user
 fi
 prefix=
 [ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"

tests/install-libmagic-mgc-on-arm64

@@ -6,7 +6,7 @@ if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
 		echo "this test modifies the system and should only be run inside a container" >&2
 		exit 1
 	fi
-	adduser --gecos user --disabled-password user
+	useradd --home-dir /home/user --create-home user
 fi
 prefix=
 [ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"

tests/jessie-or-older

@@ -5,8 +5,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+useradd --home-dir /home/user --create-home user
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
 filter() {

tests/merged-fakechroot-inside-unmerged-chroot

@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# make sure that the $FAKECHROOT_CMD_SUBST environment variable is set up
+# such that one can create a merged-/usr chroot from an unmerged-/usr system
+
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+trap "rm -f /tmp/chroot-fakechroot.tar /tmp/chroot-root.tar" EXIT INT TERM
+[ "$(whoami)" = "root" ]
+{{ CMD }} --mode=root --variant=apt --hook-dir=./hooks/merged-usr {{ DIST }} /tmp/chroot-root.tar {{ MIRROR }}
+cat << 'SCRIPT' > script.sh
+#!/bin/sh
+set -exu
+rootfs="$1"
+mkdir -p "$rootfs/mnt/hooks"
+[ -e /usr/libexec/mmdebstrap/ldconfig.fakechroot ] && cp -a /usr/libexec/mmdebstrap/ldconfig.fakechroot "$rootfs/mnt"
+[ -e ./ldconfig.fakechroot ] && cp -a ./ldconfig.fakechroot "$rootfs/mnt"
+[ -e /usr/share/mmdebstrap/hooks/merged-usr ] && cp -a /usr/share/mmdebstrap/hooks/merged-usr "$rootfs/mnt/hooks"
+[ -e ./hooks/merged-usr ] && cp -a ./hooks/merged-usr "$rootfs/mnt/hooks"
+[ -e /usr/bin/mmdebstrap ] && cp -aT /usr/bin/mmdebstrap "$rootfs/usr/bin/mmdebstrap"
+[ -e ./mmdebstrap ] && cp -aT ./mmdebstrap "$rootfs/mnt/mmdebstrap"
+chroot "$rootfs" env --chdir=/mnt \
+	runuser -u user -- \
+	{{ CMD }} --mode=fakechroot --variant=apt \
+	--hook-dir=./hooks/merged-usr \
+	--customize-hook='chroot "$1" echo "$FAKECHROOT_CMD_SUBST" | tr ":" "\n" | sort' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(readlink /bin)\" = usr/bin"' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(realpath -e /bin/ldd)\" = /usr/bin/ldd"' \
+	--customize-hook='chroot "$1" echo ":$FAKECHROOT_CMD_SUBST" | grep --quiet :/usr/bin/ldd=' \
+	--customize-hook='chroot "$1" echo ":$FAKECHROOT_CMD_SUBST" | grep --quiet :/bin/ldd=' \
+	--customize-hook='chroot "$1" env PATH=/bin ldd /bin/true 2>&1 | grep --quiet "fakeldd: objdump: command not found: install binutils package"' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(readlink /sbin)\" = usr/sbin"' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(realpath -e /sbin/ldconfig)\" = /usr/sbin/ldconfig"' \
+	--customize-hook='chroot "$1" echo ":$FAKECHROOT_CMD_SUBST" | grep --quiet :/usr/sbin/ldconfig=' \
+	--customize-hook='chroot "$1" echo ":$FAKECHROOT_CMD_SUBST" | grep --quiet :/sbin/ldconfig=' \
+	--customize-hook='chroot "$1" env PATH=/sbin ldconfig 2>&1 | grep --quiet "/usr/bin/env: ‘python3’: No such file or directory"' \
+	{{ DIST }} /tmp/chroot-fakechroot.tar {{ MIRROR }}
+SCRIPT
+chmod +x script.sh
+{{ CMD }} --mode=root --variant=apt --include=perl,python3,passwd,fakeroot,fakechroot \
+	--hook-dir=./hooks/no-merged-usr \
+	--customize-hook='chroot "$1" useradd --home-dir /home/user --create-home user' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(realpath -e /usr/bin/ldd)\" = /usr/bin/ldd"' \
+	--customize-hook='chroot "$1" sh -c "exec test ! -e /usr/sbin/ldconfig"' \
+	--customize-hook=./script.sh \
+	--customize-hook="copy-out /tmp/chroot-fakechroot.tar /tmp" \
+	{{ DIST }} /dev/null {{ MIRROR }}
+cmp /tmp/chroot-fakechroot.tar /tmp/chroot-root.tar

tests/missing-dev-sys-proc-inside-the-chroot

@@ -5,6 +5,5 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	echo "this test modifies the system and should only be run inside a container" >&2
 	exit 1
 fi
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+useradd --home-dir /home/user --create-home user
 runuser -u user -- {{ CMD }} --mode=unshare --variant=custom --include=dpkg,dash,diffutils,coreutils,libc-bin,sed {{ DIST }} /dev/null {{ MIRROR }}

tests/missing-device-nodes-outside-the-chroot

@@ -6,8 +6,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	exit 1
 fi
 rm /dev/console
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+useradd --home-dir /home/user --create-home user
 runuser -u user -- {{ CMD }} --mode=unshare --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar

tests/no-sbin-in-path

@@ -14,7 +14,7 @@ if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
 		echo "this test modifies the system and should only be run inside a container" >&2
 		exit 1
 	fi
-	adduser --gecos user --disabled-password user
+	useradd --home-dir /home/user --create-home user
 fi
 prefix=
 [ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"

tests/pivot_root

@@ -10,7 +10,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 fi
 
 if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	adduser --gecos user --disabled-password user
+	useradd --home-dir /home/user --create-home user
 fi
 
 prefix=

tests/progress-bars-on-fake-tty

@@ -2,5 +2,5 @@
 set -eu
 export LC_ALL=C.UTF-8
 trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
-script -qfc "{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}" /dev/null
+script -qfec "{{ CMD }} --mode={{ MODE }} --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}" /dev/null
 tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/root-mode-inside-unshare-chroot

@@ -10,8 +10,7 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	exit 1
 fi
 [ "$(whoami)" = "root" ]
-adduser --gecos user --disabled-password user
-sysctl -w kernel.unprivileged_userns_clone=1
+useradd --home-dir /home/user --create-home user
 cat << 'SCRIPT' > script.sh
 #!/bin/sh
 set -eu

tests/special-hooks-with-mode-mode

@@ -6,14 +6,7 @@ if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
 		echo "this test modifies the system and should only be run inside a container" >&2
 		exit 1
 	fi
-	adduser --gecos user --disabled-password user
-fi
-if [ "{{ MODE }}" = unshare ]; then
-	if [ ! -e /mmdebstrap-testenv ]; then
-		echo "this test modifies the system and should only be run inside a container" >&2
-		exit 1
-	fi
-	sysctl -w kernel.unprivileged_userns_clone=1
+	useradd --home-dir /home/user --create-home user
 fi
 prefix=
 [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"

tests/tarfilter-idshift

@@ -6,10 +6,9 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	exit 1
 fi
 trap "rm -f /tmp/debian-chroot.tar /tmp/debian-chroot-shifted.tar /tmp/debian-chroot.txt /tmp/debian-chroot-shiftedback.tar /tmp/expected; rm -rf /tmp/debian-chroot" EXIT INT TERM
-adduser --gecos user --disabled-password user
+useradd --home-dir /home/user --create-home user
 echo user:100000:65536 | cmp /etc/subuid -
 echo user:100000:65536 | cmp /etc/subgid -
-sysctl -w kernel.unprivileged_userns_clone=1
 # include iputils-ping so that we can verify that tarfilter does not remove
 # extended attributes
 # run through tarshift no-op to create a tarball that should be bit-by-bit

tests/unpack-doc-debian

@@ -6,14 +6,7 @@ if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
 		echo "this test modifies the system and should only be run inside a container" >&2
 		exit 1
 	fi
-	adduser --gecos user --disabled-password user
-fi
-if [ "{{ MODE }}" = unshare ]; then
-	if [ ! -e /mmdebstrap-testenv ]; then
-		echo "this test modifies the system and should only be run inside a container" >&2
-		exit 1
-	fi
-	sysctl -w kernel.unprivileged_userns_clone=1
+	useradd --home-dir /home/user --create-home user
 fi
 prefix=
 [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && prefix="runuser -u user --"

tests/unshare-include-deb

@@ -10,7 +10,7 @@ if [ "$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
 		echo "this test modifies the system and should only be run inside a container" >&2
 		exit 1
 	fi
-	adduser --gecos user --disabled-password user
+	useradd --home-dir /home/user --create-home user
 fi
 [ "$(id -u)" -eq 0 ] && prefix="runuser -u user --"