document maybe-merged-usr hook

Thanks: Helmut Grohne for finding this issue and help interating this

coverage.py

@@ -132,6 +132,30 @@ def format_test(num, total, name, dist, mode, variant, fmt, config_dict):
     return ret
 
 
+def print_time_per_test(time_per_test, name="test"):
+    print(
+        f"average time per {name}:",
+        sum(time_per_test.values(), start=timedelta()) / len(time_per_test),
+        file=sys.stderr,
+    )
+    print(
+        f"median time per {name}:",
+        sorted(time_per_test.values())[len(time_per_test) // 2],
+        file=sys.stderr,
+    )
+    head_tail_num = 10
+    print(f"{head_tail_num} fastests {name}s:", file=sys.stderr)
+    for k, v in sorted(time_per_test.items(), key=lambda i: i[1])[
+        : min(head_tail_num, len(time_per_test))
+    ]:
+        print(f"    {k}: {v}", file=sys.stderr)
+    print(f"{head_tail_num} slowest {name}s:", file=sys.stderr)
+    for k, v in sorted(time_per_test.items(), key=lambda i: i[1], reverse=True)[
+        : min(head_tail_num, len(time_per_test))
+    ]:
+        print(f"    {k}: {v}", file=sys.stderr)
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("test", nargs="*", help="only run these tests")
@@ -288,6 +312,7 @@ def main():
     num_success = 0
     num_finished = 0
     time_per_test = {}
+    acc_time_per_test = defaultdict(list)
     for i, (test, name, dist, mode, variant, fmt) in enumerate(tests):
         if torun and i not in torun:
             continue
@@ -380,6 +405,7 @@ def main():
             i + 1, len(tests), name, dist, mode, variant, fmt, config_dict
         )
         time_per_test[formated_test_name] = walltime
+        acc_time_per_test[name].append(walltime)
         print(separator, file=sys.stderr)
         print(f"duration: {walltime}", file=sys.stderr)
         if proc.returncode != 0 or shellcheck != "":
@@ -409,27 +435,15 @@ def main():
             print(f, file=sys.stderr)
         exit(1)
     if len(time_per_test) > 1:
-        print(
+        print_time_per_test(time_per_test)
-            "average time per test:",
+    if len(acc_time_per_test) > 1:
-            sum(time_per_test.values(), start=timedelta()) / len(time_per_test),
+        print_time_per_test(
-            file=sys.stderr,
+            {
+                f"{len(v)}x {k}": sum(v, start=timedelta())
+                for k, v in acc_time_per_test.items()
+            },
+            "accumulated test",
         )
-        print(
-            "median time per test:",
-            sorted(time_per_test.values())[len(time_per_test) // 2],
-            file=sys.stderr,
-        )
-        head_tail_num = 10
-        print(f"{head_tail_num} fastests tests:", file=sys.stderr)
-        for k, v in sorted(time_per_test.items(), key=lambda i: i[1])[
-            : min(head_tail_num, len(time_per_test))
-        ]:
-            print(f"    {k}: {v}", file=sys.stderr)
-        print(f"{head_tail_num} slowest tests:", file=sys.stderr)
-        for k, v in sorted(time_per_test.items(), key=lambda i: i[1], reverse=True)[
-            : min(head_tail_num, len(time_per_test))
-        ]:
-            print(f"    {k}: {v}", file=sys.stderr)
 
 
 if __name__ == "__main__":

coverage.txt

@@ -364,3 +364,6 @@ Test: apt-patterns
 Test: apt-patterns-custom
 
 Test: empty-sources.list
+
+Test: merged-fakechroot-inside-unmerged-chroot
+Needs-Root: true

ldconfig.fakechroot

@@ -106,10 +106,24 @@ def main():
     for d in get_libdirs(chroot, [chroot / "etc" / "ld.so.conf"]):
         make_relative(d)
 
+    rootarg = chroot
+    argv = sys.argv[1:]
+    for arg in sys.argv[1:]:
+        if arg == "-r":
+            rootarg = None
+        elif rootarg is None:
+            argpath = Path(arg)
+            if argpath.is_absolute():
+                rootarg = chroot / argpath.relative_to("/")
+            else:
+                rootarg = Path.cwd() / argpath
+    if rootarg is None:
+        rootarg = chroot
+
     # we add any additional arguments before "-r" such that any other "-r"
     # option will be overwritten by the one we set
     subprocess.check_call(
-        [chroot / "sbin" / "ldconfig"] + sys.argv[1:] + ["-r", chroot]
+        [chroot / "sbin" / "ldconfig"] + sys.argv[1:] + ["-r", rootarg]
     )
 
 

make_mirror.sh

@@ -264,7 +264,7 @@ END
 			--or --field=Priority important --or --field=Priority standard \
 			\))
 
-	pkgs="$pkgs build-essential busybox gpg eatmydata"
+	pkgs="$pkgs build-essential busybox gpg eatmydata fakechroot fakeroot"
 
 	# we need usr-is-merged to simulate debootstrap behaviour for all dists
 	# starting from Debian 12 (Bullseye)

mmdebstrap

@@ -2251,10 +2251,21 @@ sub run_setup() {
         # root mode when the path of the chroot is not in a world-readable
         # location.
         my $partial = '/var/lib/apt/lists/partial';
-        if (
+        my @testcmd = (
-            system('/usr/lib/apt/apt-helper', 'drop-privs', '--', 'test',
+            '/usr/lib/apt/apt-helper', 'drop-privs', '--', 'test',
-                '-r', "$options->{root}$partial") != 0
+            '-r', "$options->{root}$partial"
-        ) {
+        );
+        my $pid = fork() // error "fork() failed: $!";
+        if ($pid == 0) {
+            open(STDOUT, '>', '/dev/null')
+              or error "cannot open /dev/null for writing: $!";
+            open(STDERR, '>', '/dev/null')
+              or error "cannot open /dev/null for writing: $!";
+            exec { $testcmd[0] } @testcmd
+              or error("cannot exec " . (join " ", @testcmd) . ": $!");
+        }
+        waitpid $pid, 0;
+        if ($? != 0) {
             warning "Download is performed unsandboxed as root as file"
               . " $options->{root}$partial couldn't be accessed by user _apt";
             open my $fh, '>>', $tmpfile
@@ -2602,20 +2613,30 @@ sub run_prepare {
                 $subst{ldconfig}
                   = '/usr/libexec/mmdebstrap/ldconfig.fakechroot';
             }
-            my @fakechrootsubst = ();
+            my %mergedusrmap = (
+                "/bin"      => "/usr/bin",
+                "/sbin"     => "/usr/sbin",
+                "/usr/bin/" => "/bin",
+                "/usr/sbin" => "/sbin"
+            );
+            my %fakechrootsubst;
             foreach my $d (split ':', $ENV{PATH}) {
-                foreach my $k (sort keys %subst) {
+                foreach my $k (sort %subst) {
-                    if (-e "$d/$k") {
+                    my $mapped_path = $mergedusrmap{$d} // $d;
-                        push @fakechrootsubst, "$d/$k=$subst{$k}";
+                    next if !-e "$d/$k" && !-e "$mapped_path/$k";
-                    }
+                    $fakechrootsubst{"$d/$k=$subst{$k}"}           = 1;
+                    $fakechrootsubst{"$mapped_path/$k=$subst{$k}"} = 1;
                 }
             }
             if (defined $ENV{FAKECHROOT_CMD_SUBST}
                 && $ENV{FAKECHROOT_CMD_SUBST} ne "") {
-                push @fakechrootsubst, split /:/, $ENV{FAKECHROOT_CMD_SUBST};
+                foreach my $e (split /:/, $ENV{FAKECHROOT_CMD_SUBST}) {
+                    $fakechrootsubst{$e} = 1;
+                }
             }
             ## no critic (Variables::RequireLocalizedPunctuationVars)
-            $ENV{FAKECHROOT_CMD_SUBST} = join ':', @fakechrootsubst;
+            $ENV{FAKECHROOT_CMD_SUBST} = join ':',
+              (sort keys %fakechrootsubst);
         }
         if (defined $ENV{FAKECHROOT_EXCLUDE_PATH}
             && $ENV{FAKECHROOT_EXCLUDE_PATH} ne "") {
@@ -7459,6 +7480,15 @@ F</etc/unsupported-skip-usrmerge-conversion> inside the chroot and install the
 B<usr-is-merged> package to avoid the installation of the B<usrmerge> package
 and its dependencies.
 
+If you are using B<mmdebstrap> in a setup where you do not know upfront whether
+the chroot you are creating should be merged-/usr or not and you want to avoid
+installation of the B<usrmerge> package and it's dependencies, you can use:
+
+    --hook-dir=/usr/share/mmdebstrap/hooks/maybe-merged-usr
+
+That hook will use the availability of the B<usr-is-merged> package to decide
+whether to call the B<merged-usr> hook or not.
+
 =head1 COMPRESSION
 
 B<mmdebstrap> will choose a suitable compressor for the output tarball

tests/merged-fakechroot-inside-unmerged-chroot

@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# make sure that the $FAKECHROOT_CMD_SUBST environment variable is set up
+# such that one can create a merged-/usr chroot from an unmerged-/usr system
+
+set -eu
+export LC_ALL=C.UTF-8
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
+trap "rm -f /tmp/chroot-fakechroot.tar /tmp/chroot-root.tar" EXIT INT TERM
+[ "$(whoami)" = "root" ]
+{{ CMD }} --mode=root --variant=apt --hook-dir=./hooks/merged-usr {{ DIST }} /tmp/chroot-root.tar {{ MIRROR }}
+cat << 'SCRIPT' > script.sh
+#!/bin/sh
+set -exu
+rootfs="$1"
+mkdir -p "$rootfs/mnt/hooks"
+[ -e /usr/libexec/mmdebstrap/ldconfig.fakechroot ] && cp -a /usr/libexec/mmdebstrap/ldconfig.fakechroot "$rootfs/mnt"
+[ -e ./ldconfig.fakechroot ] && cp -a ./ldconfig.fakechroot "$rootfs/mnt"
+[ -e /usr/share/mmdebstrap/hooks/merged-usr ] && cp -a /usr/share/mmdebstrap/hooks/merged-usr "$rootfs/mnt/hooks"
+[ -e ./hooks/merged-usr ] && cp -a ./hooks/merged-usr "$rootfs/mnt/hooks"
+[ -e /usr/bin/mmdebstrap ] && cp -aT /usr/bin/mmdebstrap "$rootfs/usr/bin/mmdebstrap"
+[ -e ./mmdebstrap ] && cp -aT ./mmdebstrap "$rootfs/mnt/mmdebstrap"
+chroot "$rootfs" env --chdir=/mnt \
+	runuser -u user -- \
+	{{ CMD }} --mode=fakechroot --variant=apt \
+	--hook-dir=./hooks/merged-usr \
+	--customize-hook='chroot "$1" echo "$FAKECHROOT_CMD_SUBST" | tr ":" "\n" | sort' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(readlink /bin)\" = usr/bin"' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(realpath -e /bin/ldd)\" = /usr/bin/ldd"' \
+	--customize-hook='chroot "$1" echo ":$FAKECHROOT_CMD_SUBST" | grep --quiet :/usr/bin/ldd=' \
+	--customize-hook='chroot "$1" echo ":$FAKECHROOT_CMD_SUBST" | grep --quiet :/bin/ldd=' \
+	--customize-hook='chroot "$1" env PATH=/bin ldd /bin/true 2>&1 | grep --quiet "fakeldd: objdump: command not found: install binutils package"' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(readlink /sbin)\" = usr/sbin"' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(realpath -e /sbin/ldconfig)\" = /usr/sbin/ldconfig"' \
+	--customize-hook='chroot "$1" echo ":$FAKECHROOT_CMD_SUBST" | grep --quiet :/usr/sbin/ldconfig=' \
+	--customize-hook='chroot "$1" echo ":$FAKECHROOT_CMD_SUBST" | grep --quiet :/sbin/ldconfig=' \
+	--customize-hook='chroot "$1" env PATH=/sbin ldconfig 2>&1 | grep --quiet "/usr/bin/env: ‘python3’: No such file or directory"' \
+	{{ DIST }} /tmp/chroot-fakechroot.tar {{ MIRROR }}
+SCRIPT
+chmod +x script.sh
+{{ CMD }} --mode=root --variant=apt --include=perl,python3,adduser,fakeroot,fakechroot \
+	--hook-dir=./hooks/no-merged-usr \
+	--customize-hook='chroot "$1" adduser --gecos user --disabled-password user' \
+	--customize-hook='chroot "$1" sh -c "exec test \"\$(realpath -e /usr/bin/ldd)\" = /usr/bin/ldd"' \
+	--customize-hook='chroot "$1" sh -c "exec test ! -e /usr/sbin/ldconfig"' \
+	--customize-hook=./script.sh \
+	--customize-hook="copy-out /tmp/chroot-fakechroot.tar /tmp" \
+	{{ DIST }} /dev/null {{ MIRROR }}
+cmp /tmp/chroot-fakechroot.tar /tmp/chroot-root.tar