add hooks/maybe-jessie-or-older and hooks/maybe-merged-usr

coverage.txt

@@ -307,7 +307,7 @@ Variants: essential
 Modes: chrootless
 Skip-If:
  dist in ["oldstable", "stable"]
- hostarch != "amd64"
+ hostarch not in ["amd64", "arm64"]
  not run_ma_same_tests
 Needs-QEMU: true
 

hooks/maybe-jessie-or-older/extract00.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -eu
+
+# if dpkg is new enough, do nothing
+# we cannot ask dpkg-query about the version because dpkg is only extracted
+# but not installed at this point
+dpkg_ver="$(chroot "$1" dpkg --version | grep --extended-regexp --only-matching '[0-9]+\.[0-9.]+')"
+if dpkg --compare-versions "$dpkg_ver" ge 1.17.11; then
+	echo "dpkg version $dpkg_ver is >= 1.17.11 -- not running jessie-or-older extract00 hook" >&2
+	exit 0
+else
+	echo "dpkg version $dpkg_ver is << 1.17.11 -- running jessie-or-older extract00 hook" >&2
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/jessie-or-older/extract00.sh" ] && [ -x "$p/jessie-or-older/extract01.sh" ]; then
+		"$p/jessie-or-older/extract00.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find jessie-or-older hook anywhere" >&2
+exit 1

hooks/maybe-jessie-or-older/extract01.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -eu
+
+# If the init package has not been extracted, then it is not part of the
+# Essential:yes set and we do not need this workaround. This holds true for the
+# init package version 1.34 and later. Instead of asking apt about the init
+# version (which might not be the same version that was picked to be installed)
+# we check for the presence of the init package by checking whether
+# /usr/share/doc/init/copyright exists.
+
+if [ -e "$1/usr/share/doc/init/copyright" ]; then
+	echo "the init package is not Essential:yes -- not running jessie-or-older extract01 hook" >&2
+	exit 0
+else
+	echo "the init package is Essential:yes -- running jessie-or-older extract01 hook" >&2
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/jessie-or-older/extract00.sh" ] && [ -x "$p/jessie-or-older/extract01.sh" ]; then
+		"$p/jessie-or-older/extract01.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find jessie-or-older hook anywhere" >&2
+exit 1

hooks/maybe-merged-usr/essential00.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -eu
+
+# if the usr-is-merged package cannot be installed with apt, do nothing
+if ! env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
+	echo "no package called usr-is-merged found -- not running merged-usr essential hook" >&2
+	exit 0
+else
+	echo "package usr-is-merged found -- running merged-usr essential hook" >&2
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
+		"$p/merged-usr/essential00.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find merged-usr hook anywhere" >&2
+exit 1

hooks/maybe-merged-usr/setup00.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -eu
+
+env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get update --error-on=any
+
+# if the usr-is-merged package cannot be installed with apt, do nothing
+if ! env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
+	echo "no package called usr-is-merged found -- not running merged-usr setup hook" >&2
+	exit 0
+else
+	echo "package usr-is-merged found -- running merged-usr setup hook" >&2
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
+		"$p/merged-usr/setup00.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find merged-usr hook anywhere" >&2
+exit 1

mmdebstrap

@@ -5213,9 +5213,13 @@ sub main() {
                     $signedby = " [signed-by=\"$keyring\"]";
                     last;
                 }
-                {
-                    open(my $fh, '-|', @gpgcmd, '--with-colons', '--show-keys',
-                        @keyrings) // error "failed to fork(): $!";
+                info "Finding correct signed-by value...";
+                my $progress = 0.0;
+                print_progress($progress);
+                for (my $i = 0 ; $i < scalar @keyrings ; $i++) {
+                    my $k = $keyrings[$i];
+                    open(my $fh, '-|', @gpgcmd, '--with-colons',
+                        '--show-keys', $k) // error "failed to fork(): $!";
                     while (my $line = <$fh>) {
                         if ($line !~ /^fpr:::::::::([^:]+):/) {
                             next;
@@ -5223,10 +5227,12 @@ sub main() {
                         push @aptfingerprints, $1;
                     }
                     close $fh;
+                    if ($? != 0) {
+                        warning("gpg failed to read $k");
+                    }
+                    print_progress($i / (scalar @keyrings) * 100.0, undef);
                 }
-                if ($? != 0) {
-                    error "gpg failed";
-                }
+                print_progress("done");
                 if (scalar @aptfingerprints == 0) {
                     $signedby = " [signed-by=\"$keyring\"]";
                     last;
@@ -6233,7 +6239,12 @@ I<TARGET> is C<-> or if no I<TARGET> was specified.
 =head1 OPTIONS
 
 Options are case insensitive. Short options may be bundled. Long options
-require a double dash and may be abbreviated to uniqueness.
+require a double dash and may be abbreviated to uniqueness. Options can be
+placed anywhere on the command line, even before or mixed with the I<SUITE>,
+I<TARGET>, and I<MIRROR> arguments. A double dash C<--> can be used to stop
+interpreting command line arguments as options to allow I<SUITE>, I<TARGET> and
+I<MIRROR> arguments that start with a single or double dash. Option order only
+matters for options that can be passed multiple times as documented below.
 
 =over 8
 

run_qemu.sh

@@ -11,6 +11,9 @@ cleanup() {
 	rm -f "$tmpdir/debian-$DEFAULT_DIST-overlay.qcow"
 	rm -f "$tmpdir/log"
 	[ -e "$tmpdir" ] && rmdir "$tmpdir"
+	if [ -n "${TAIL_PID:-}" ]; then
+		kill "$TAIL_PID"
+	fi
 	if [ -e shared/output.txt ]; then
 		res="$(cat shared/exitstatus.txt)"
 		if [ "$res" != "0" ]; then
@@ -48,6 +51,8 @@ if [ -e shared/output.txt ]; then
 	rm shared/output.txt
 fi
 touch shared/output.txt
+tail -f shared/output.txt &
+TAIL_PID=$!
 
 # the path to debian-$DEFAULT_DIST.qcow must be absolute or otherwise qemu will
 # look for the path relative to debian-$DEFAULT_DIST-overlay.qcow

tests/as-debootstrap-unshare-wrapper

@@ -69,7 +69,7 @@ if [ -e /tmp/debian-debootstrap/etc/apparmor.d/local/sbin.dhclient ] && [ ! -s /
 fi
 
 # check if the file content differs
-diff --no-dereference --recursive /tmp/debian-debootstrap /tmp/debian-mm >&2
+diff --unified --no-dereference --recursive /tmp/debian-debootstrap /tmp/debian-mm >&2
 
 # check permissions, ownership, symlink targets, modification times using tar
 # mtimes of directories created by mmdebstrap will differ, thus we equalize them first

tests/chrootless

@@ -2,14 +2,10 @@
 set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-if dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libpam-runtime)" le 1.5.2-5; then
-	# https://bugs.debian.org/1022952
-	exit 0
-fi
 trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
 # we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
 # DPKG_ROOT
-for INCLUDE in '' 'systemd-sysv'; do
+for INCLUDE in '' 'apt' 'apt,build-essential' 'systemd-sysv'; do
 	for MODE in root chrootless; do
 		{{ CMD }} --mode=$MODE --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
 			${INCLUDE:+--include="$INCLUDE"} \

tests/chrootless-fakeroot

@@ -16,7 +16,7 @@ prefix=
 # DPKG_ROOT
 # permissions drwxr-sr-x and extended attributes of ./var/log/journal/ cannot
 # be preserved under fakeroot
-for INCLUDE in '' 'systemd-sysv'; do
+for INCLUDE in '' 'apt' 'apt,build-essential' 'systemd-sysv'; do
 	{{ CMD }} --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
 		--customize-hook='if [ -d "$1"/var/log/journal ]; then rmdir "$1"/var/log/journal; mkdir --mode=2755 "$1"/var/log/journal; chroot "$1" chown root:systemd-journal /var/log/journal; fi' \
 		${INCLUDE:+--include="$INCLUDE"} \

tests/chrootless-foreign

@@ -8,20 +8,42 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	exit 1
 fi
 
+deb2qemu() {
+	case "$1" in
+		amd64) echo x86_64;;
+		arm64) echo aarch64;;
+		armel|armhf) echo arm;;
+		ppc64el) echo ppc64le;;
+		*) echo "$1";;
+	esac
+}
+if [ "$(dpkg --print-architecture)" = "arm64" ]; then
+	arch=amd64
+else
+	arch=arm64
+fi
+
 [ "$(id -u)" -eq 0 ]
-[ -e /proc/sys/fs/binfmt_misc/qemu-aarch64 ]
+[ -e "/proc/sys/fs/binfmt_misc/qemu-$(deb2qemu "$arch")" ]
+
 
 # we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
 # DPKG_ROOT
-for INCLUDE in '' 'systemd-sysv'; do
-	echo 1 > /proc/sys/fs/binfmt_misc/qemu-aarch64
-	arch-test arm64
-	{{ CMD }} --mode=root --architecture=arm64 --variant={{ VARIANT }} \
+#
+# dpkg is unable to install architecture arch:all packages with a
+# dependency on an arch:any package (perl-modules-5.34 in this case)
+# inside foreign architecture chrootless chroots, because dpkg will use
+# its own architecture as the native architecture, see #825385 and #1020533
+# So we are not testing the installation of apt,build-essential here.
+for INCLUDE in '' 'apt' 'systemd-sysv'; do
+	echo 1 > "/proc/sys/fs/binfmt_misc/qemu-$(deb2qemu "$arch")"
+	arch-test "$arch"
+	{{ CMD }} --mode=root --architecture="$arch" --variant={{ VARIANT }} \
 		--hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
 		{{ DIST }} "/tmp/root.tar" {{ MIRROR }}
-	echo 0 > /proc/sys/fs/binfmt_misc/qemu-aarch64
-	arch-test arm64 && exit 1
-	{{ CMD }} --mode=chrootless --architecture=arm64 --variant={{ VARIANT }} \
+	echo 0 > "/proc/sys/fs/binfmt_misc/qemu-$(deb2qemu "$arch")"
+	arch-test "$arch" && exit 1
+	{{ CMD }} --mode=chrootless --architecture="$arch" --variant={{ VARIANT }} \
 		--hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
 		{{ DIST }} "/tmp/chrootless.tar" {{ MIRROR }}
 	# when creating a foreign architecture chroot, the tarballs are not

tests/logfile

@@ -7,6 +7,7 @@ export LC_ALL=C.UTF-8
 head --lines=-1 /tmp/log > /tmp/trimmed
 cat << LOG | diff -u - /tmp/trimmed
 I: chroot architecture {{ HOSTARCH }} is equal to the host's architecture
+I: Finding correct signed-by value...
 I: automatically chosen format: directory
 I: running apt-get update...
 I: downloading packages with apt...

tests/multiple-include

@@ -9,7 +9,6 @@ rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
 rm /tmp/debian-chroot/usr/share/lintian/overrides/tzdata
 rm /tmp/debian-chroot/etc/localtime
 rm /tmp/debian-chroot/etc/timezone
-rm /tmp/debian-chroot/usr/sbin/tzconfig
 rm -r /tmp/debian-chroot/usr/share/doc/tzdata
 rm -r /tmp/debian-chroot/usr/share/zoneinfo
 rm /tmp/debian-chroot/var/lib/apt/extended_states