release 1.3.0

Closes: #1028977

CHANGELOG.md

@@ -1,3 +1,18 @@
+1.3.0 (2023-01-16)
+------------------
+
+ - add hooks/maybe-jessie-or-older and hooks/maybe-merged-usr
+ - add --skip=check/signed-by
+ - hooks/jessie-or-older: split into two individual hook files
+ - skip running apt-get update if we are very sure that it was already run
+ - be more verbose when 'apt-get update' failed
+ - warn if a hook is named like one but not executable and if a hook is
+   executable but not named like one
+ - to find signed-by value, run gpg on the individual keys to print better
+   error messages in case it fails (gpg doesn't give an indication which file
+   it was unable to read) and print progress bar
+ - allow empty sources.list entries
+
 1.2.5 (2023-01-04)
 ------------------
 

coverage.py

@@ -119,7 +119,7 @@ def parse_config(confname):
     return config_order, config_dict
 
 
-def format_failed(num, total, name, dist, mode, variant, fmt, config_dict):
+def format_test(num, total, name, dist, mode, variant, fmt, config_dict):
     ret = f"({num}/{total}) {name}"
     if len(config_dict[name].get("Dists", [])) > 1:
         ret += f" --dist={dist}"
@@ -287,6 +287,7 @@ def main():
     failed = []
     num_success = 0
     num_finished = 0
+    time_per_test = {}
     for i, (test, name, dist, mode, variant, fmt) in enumerate(tests):
         if torun and i not in torun:
             continue
@@ -365,6 +366,7 @@ def main():
         if args.format and args.format != fmt:
             print(f"skipping because of --format={args.format}", file=sys.stderr)
             continue
+        before = time.time()
         proc = subprocess.Popen(argv)
         try:
             proc.wait()
@@ -372,21 +374,25 @@ def main():
             proc.terminate()
             proc.wait()
             break
+        after = time.time()
+        walltime = timedelta(seconds=int(after - before))
+        formated_test_name = format_test(
+            i + 1, len(tests), name, dist, mode, variant, fmt, config_dict
+        )
+        time_per_test[formated_test_name] = walltime
         print(separator, file=sys.stderr)
+        print(f"duration: {walltime}", file=sys.stderr)
         if proc.returncode != 0 or shellcheck != "":
             if shellcheck != "":
                 print(shellcheck)
-            failed.append(
+            failed.append(formated_test_name)
-                format_failed(
-                    i + 1, len(tests), name, dist, mode, variant, fmt, config_dict
-                )
-            )
             print("result: FAILURE", file=sys.stderr)
         else:
             print("result: SUCCESS", file=sys.stderr)
             num_success += 1
         if args.maxfail and len(failed) >= args.maxfail:
             break
+    print(separator, file=sys.stderr)
     print(
         "successfully ran %d tests" % num_success,
         file=sys.stderr,
@@ -402,6 +408,28 @@ def main():
         for f in failed:
             print(f, file=sys.stderr)
         exit(1)
+    if len(time_per_test) > 1:
+        print(
+            "average time per test:",
+            sum(time_per_test.values(), start=timedelta()) / len(time_per_test),
+            file=sys.stderr,
+        )
+        print(
+            "median time per test:",
+            sorted(time_per_test.values())[len(time_per_test) // 2],
+            file=sys.stderr,
+        )
+        head_tail_num = 10
+        print(f"{head_tail_num} fastests tests:", file=sys.stderr)
+        for k, v in sorted(time_per_test.items(), key=lambda i: i[1])[
+            : min(head_tail_num, len(time_per_test))
+        ]:
+            print(f"    {k}: {v}", file=sys.stderr)
+        print(f"{head_tail_num} slowest tests:", file=sys.stderr)
+        for k, v in sorted(time_per_test.items(), key=lambda i: i[1], reverse=True)[
+            : min(head_tail_num, len(time_per_test))
+        ]:
+            print(f"    {k}: {v}", file=sys.stderr)
 
 
 if __name__ == "__main__":

coverage.txt

@@ -307,7 +307,7 @@ Variants: essential
 Modes: chrootless
 Skip-If:
  dist in ["oldstable", "stable"]
- hostarch != "amd64"
+ hostarch not in ["amd64", "arm64"]
  not run_ma_same_tests
 Needs-QEMU: true
 
@@ -358,3 +358,9 @@ Needs-QEMU: true
 Test: jessie-or-older
 Needs-QEMU: true
 Variants: essential apt minbase
+
+Test: apt-patterns
+
+Test: apt-patterns-custom
+
+Test: empty-sources.list

hooks/jessie-or-older/extract00.sh

@@ -8,41 +8,9 @@ fi
 
 TARGET="$1"
 
+# not needed since dpkg 1.17.11
 for f in available diversions cmethopt; do
 	if [ ! -e "$TARGET/var/lib/dpkg/$f" ]; then
 		touch "$TARGET/var/lib/dpkg/$f"
 	fi
 done
-
-if [ -z "${MMDEBSTRAP_ESSENTIAL+x}" ]; then
-	MMDEBSTRAP_ESSENTIAL=
-	for f in "$TARGET/var/cache/apt/archives/"*.deb; do
-		[ -f "$f" ] || continue
-		f="${f#"$TARGET"}"
-		MMDEBSTRAP_ESSENTIAL="$MMDEBSTRAP_ESSENTIAL $f"
-	done
-fi
-
-fname_base_passwd=
-fname_base_files=
-fname_dpkg=
-for pkg in $MMDEBSTRAP_ESSENTIAL; do
-	pkgname=$(dpkg-deb --show --showformat='${Package}' "$TARGET/$pkg")
-	# shellcheck disable=SC2034
-	case $pkgname in
-		base-passwd) fname_base_passwd=$pkg;;
-		base-files)  fname_base_files=$pkg;;
-		dpkg)        fname_dpkg=$pkg;;
-	esac
-done
-
-for var in base_passwd base_files dpkg; do
-	eval 'val=$fname_'"$var"
-	[ -z "$val" ] && continue
-	chroot "$TARGET" dpkg --install --force-depends "$val"
-done
-
-# shellcheck disable=SC2086
-chroot "$TARGET" dpkg --unpack --force-depends $MMDEBSTRAP_ESSENTIAL
-
-chroot "$TARGET" dpkg --configure --pending

hooks/jessie-or-older/extract01.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# needed until init 1.33 which pre-depends on systemd-sysv
+# starting with init 1.34, init is not Essential:yes anymore
+#
+# jessie has init 1.22
+
+set -eu
+
+if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
+	set -x
+fi
+
+TARGET="$1"
+
+if [ -z "${MMDEBSTRAP_ESSENTIAL+x}" ]; then
+	MMDEBSTRAP_ESSENTIAL=
+	for f in "$TARGET/var/cache/apt/archives/"*.deb; do
+		[ -f "$f" ] || continue
+		f="${f#"$TARGET"}"
+		MMDEBSTRAP_ESSENTIAL="$MMDEBSTRAP_ESSENTIAL $f"
+	done
+fi
+
+fname_base_passwd=
+fname_base_files=
+fname_dpkg=
+for pkg in $MMDEBSTRAP_ESSENTIAL; do
+	pkgname=$(dpkg-deb --show --showformat='${Package}' "$TARGET/$pkg")
+	# shellcheck disable=SC2034
+	case $pkgname in
+		base-passwd) fname_base_passwd=$pkg;;
+		base-files)  fname_base_files=$pkg;;
+		dpkg)        fname_dpkg=$pkg;;
+	esac
+done
+
+for var in base_passwd base_files dpkg; do
+	eval 'val=$fname_'"$var"
+	[ -z "$val" ] && continue
+	chroot "$TARGET" dpkg --install --force-depends "$val"
+done
+
+# shellcheck disable=SC2086
+chroot "$TARGET" dpkg --unpack --force-depends $MMDEBSTRAP_ESSENTIAL
+
+chroot "$TARGET" dpkg --configure --pending

hooks/maybe-jessie-or-older/extract00.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -eu
+
+# if dpkg is new enough, do nothing
+# we cannot ask dpkg-query about the version because dpkg is only extracted
+# but not installed at this point
+dpkg_ver="$(chroot "$1" dpkg --version | grep --extended-regexp --only-matching '[0-9]+\.[0-9.]+')"
+if dpkg --compare-versions "$dpkg_ver" ge 1.17.11; then
+	echo "dpkg version $dpkg_ver is >= 1.17.11 -- not running jessie-or-older extract00 hook" >&2
+	exit 0
+else
+	echo "dpkg version $dpkg_ver is << 1.17.11 -- running jessie-or-older extract00 hook" >&2
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/jessie-or-older/extract00.sh" ] && [ -x "$p/jessie-or-older/extract01.sh" ]; then
+		"$p/jessie-or-older/extract00.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find jessie-or-older hook anywhere" >&2
+exit 1

hooks/maybe-jessie-or-older/extract01.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -eu
+
+# If the init package has not been extracted, then it is not part of the
+# Essential:yes set and we do not need this workaround. This holds true for the
+# init package version 1.34 and later. Instead of asking apt about the init
+# version (which might not be the same version that was picked to be installed)
+# we check for the presence of the init package by checking whether
+# /usr/share/doc/init/copyright exists.
+
+if [ -e "$1/usr/share/doc/init/copyright" ]; then
+	echo "the init package is not Essential:yes -- not running jessie-or-older extract01 hook" >&2
+	exit 0
+else
+	echo "the init package is Essential:yes -- running jessie-or-older extract01 hook" >&2
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/jessie-or-older/extract00.sh" ] && [ -x "$p/jessie-or-older/extract01.sh" ]; then
+		"$p/jessie-or-older/extract01.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find jessie-or-older hook anywhere" >&2
+exit 1

hooks/maybe-merged-usr/essential00.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -eu
+
+# if the usr-is-merged package cannot be installed with apt, do nothing
+if ! env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
+	echo "no package called usr-is-merged found -- not running merged-usr essential hook" >&2
+	exit 0
+else
+	echo "package usr-is-merged found -- running merged-usr essential hook" >&2
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
+		"$p/merged-usr/essential00.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find merged-usr hook anywhere" >&2
+exit 1

hooks/maybe-merged-usr/setup00.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -eu
+
+env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get update --error-on=any
+
+# if the usr-is-merged package cannot be installed with apt, do nothing
+if ! env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
+	echo "no package called usr-is-merged found -- not running merged-usr setup hook" >&2
+	exit 0
+else
+	echo "package usr-is-merged found -- running merged-usr setup hook" >&2
+fi
+
+# resolve the script path using several methods in order:
+#  1. using dirname -- "$0"
+#  2. using ./hooks
+#  3. using /usr/share/mmdebstrap/hooks/
+for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
+	if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
+		"$p/merged-usr/setup00.sh" "$1"
+		exit 0
+	fi
+done
+
+echo "cannot find merged-usr hook anywhere" >&2
+exit 1

hooks/no-merged-usr/essential00.sh

@@ -1,15 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
-        set -x
-fi
-
-TARGET="$1"
-
-APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$TARGET" usr-is-merged
-
-chroot "$TARGET" dpkg-query --showformat '${db:Status-Status}\n' --show usr-is-merged | grep -q '^installed$'
-chroot "$TARGET" dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
-dpkg --compare-versions "1" "lt" "$(chroot "$TARGET" dpkg-query --showformat '${Version}\n' --show usr-is-merged)"

hooks/no-merged-usr/essential00.sh

@@ -0,0 +1 @@
+../merged-usr/essential00.sh

make_mirror.sh

@@ -59,7 +59,9 @@ deletecache() {
 		esac
 	done
 	for f in "$dir/debian-"*.qcow; do
+		if [ -e "$f" ]; then
 			rm --one-file-system "$f"
+		fi
 	done
 	if [ -e "$dir/debian/pool/main" ]; then
 		rm --one-file-system --recursive "$dir/debian/pool/main"
@@ -358,16 +360,22 @@ if [ -e "./shared/cache.A" ] && [ -e "./shared/cache.B" ]; then
 		echo "cache symlink points to $(readlink ./shared/cache)" >&2
 		case "$(readlink ./shared/cache)" in
 			cache.A)
-				echo "maybe rm -r ./shared/cache.B" >&2
+				echo "removing ./shared/cache.B" >&2
+				rm -r ./shared/cache.B
 				;;
 			cache.B)
-				echo "maybe rm -r ./shared/cache.A" >&2
+				echo "removing ./shared/cache.A" >&2
+				rm -r ./shared/cache.A
 				;;
 			*)
 				echo "unexpected" >&2
-		esac
-	fi
 				exit 1
+				;;
+		esac
+	else
+		echo "./shared/cache doesn't exist" >&2
+		exit 1
+	fi
 fi
 
 if [ -e "./shared/cache.A" ]; then
@@ -409,10 +417,11 @@ mkdir -p "$newcachedir"
 touch "$newcachedir/mmdebstrapcache"
 
 HOSTARCH=$(dpkg --print-architecture)
+arches="$HOSTARCH"
 if [ "$HOSTARCH" = amd64 ]; then
-	arches="amd64 arm64 i386"
+	arches="$arches arm64 i386"
-else
+elif [ "$HOSTARCH" = arm64 ]; then
-	arches="$HOSTARCH"
+	arches="$arches amd64 armhf"
 fi
 
 for nativearch in $arches; do
@@ -525,6 +534,9 @@ if [ "$HAVE_QEMU" = "yes" ]; then
 	if [ "$HOSTARCH" = amd64 ] && [ "$RUN_MA_SAME_TESTS" = "yes" ]; then
 		arches=amd64,arm64
 		pkgs="$pkgs,libfakechroot:arm64,libfakeroot:arm64"
+	elif [ "$HOSTARCH" = arm64 ] && [ "$RUN_MA_SAME_TESTS" = "yes" ]; then
+		arches=arm64,amd64
+		pkgs="$pkgs,libfakechroot:amd64,libfakeroot:amd64"
 	else
 		arches=$HOSTARCH
 	fi
@@ -582,12 +594,12 @@ handler () {
 	    } 2>&1;
 	  } | { read xs; exit $xs; };
 	} 3>&1 || ret=$?
+	echo $ret > /mnt/exitstatus.txt
 	if [ -e cover_db.img ]; then
 		df -h cover_db
 		umount cover_db
 	fi
-	echo $ret
+) > /mnt/output.txt 2>&1
-) > /mnt/result.txt 2>&1
 umount /mnt
 systemctl poweroff
 END

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '1.2.5';
+our $VERSION = '1.3.0';
 
 use English;
 use Getopt::Long;
@@ -2072,7 +2072,7 @@ sub run_setup() {
     }
 
     # write /etc/apt/sources.list and files in /etc/apt/sources.list.d/
-    {
+    if (scalar @{ $options->{sourceslists} } > 0) {
         my $firstentry = $options->{sourceslists}->[0];
         # if the first sources.list entry is of one-line type and without
         # explicit filename, then write out an actual /etc/apt/sources.list
@@ -2275,8 +2275,19 @@ sub run_update() {
         CHDIR => $options->{root},
     };
 
+    # Maybe "apt-get update" was already run in the setup hook? If yes, skip
+    # running it here. We are overly strict on purpose because better to run it
+    # twice on accident than not at all.
+    if (   !-d "$options->{root}/var/lib/apt/lists/auxfiles"
+        || !-d "$options->{root}/var/lib/apt/lists/partial"
+        || !-e "$options->{root}/var/lib/apt/lists/lock"
+        || !-e "$options->{root}/var/cache/apt/pkgcache.bin"
+        || !-e "$options->{root}/var/cache/apt/srcpkgcache.bin") {
         info "running apt-get update...";
         run_apt_progress($aptopts);
+    } else {
+        info "skipping apt-get update because it was already run";
+    }
 
     # check if anything was downloaded at all
     {
@@ -2287,11 +2298,36 @@ sub run_update() {
         );
         close $fh;
         if ($indextargets eq '') {
-            if ($verbosity_level >= 1) {
+            warning("apt-get indextargets output is empty");
-                0 == system('apt-cache', 'policy')
+            if (scalar @{ $options->{sourceslists} } == 0) {
-                  or error "apt-cache failed: $?";
+                warning "no known apt sources.list entry";
             }
-            error "apt-get update didn't download anything";
+            for my $list (@{ $options->{sourceslists} }) {
+                if (defined $list->{fname}) {
+                    info("Filename: $list->{fname}");
+                }
+                info("Type: $list->{type}");
+                info("Content:");
+                for my $line (split "\n", $list->{content}) {
+                    info("  $line");
+                }
+            }
+            open(my $fh, '-|', 'apt-cache', 'policy')
+              // error "failed to fork(): $!";
+            while (my $line = <$fh>) {
+                chomp $line;
+                info $line;
+            }
+            close $fh;
+            my $msg
+              = "apt-get update did not find any indices "
+              . "for architecture '$options->{nativearch}' in ";
+            if (length $options->{suite}) {
+                $msg .= "suite '$options->{suite}'";
+            } else {
+                $msg .= "the configured apt sources";
+            }
+            error $msg;
         }
     }
 
@@ -2341,18 +2377,7 @@ sub run_download() {
             info "nothing to download -- skipping...";
             return ([], \@cached_debs);
         }
-        my @apt_argv = ('install');
+        my @apt_argv = ('install', @{ $options->{include} });
-        for my $incl (@{ $options->{include} }) {
-            for my $pkg (split /[,\s]+/, $incl) {
-                # strip leading and trailing whitespace
-                $pkg =~ s/^\s+|\s+$//g;
-                # skip if the remainder is an empty string
-                if ($pkg eq '') {
-                    next;
-                }
-                push @apt_argv, $pkg;
-            }
-        }
 
         @dl_debs = run_apt_download_progress({
                 APT_ARGV => [@apt_argv],
@@ -2880,20 +2905,9 @@ sub run_essential() {
 sub run_install() {
     my $options = shift;
 
-    my %pkgs_to_install;
+    my @pkgs_to_install = (@{ $options->{include} });
-    for my $incl (@{ $options->{include} }) {
-        for my $pkg (split /[,\s]+/, $incl) {
-            # strip leading and trailing whitespace
-            $pkg =~ s/^\s+|\s+$//g;
-            # skip if the remainder is an empty string
-            if ($pkg eq '') {
-                next;
-            }
-            $pkgs_to_install{$pkg} = ();
-        }
-    }
     if ($options->{variant} eq 'buildd') {
-        $pkgs_to_install{'build-essential'} = ();
+        push @pkgs_to_install, 'build-essential';
     }
     if (any { $_ eq $options->{variant} }
         ('required', 'important', 'standard', 'buildd')) {
@@ -2910,7 +2924,8 @@ sub run_install() {
             $priority = '?and(?or(~prequired,~pimportant,~pstandard),'
               . '?not(?essential))';
         }
-        $pkgs_to_install{
+        push @pkgs_to_install,
+          (
             "?narrow("
               . (
                 length($options->{suite})
@@ -2922,9 +2937,8 @@ sub run_install() {
               )
               . "?architecture($options->{nativearch}),"
               . "$priority)"
-        } = ();
+          );
     }
-    my @pkgs_to_install = keys %pkgs_to_install;
 
     if ($options->{mode} eq 'chrootless') {
         if (scalar @pkgs_to_install > 0) {
@@ -4493,14 +4507,28 @@ sub main() {
             opendir(my $dh, $opt_value)
               or error "Can't opendir($opt_value): $!";
             while (my $entry = readdir $dh) {
+                # skip the "." and ".." entries
+                next if $entry eq ".";
+                next if $entry eq "..";
+                my $found = 0;
                 foreach
                   my $hook ('setup', 'extract', 'essential', 'customize') {
-                    if ($entry =~ m/^\Q$hook\E/ and -x "$opt_value/$entry") {
+                    if ($entry =~ m/^\Q$hook\E/) {
+                        if (-x "$opt_value/$entry") {
                             push @{ $scripts{$hook} }, "$opt_value/$entry";
                             $count += 1;
+                            $found = 1;
+                        } else {
+                            warning("$opt_value/$entry is named like a "
+                                  . "hook but not executable");
                         }
                     }
                 }
+                if (!$found && -x "$opt_value/$entry") {
+                    warning("$opt_value/$entry: is executable "
+                          . "but not prefixed with a hook name");
+                }
+            }
             closedir($dh);
             if ($count == 0) {
                 warning "No executable hook scripts found in $opt_value";
@@ -5057,6 +5085,9 @@ sub main() {
                 ## no critic (InputOutput::ProhibitExplicitStdin)
                 <STDIN>;
             };
+            if ($content eq "") {
+                warning "sources.list from standard input is empty";
+            } else {
                 my $type = guess_sources_format($content);
                 if (!defined $type
                     || ($type ne "deb822" and $type ne "one-line")) {
@@ -5068,6 +5099,7 @@ sub main() {
                     fname   => undef,
                     content => $content,
                   };
+            }
         } else {
             my @components = ();
             foreach my $comp (@{ $options->{components} }) {
@@ -5087,22 +5119,24 @@ sub main() {
                 }
             }
             my $compstr = join " ", @components;
-            # if the currently selected apt keyrings do not contain the
+            # From the suite name we can maybe infer which key we need. If we
-            # necessary key material for the chosen suite, then attempt adding
+            # can infer this information, then we need to check whether the
-            # a signed-by option
+            # currently running apt actually trusts this key or not. If it
+            # doesn't, then we need to add a signed-by line to the sources.list
+            # entry.
             my $signedby        = '';
             my %suite_by_vendor = get_suite_by_vendor();
-            {
+            my $gpgproc         = sub {
                 my $keyring
                   = get_keyring_by_suite($options->{suite}, \%suite_by_vendor);
                 if (!defined $keyring) {
-                    last;
+                    return '';
                 }
 
                 # we can only check if we need the signed-by entry if we u
                 # automatically chosen keyring exists
                 if (!defined $keyring || !-e $keyring) {
-                    last;
+                    return '';
                 }
 
                 # we can only check key material if gpg is installed
@@ -5130,9 +5164,9 @@ sub main() {
                     close $fh;
                 }
                 if ($? != 0 || !defined $ret || defined $message) {
-                    info "gpg --version failed: cannot determine the right"
+                    warning
-                      . " signed-by value";
+                      "gpg --version failed: cannot infer signed-by value";
-                    last;
+                    return '';
                 }
                 # initialize gpg trustdb with empty one
                 {
@@ -5141,7 +5175,7 @@ sub main() {
                 }
                 if (!-d $options->{apttrustedparts}) {
                     warning "$options->{apttrustedparts} doesn't exist";
-                    last;
+                    return '';
                 }
                 # find all the fingerprints of the keys apt currently
                 # knows about
@@ -5163,12 +5197,15 @@ sub main() {
                 }
                 my @aptfingerprints = ();
                 if (scalar @keyrings == 0) {
-                    $signedby = " [signed-by=\"$keyring\"]";
+                    return " [signed-by=\"$keyring\"]";
-                    last;
                 }
-                {
+                info "finding correct signed-by value...";
-                    open(my $fh, '-|', @gpgcmd, '--with-colons', '--show-keys',
+                my $progress = 0.0;
-                        @keyrings) // error "failed to fork(): $!";
+                print_progress($progress);
+                for (my $i = 0 ; $i < scalar @keyrings ; $i++) {
+                    my $k = $keyrings[$i];
+                    open(my $fh, '-|', @gpgcmd, '--with-colons',
+                        '--show-keys', $k) // error "failed to fork(): $!";
                     while (my $line = <$fh>) {
                         if ($line !~ /^fpr:::::::::([^:]+):/) {
                             next;
@@ -5176,13 +5213,14 @@ sub main() {
                         push @aptfingerprints, $1;
                     }
                     close $fh;
-                }
                     if ($? != 0) {
-                    error "gpg failed";
+                        warning("gpg failed to read $k");
                     }
+                    print_progress($i / (scalar @keyrings) * 100.0, undef);
+                }
+                print_progress("done");
                 if (scalar @aptfingerprints == 0) {
-                    $signedby = " [signed-by=\"$keyring\"]";
+                    return " [signed-by=\"$keyring\"]";
-                    last;
                 }
                 # check if all fingerprints from the keyring that we guessed
                 # are known by apt and only add signed-by option if that's not
@@ -5198,16 +5236,21 @@ sub main() {
                         # if this fingerprint is not known by apt, then we need
                         #to add the signed-by option
                         if (none { $_ eq $1 } @aptfingerprints) {
-                            $signedby = " [signed-by=\"$keyring\"]";
+                            return " [signed-by=\"$keyring\"]";
-                            last;
                         }
                     }
                     close $fh;
-                }
                     if ($? != 0) {
-                    error "gpg failed";
+                        warning "gpg failed -- cannot infer signed-by value";
                     }
                 }
+                return '';
+            };
+            if (any { $_ eq 'check/signed-by' } @{ $options->{skip} }) {
+                info "skipping check/signed-by as requested";
+            } else {
+                $signedby = $gpgproc->();
+            }
             if (scalar @ARGV > 0) {
                 for my $arg (@ARGV) {
                     if ($arg eq '-') {
@@ -5217,9 +5260,14 @@ sub main() {
                             ## no critic (InputOutput::ProhibitExplicitStdin)
                             <STDIN>;
                         };
+                        if ($content eq "") {
+                            warning
+                              "sources.list from standard input is empty";
+                        } else {
                             my $type = guess_sources_format($content);
                             if (!defined $type
-                            || ($type ne 'deb822' and $type ne 'one-line')) {
+                                || ($type ne 'deb822' and $type ne 'one-line'))
+                            {
                                 error "cannot determine sources.list format";
                             }
                           # if last entry is of same type and without filename,
@@ -5238,6 +5286,7 @@ sub main() {
                                     content => $content,
                                   };
                             }
+                        }
                     } elsif ($arg =~ /^deb(-src)? /) {
                         my $content = "$arg\n";
                         # if last entry is of same type and without filename,
@@ -5281,6 +5330,9 @@ sub main() {
                             $content .= $line;
                         }
                         close $fh;
+                        if ($content eq "") {
+                            warning "$arg is empty";
+                        } else {
                             my $type = undef;
                             if ($arg =~ /\.list$/) {
                                 $type = 'one-line';
@@ -5290,7 +5342,8 @@ sub main() {
                                 $type = guess_sources_format($content);
                             }
                             if (!defined $type
-                            || ($type ne 'deb822' and $type ne 'one-line')) {
+                                || ($type ne 'deb822' and $type ne 'one-line'))
+                            {
                                 error "cannot determine sources.list format";
                             }
                             push @{$sourceslists},
@@ -5299,6 +5352,9 @@ sub main() {
                                 fname   => basename($arg),
                                 content => $content,
                               };
+                        }
+                    } elsif ($arg eq '') {
+                        # empty
                     } else {
                         error "invalid mirror: $arg";
                     }
@@ -5317,7 +5373,7 @@ sub main() {
             }
         }
         if (scalar @{$sourceslists} == 0) {
-            error "empty apt sources.list";
+            warning "empty apt sources.list";
         }
         debug("sources list entries:");
         for my $list (@{$sourceslists}) {
@@ -6186,7 +6242,12 @@ I<TARGET> is C<-> or if no I<TARGET> was specified.
 =head1 OPTIONS
 
 Options are case insensitive. Short options may be bundled. Long options
-require a double dash and may be abbreviated to uniqueness.
+require a double dash and may be abbreviated to uniqueness. Options can be
+placed anywhere on the command line, even before or mixed with the I<SUITE>,
+I<TARGET>, and I<MIRROR> arguments. A double dash C<--> can be used to stop
+interpreting command line arguments as options to allow I<SUITE>, I<TARGET> and
+I<MIRROR> arguments that start with a single or double dash. Option order only
+matters for options that can be passed multiple times as documented below.
 
 =over 8
 
@@ -6907,6 +6968,8 @@ Upon startup, several checks are carried out, like:
 
 =item * whether the output directory is empty. This check can be disabled using B<--skip=check/empty>
 
+=item * whether adding a C<signed-by> to C<apt/sources.list> is necessary. This requires gpg and can be disabled using B<--skip=check/signed-by>
+
 =back
 
 =item B<setup>
@@ -6946,7 +7009,10 @@ variant uses the fact that libapt treats the C<apt> packages as implicitly
 essential to download only all C<Essential:yes> packages plus apt using
 C<apt-get dist-upgrade>. In the remaining variants, all Packages files
 downloaded by the B<update> step are inspected to find the C<Essential:yes>
-package set as well as all packages of the required priority.
+package set as well as all packages of the required priority. If I<SUITE> is a
+non-empty string, then only packages from the archive with suite or codename
+matching I<SUITE> will be considered for selection of C<Essential:yes>
+packages.
 
 =item B<mount>
 

run_null.sh

@@ -23,8 +23,8 @@ done
 	cd ./shared;
 	$SUDO sh -x ./test.sh;
 	echo $?;
-) 2>&1 | tee shared/result.txt | head --lines=-1
+) 2>&1 | tee shared/output.txt
-if [ "$(tail --lines=1 shared/result.txt)" -ne 0 ]; then
+if [ "$(cat shared/exitstatus.txt)" -ne 0 ]; then
 	echo "test.sh failed"
 	exit 1
 fi

run_qemu.sh

@@ -11,10 +11,11 @@ cleanup() {
 	rm -f "$tmpdir/debian-$DEFAULT_DIST-overlay.qcow"
 	rm -f "$tmpdir/log"
 	[ -e "$tmpdir" ] && rmdir "$tmpdir"
-	if [ -e shared/result.txt ]; then
+	if [ -n "${TAIL_PID:-}" ]; then
-		head --lines=-1 shared/result.txt
+		kill "$TAIL_PID"
-		res="$(tail --lines=1 shared/result.txt)"
+	fi
-		rm shared/result.txt
+	if [ -e shared/output.txt ]; then
+		res="$(cat shared/exitstatus.txt)"
 		if [ "$res" != "0" ]; then
 			# this might possibly overwrite another non-zero rv
 			rv=1
@@ -45,6 +46,14 @@ case $ARCH in
 	*) echo "qemu kvm not supported on $ARCH" >&2;;
 esac
 
+echo 1 > shared/exitstatus.txt
+if [ -e shared/output.txt ]; then
+	rm shared/output.txt
+fi
+touch shared/output.txt
+tail -f shared/output.txt &
+TAIL_PID=$!
+
 # the path to debian-$DEFAULT_DIST.qcow must be absolute or otherwise qemu will
 # look for the path relative to debian-$DEFAULT_DIST-overlay.qcow
 qemu-img create -f qcow2 -b "$(realpath "$cachedir")/debian-$DEFAULT_DIST.qcow" -F qcow2 "$tmpdir/debian-$DEFAULT_DIST-overlay.qcow"

tests/apt-patterns

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+{{ CMD }} --mode={{ MODE }} --variant=essential \
+	--include '?or(?exact-name(dummy-does-not-exist),?exact-name(apt))' \
+	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -

tests/apt-patterns-custom

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+{{ CMD }} --mode={{ MODE }} --variant=custom \
+	--include '?narrow(?archive(^{{ DIST }}$),?essential)' \
+	--include apt \
+	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/as-debootstrap-unshare-wrapper

@@ -61,8 +61,15 @@ for f in shadow shadow-; do
 	fi
 done
 
+# isc-dhcp-client postinst doesn't create this file in debootstrap run with
+# unshared wrapper. The responsible postinst snippet was automatically added
+# by dh_apparmor since isc-dhcp-client 4.4.3-P1-1.1
+if [ -e /tmp/debian-debootstrap/etc/apparmor.d/local/sbin.dhclient ] && [ ! -s /tmp/debian-debootstrap/etc/apparmor.d/local/sbin.dhclient ]; then
+	echo /sbin/setcap > /tmp/debian-debootstrap/etc/apparmor.d/local/sbin.dhclient
+fi
+
 # check if the file content differs
-diff --no-dereference --recursive /tmp/debian-debootstrap /tmp/debian-mm >&2
+diff --unified --no-dereference --recursive /tmp/debian-debootstrap /tmp/debian-mm >&2
 
 # check permissions, ownership, symlink targets, modification times using tar
 # mtimes of directories created by mmdebstrap will differ, thus we equalize them first

tests/chrootless

@@ -2,14 +2,10 @@
 set -eu
 export LC_ALL=C.UTF-8
 export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-if dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libpam-runtime)" le 1.5.2-5; then
-	# https://bugs.debian.org/1022952
-	exit 0
-fi
 trap "rm -f /tmp/chrootless.tar /tmp/root.tar" EXIT INT TERM
 # we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
 # DPKG_ROOT
-for INCLUDE in '' 'systemd-sysv'; do
+for INCLUDE in '' 'apt' 'apt,build-essential' 'systemd-sysv'; do
 	for MODE in root chrootless; do
 		{{ CMD }} --mode=$MODE --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
 			${INCLUDE:+--include="$INCLUDE"} \

tests/chrootless-fakeroot

@@ -16,7 +16,7 @@ prefix=
 # DPKG_ROOT
 # permissions drwxr-sr-x and extended attributes of ./var/log/journal/ cannot
 # be preserved under fakeroot
-for INCLUDE in '' 'systemd-sysv'; do
+for INCLUDE in '' 'apt' 'apt,build-essential' 'systemd-sysv'; do
 	{{ CMD }} --variant={{ VARIANT }} --hook-dir=./hooks/merged-usr \
 		--customize-hook='if [ -d "$1"/var/log/journal ]; then rmdir "$1"/var/log/journal; mkdir --mode=2755 "$1"/var/log/journal; chroot "$1" chown root:systemd-journal /var/log/journal; fi' \
 		${INCLUDE:+--include="$INCLUDE"} \

tests/chrootless-foreign

@@ -8,20 +8,42 @@ if [ ! -e /mmdebstrap-testenv ]; then
 	exit 1
 fi
 
+deb2qemu() {
+	case "$1" in
+		amd64) echo x86_64;;
+		arm64) echo aarch64;;
+		armel|armhf) echo arm;;
+		ppc64el) echo ppc64le;;
+		*) echo "$1";;
+	esac
+}
+if [ "$(dpkg --print-architecture)" = "arm64" ]; then
+	arch=amd64
+else
+	arch=arm64
+fi
+
 [ "$(id -u)" -eq 0 ]
-[ -e /proc/sys/fs/binfmt_misc/qemu-aarch64 ]
+[ -e "/proc/sys/fs/binfmt_misc/qemu-$(deb2qemu "$arch")" ]
+
 
 # we need --hook-dir=./hooks/merged-usr because usrmerge does not understand
 # DPKG_ROOT
-for INCLUDE in '' 'systemd-sysv'; do
+#
-	echo 1 > /proc/sys/fs/binfmt_misc/qemu-aarch64
+# dpkg is unable to install architecture arch:all packages with a
-	arch-test arm64
+# dependency on an arch:any package (perl-modules-5.34 in this case)
-	{{ CMD }} --mode=root --architecture=arm64 --variant={{ VARIANT }} \
+# inside foreign architecture chrootless chroots, because dpkg will use
+# its own architecture as the native architecture, see #825385 and #1020533
+# So we are not testing the installation of apt,build-essential here.
+for INCLUDE in '' 'apt' 'systemd-sysv'; do
+	echo 1 > "/proc/sys/fs/binfmt_misc/qemu-$(deb2qemu "$arch")"
+	arch-test "$arch"
+	{{ CMD }} --mode=root --architecture="$arch" --variant={{ VARIANT }} \
 		--hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
 		{{ DIST }} "/tmp/root.tar" {{ MIRROR }}
-	echo 0 > /proc/sys/fs/binfmt_misc/qemu-aarch64
+	echo 0 > "/proc/sys/fs/binfmt_misc/qemu-$(deb2qemu "$arch")"
-	arch-test arm64 && exit 1
+	arch-test "$arch" && exit 1
-	{{ CMD }} --mode=chrootless --architecture=arm64 --variant={{ VARIANT }} \
+	{{ CMD }} --mode=chrootless --architecture="$arch" --variant={{ VARIANT }} \
 		--hook-dir=./hooks/merged-usr ${INCLUDE:+--include="$INCLUDE"} \
 		{{ DIST }} "/tmp/chrootless.tar" {{ MIRROR }}
 	# when creating a foreign architecture chroot, the tarballs are not

tests/empty-sources.list

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+printf '' | {{ CMD }} --mode={{ MODE }} --variant=apt \
+	--setup-hook='echo "deb {{ MIRROR }} {{ DIST }} main" > "$1"/etc/apt/sources.list' \
+	{{ DIST }} /tmp/debian-chroot.tar -
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/logfile

@@ -7,6 +7,7 @@ export LC_ALL=C.UTF-8
 head --lines=-1 /tmp/log > /tmp/trimmed
 cat << LOG | diff -u - /tmp/trimmed
 I: chroot architecture {{ HOSTARCH }} is equal to the host's architecture
+I: finding correct signed-by value...
 I: automatically chosen format: directory
 I: running apt-get update...
 I: downloading packages with apt...

tests/multiple-include

@@ -9,7 +9,6 @@ rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
 rm /tmp/debian-chroot/usr/share/lintian/overrides/tzdata
 rm /tmp/debian-chroot/etc/localtime
 rm /tmp/debian-chroot/etc/timezone
-rm /tmp/debian-chroot/usr/sbin/tzconfig
 rm -r /tmp/debian-chroot/usr/share/doc/tzdata
 rm -r /tmp/debian-chroot/usr/share/zoneinfo
 rm /tmp/debian-chroot/var/lib/apt/extended_states