release 1.5.1

The dist-upgrade method will fail to install essential packages but
still exit successfully without reporting an error, see #1072218

CHANGELOG.md

@@ -1,3 +1,9 @@
+1.5.1 (2024-06-03)
+------------------
+
+ - in root and unshare mode, run 'mount --make-rprivate /' before bind-mounting
+ - switch apt variant from using 'apt-get dist-upgrade' to apt patterns
+
 1.5.0 (2024-05-14)
 ------------------
 

make_mirror.sh

@@ -550,6 +550,7 @@ END
 		--customize-hook='copy-in "'"$tmpdir"'/worker.sh" /' \
 		--customize-hook='echo 127.0.0.1 localhost > "$1/etc/hosts"' \
 		--customize-hook='printf "START=1\nDAEMON_OPTS=\"-h 127.0.0.1 -p 80 -u nobody -dd /mnt/cache -i /var/run/mini-httpd.pid -T UTF-8\"\n" > "$1/etc/default/mini-httpd"' \
+		--customize-hook='touch "$1/etc/systemd/system/tmp.mount"' \
 		"$mirror"
 
 	kill $PROXYPID

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '1.5.0';
+our $VERSION = '1.5.1';
 
 use English;
 use Getopt::Long;
@@ -36,7 +36,8 @@ use File::Find;
 use Cwd qw(abs_path getcwd);
 require "syscall.ph";      ## no critic (Modules::RequireBarewordIncludes)
 require "sys/ioctl.ph";    ## no critic (Modules::RequireBarewordIncludes)
-use Fcntl      qw(S_IFCHR S_IFBLK FD_CLOEXEC F_GETFD F_SETFD);
+use Fcntl
+  qw(S_IFCHR S_IFBLK FD_CLOEXEC F_GETFD F_SETFD LOCK_EX O_RDONLY O_DIRECTORY);
 use List::Util qw(any none);
 use POSIX
   qw(SIGINT SIGHUP SIGPIPE SIGTERM SIG_BLOCK SIG_UNBLOCK strftime isatty);
@@ -2069,19 +2070,16 @@ sub run_setup() {
     # for authentication, use the keyrings from the host
     print $conf "Dir::Etc::Trusted \"$options->{apttrusted}\";\n";
     print $conf "Dir::Etc::TrustedParts \"$options->{apttrustedparts}\";\n";
-    if ($options->{variant} ne 'apt') {
-        # apt considers itself essential. Thus, when generating an EDSP
-        # document for an external solver, it will add the Essential:yes field
-        # to the apt package stanza. This is unnecessary for any other variant
-        # than 'apt' because in all other variants we compile the set of
-        # packages we consider essential ourselves and for the 'essential'
-        # variant it would even be wrong to add apt. This workaround is only
-        # needed when apt is used with an external solver but doesn't hurt
-        # otherwise and we don't have a good way to figure out whether apt is
-        # using an external solver or not short of parsing the --aptopt
-        # options.
+    # apt considers itself essential. Thus, when generating an EDSP document
+    # for an external solver, it will add the Essential:yes field to the apt
+    # package stanza. This is unnecessary because we compile the set of
+    # packages we consider essential ourselves and for the 'essential' variant
+    # it would even be wrong to add apt. This workaround is only needed when
+    # apt is used with an external solver but doesn't hurt otherwise and we
+    # don't have a good way to figure out whether apt is using an external
+    # solver or not short of parsing the --aptopt options.
     print $conf "pkgCacheGen::ForceEssential \",\";\n";
-    }
+
     close $conf;
 
     # We put certain configuration items in their own configuration file
@@ -2487,27 +2485,8 @@ sub run_download() {
                 dryrun   => $options->{dryrun},
             },
         );
-    } elsif ($options->{variant} eq 'apt') {
-        # if we just want to install Essential:yes packages, apt and their
-        # dependencies then we can make use of libapt treating apt as
-        # implicitly essential. An upgrade with the (currently) empty status
-        # file will trigger an installation of the essential packages plus apt.
-        #
-        # 2018-09-02, #debian-dpkg on OFTC, times in UTC+2
-        # 23:39 < josch> I'll just put it in my script and if it starts
-        #                breaking some time I just say it's apt's fault. :P
-        # 23:42 < DonKult> that is how it usually works, so yes, do that :P (<-
-        #                  and please add that line next to it so you can
-        #                  remind me in 5+ years that I said that after I wrote
-        #                  in the bugreport: "Are you crazy?!? Nobody in his
-        #                  right mind would even suggest depending on it!")
-        @dl_debs = run_apt_download_progress({
-                APT_ARGV => ['dist-upgrade'],
-                dryrun   => $options->{dryrun},
-            },
-        );
     } elsif (any { $_ eq $options->{variant} }
-        ('essential', 'standard', 'important', 'required', 'buildd')) {
+        ('essential', 'apt', 'standard', 'important', 'required', 'buildd')) {
         # 2021-06-07, #debian-apt on OFTC, times in UTC+2
         # 17:27 < DonKult> (?essential includes 'apt' through)
         # 17:30 < josch> DonKult: no, because pkgCacheGen::ForceEssential ",";
@@ -2981,8 +2960,14 @@ sub run_install() {
     my $options = shift;
 
     my @pkgs_to_install = (@{ $options->{include} });
+    if ($options->{variant} eq 'extract') {
+        error "must not be called with variant extract";
+    }
+    if (none { $_ eq $options->{variant} } ('custom', 'essential')) {
+        push @pkgs_to_install, 'apt';
+    }
     if ($options->{variant} eq 'buildd') {
-        push @pkgs_to_install, 'build-essential', 'apt';
+        push @pkgs_to_install, 'build-essential';
     }
     if (any { $_ eq $options->{variant} }
         ('required', 'important', 'standard')) {
@@ -4454,8 +4439,9 @@ sub approx_disk_usage {
     # image. See https://bugs.debian.org/1005857
     find({ wanted => $scan_installed_size, no_chdir => 1 }, $directory);
 
-    # because the above is only a heuristic we add 10% extra for good measure
-    return int($installed_size * 1.1);
+    # the above is only a heuristic and especially ext4 will consume quite a
+    # few more blocks than the heuristic above is going to compute
+    return int($installed_size * 1.2);
 }
 
 sub main() {
@@ -5759,6 +5745,7 @@ sub main() {
         $blocksize = 1048576;
     }
 
+    my $rootdir_handle;
     if (any { $_ eq $options->{format} }
         ('tar', 'squashfs', 'ext2', 'ext4', 'null')) {
         if ($options->{format} ne 'null') {
@@ -5789,6 +5776,13 @@ sub main() {
         # directory
         $options->{root} = tempdir('mmdebstrap.XXXXXXXXXX', TMPDIR => 1);
         info "using $options->{root} as tempdir";
+        # add an flock on the temporary directory to prevent cleanup by systemd
+        # see section Age in tmpfiles.d(5)
+        sysopen($rootdir_handle, $options->{root}, O_RDONLY | O_DIRECTORY)
+          or error "Failed to sysopen $options->{root}: $!\n";
+        flock($rootdir_handle, LOCK_EX)
+          or error "Unable to flock $options->{root}: $!\n";
+
         # in unshare and root mode, other users than the current user need to
         # access the rootfs, most prominently, the _apt user. Thus, make the
         # temporary directory world readable.
@@ -7074,14 +7068,10 @@ considered for selection of C<Essential:yes> packages.
 
 =item B<apt>
 
-The B<essential> set plus apt. This variant uses the fact that B<apt> treats
-itself as essential and thus running C<apt-get dist-upgrade> without any
-packages installed will install the B<essential> set plus B<apt>. If you just
-want B<essential> and B<apt>, then this variant is faster than using the
-B<essential> variant and adding B<apt> via C<--include> because all packages
-get installed at once. The downside of this variant is, that if it should
-happen that an B<essential> package is not installable, then it will just get
-ignored without throwing an error.
+The B<essential> set plus apt.
+It is roughly equivalent to running mmdebstrap with
+
+    --variant=essential --include="apt"
 
 =item B<buildd>
 
@@ -7410,15 +7400,11 @@ the B<setup> step. This can be disabled using B<--skip=update>.
 =item B<download>
 
 In the B<extract> and B<custom> variants, C<apt-get install> is used to
-download all the packages requested via the B<--include> option. The B<apt>
-variant uses the fact that libapt treats the C<apt> packages as implicitly
-essential to download only all C<Essential:yes> packages plus apt using
-C<apt-get dist-upgrade>. In the remaining variants, all Packages files
-downloaded by the B<update> step are inspected to find the C<Essential:yes>
-package set as well as all packages of the required priority. If I<SUITE> is a
-non-empty string, then only packages from the archive with suite or codename
-matching I<SUITE> will be considered for selection of C<Essential:yes>
-packages.
+download all the packages requested via the B<--include> option. In the
+remaining variants, apt patterns are used to find the C<Essential:yes> packages
+from the native architecture.  If I<SUITE> is a non-empty string, then only
+packages from the archive with suite or codename matching I<SUITE> will be
+considered for selection of C<Essential:yes> packages.
 
 =item B<mount>
 

run_qemu.sh

@@ -4,6 +4,7 @@ set -eu
 
 : "${DEFAULT_DIST:=unstable}"
 : "${cachedir:=./shared/cache}"
+: "${MMDEBSTRAP_TESTS_DEBUG:=no}"
 tmpdir="$(mktemp -d)"
 
 cleanup() {
@@ -29,20 +30,42 @@ fi
 touch shared/output.txt
 setpriv --pdeathsig TERM tail -f shared/output.txt &
 
+set -- timeout --foreground 40m \
+	debvm-run --image="$(realpath "$cachedir")/debian-$DEFAULT_DIST.ext4" \
+	--
+cpuname=$(lscpu | awk '/Model name:/ {print $3}' | tr '\n' '+')
+ncpu=$(lscpu | awk '/Core\(s\) per socket:/ {print $4}' | tr '\n' '+')
+if [ "$cpuname" = "Cortex-A53+Cortex-A73+" ] && [ "$ncpu" = "2+4+" ]; then
+	# crude detection of the big.LITTLE heterogeneous setup of cores on the
+	# amlogic a311d bananapi
+	#
+	# https://lists.nongnu.org/archive/html/qemu-devel/2020-10/msg08494.html
+	# https://gitlab.com/qemu-project/qemu/-/issues/239
+	# https://segments.zhan.science/posts/kvm_on_pinehone_pro/#trouble-with-heterogeneous-architecture
+	set -- taskset --cpu-list 2,3,4,5 "$@" -smp 4
+fi
+
+set -- "$@" -nic none -m 4G -snapshot
+
+if [ "$MMDEBSTRAP_TESTS_DEBUG" = "no" ]; then
 	# to connect to serial use:
 	#   minicom -D 'unix#/tmp/ttyS0'
-#
 	# or this (quit with ctrl+q):
 	#   socat stdin,raw,echo=0,escape=0x11 unix-connect:/tmp/ttyS0
-ret=0
-timeout --foreground 40m debvm-run --image="$(realpath "$cachedir")/debian-$DEFAULT_DIST.ext4" -- \
-	-nic none \
-	-m 4G -snapshot \
+	set -- "$@" \
 	-monitor unix:/tmp/monitor,server,nowait \
 	-serial unix:/tmp/ttyS0,server,nowait \
-	-serial unix:/tmp/ttyS1,server,nowait \
-	-virtfs local,id=mmdebstrap,path="$(pwd)/shared",security_model=none,mount_tag=mmdebstrap \
-	>"$tmpdir/log" 2>&1 || ret=$?
+	-serial unix:/tmp/ttyS1,server,nowait
+fi
+
+set -- "$@" -virtfs local,id=mmdebstrap,path="$(pwd)/shared",security_model=none,mount_tag=mmdebstrap
+
+ret=0
+if [ "$MMDEBSTRAP_TESTS_DEBUG" = "no" ]; then
+	"$@" >"$tmpdir/log" 2>&1 || ret=$?
+else
+	"$@" 2>&1 | tee "$tmpdir/log" || ret=$?
+fi
 if [ "$ret" -ne 0 ]; then
 	cat "$tmpdir/log"
 	exit $ret

tests/apt-patterns

@@ -5,4 +5,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 {{ CMD }} --mode={{ MODE }} --variant=essential \
 	--include '?or(?exact-name(dummy-does-not-exist),?exact-name(apt))' \
 	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/apt-patterns-custom

@@ -6,4 +6,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 	--include '?narrow(?archive(^{{ DIST }}$),?essential)' \
 	--include apt \
 	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+{ tar -tf /tmp/debian-chroot.tar; echo ./var/lib/apt/extended_states; } | sort | diff -u tar1.txt -

tests/aspcud-apt-solver

@@ -6,6 +6,6 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
     --include "$(tr '\n' ',' < pkglist.txt)" \
     --aptopt='APT::Solver "aspcud"' \
     {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-tar -tf /tmp/debian-chroot.tar | sort \
+{ tar -tf /tmp/debian-chroot.tar; echo ./var/lib/apt/extended_states; } | sort \
     | grep -v '^./etc/apt/apt.conf.d/99mmdebstrap$' \
     | diff -u tar1.txt -

tests/essential-hook

@@ -17,5 +17,4 @@ tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort \
 	| grep -v '^./usr/share/lintian/overrides/tzdata' \
 	| grep -v '^./usr/share/zoneinfo' \
 	| grep -v '^./var/lib/dpkg/info/tzdata.' \
-	| grep -v '^./var/lib/apt/extended_states$' \
 	| diff -u tar1.txt -

tests/include

@@ -6,7 +6,6 @@ trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
 rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
-rm /tmp/debian-chroot/var/lib/apt/extended_states
 rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.list
 rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.md5sums
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/include-foreign-libmagic-mgc

@@ -36,7 +36,6 @@ rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
 rm /tmp/debian-chroot/usr/share/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/misc/magic.mgc
-rm /tmp/debian-chroot/var/lib/apt/extended_states
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
 rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/

tests/include-foreign-libmagic-mgc-with-multiple-arch-options

@@ -37,7 +37,6 @@ rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
 rm /tmp/debian-chroot/usr/share/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/misc/magic.mgc
-rm /tmp/debian-chroot/var/lib/apt/extended_states
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
 rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/

tests/logfile

@@ -16,6 +16,7 @@ I: running apt-get update...
 I: downloading packages with apt...
 I: extracting archives...
 I: installing essential packages...
+I: installing remaining packages inside the chroot...
 I: cleaning package lists and apt cache...
 LOG
 tail --lines=1 /tmp/log | grep '^I: success in .* seconds$'

tests/multiple-include

@@ -11,7 +11,6 @@ rm /tmp/debian-chroot/etc/localtime
 rm /tmp/debian-chroot/etc/timezone
 rm -r /tmp/debian-chroot/usr/share/doc/tzdata
 rm -r /tmp/debian-chroot/usr/share/zoneinfo
-rm /tmp/debian-chroot/var/lib/apt/extended_states
 for p in doc-debian tzdata; do
 	for f in list md5sums config postinst postrm templates preinst prerm; do
 		[ -e "/tmp/debian-chroot/var/lib/dpkg/info/$p.$f" ] || continue

tests/not-having-to-install-apt-in-include-because-a-hook-did-it-before

@@ -6,4 +6,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 	--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get update' \
 	--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install apt' \
 	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -