Switch variant apt from 'apt-get dist-upgrade' to apt patterns

The dist-upgrade method will fail to install essential packages but
still exit successfully without reporting an error, see #1072218

mmdebstrap

@@ -2070,19 +2070,16 @@ sub run_setup() {
     # for authentication, use the keyrings from the host
     print $conf "Dir::Etc::Trusted \"$options->{apttrusted}\";\n";
     print $conf "Dir::Etc::TrustedParts \"$options->{apttrustedparts}\";\n";
-    if ($options->{variant} ne 'apt') {
+    # apt considers itself essential. Thus, when generating an EDSP document
-        # apt considers itself essential. Thus, when generating an EDSP
+    # for an external solver, it will add the Essential:yes field to the apt
-        # document for an external solver, it will add the Essential:yes field
+    # package stanza. This is unnecessary because we compile the set of
-        # to the apt package stanza. This is unnecessary for any other variant
+    # packages we consider essential ourselves and for the 'essential' variant
-        # than 'apt' because in all other variants we compile the set of
+    # it would even be wrong to add apt. This workaround is only needed when
-        # packages we consider essential ourselves and for the 'essential'
+    # apt is used with an external solver but doesn't hurt otherwise and we
-        # variant it would even be wrong to add apt. This workaround is only
+    # don't have a good way to figure out whether apt is using an external
-        # needed when apt is used with an external solver but doesn't hurt
+    # solver or not short of parsing the --aptopt options.
-        # otherwise and we don't have a good way to figure out whether apt is
+    print $conf "pkgCacheGen::ForceEssential \",\";\n";
-        # using an external solver or not short of parsing the --aptopt
+
-        # options.
-        print $conf "pkgCacheGen::ForceEssential \",\";\n";
-    }
     close $conf;
 
     # We put certain configuration items in their own configuration file
@@ -2488,27 +2485,8 @@ sub run_download() {
                 dryrun   => $options->{dryrun},
             },
         );
-    } elsif ($options->{variant} eq 'apt') {
-        # if we just want to install Essential:yes packages, apt and their
-        # dependencies then we can make use of libapt treating apt as
-        # implicitly essential. An upgrade with the (currently) empty status
-        # file will trigger an installation of the essential packages plus apt.
-        #
-        # 2018-09-02, #debian-dpkg on OFTC, times in UTC+2
-        # 23:39 < josch> I'll just put it in my script and if it starts
-        #                breaking some time I just say it's apt's fault. :P
-        # 23:42 < DonKult> that is how it usually works, so yes, do that :P (<-
-        #                  and please add that line next to it so you can
-        #                  remind me in 5+ years that I said that after I wrote
-        #                  in the bugreport: "Are you crazy?!? Nobody in his
-        #                  right mind would even suggest depending on it!")
-        @dl_debs = run_apt_download_progress({
-                APT_ARGV => ['dist-upgrade'],
-                dryrun   => $options->{dryrun},
-            },
-        );
     } elsif (any { $_ eq $options->{variant} }
-        ('essential', 'standard', 'important', 'required', 'buildd')) {
+        ('essential', 'apt', 'standard', 'important', 'required', 'buildd')) {
         # 2021-06-07, #debian-apt on OFTC, times in UTC+2
         # 17:27 < DonKult> (?essential includes 'apt' through)
         # 17:30 < josch> DonKult: no, because pkgCacheGen::ForceEssential ",";
@@ -2982,8 +2960,14 @@ sub run_install() {
     my $options = shift;
 
     my @pkgs_to_install = (@{ $options->{include} });
+    if ($options->{variant} eq 'extract') {
+        error "must not be called with variant extract";
+    }
+    if (none { $_ eq $options->{variant} } ('custom', 'essential')) {
+        push @pkgs_to_install, 'apt';
+    }
     if ($options->{variant} eq 'buildd') {
-        push @pkgs_to_install, 'build-essential', 'apt';
+        push @pkgs_to_install, 'build-essential';
     }
     if (any { $_ eq $options->{variant} }
         ('required', 'important', 'standard')) {
@@ -7084,14 +7068,10 @@ considered for selection of C<Essential:yes> packages.
 
 =item B<apt>
 
-The B<essential> set plus apt. This variant uses the fact that B<apt> treats
+The B<essential> set plus apt.
-itself as essential and thus running C<apt-get dist-upgrade> without any
+It is roughly equivalent to running mmdebstrap with
-packages installed will install the B<essential> set plus B<apt>. If you just
+
-want B<essential> and B<apt>, then this variant is faster than using the
+    --variant=essential --include="apt"
-B<essential> variant and adding B<apt> via C<--include> because all packages
-get installed at once. The downside of this variant is, that if it should
-happen that an B<essential> package is not installable, then it will just get
-ignored without throwing an error.
 
 =item B<buildd>
 
@@ -7420,15 +7400,11 @@ the B<setup> step. This can be disabled using B<--skip=update>.
 =item B<download>
 
 In the B<extract> and B<custom> variants, C<apt-get install> is used to
-download all the packages requested via the B<--include> option. The B<apt>
+download all the packages requested via the B<--include> option. In the
-variant uses the fact that libapt treats the C<apt> packages as implicitly
+remaining variants, apt patterns are used to find the C<Essential:yes> packages
-essential to download only all C<Essential:yes> packages plus apt using
+from the native architecture.  If I<SUITE> is a non-empty string, then only
-C<apt-get dist-upgrade>. In the remaining variants, all Packages files
+packages from the archive with suite or codename matching I<SUITE> will be
-downloaded by the B<update> step are inspected to find the C<Essential:yes>
+considered for selection of C<Essential:yes> packages.
-package set as well as all packages of the required priority. If I<SUITE> is a
-non-empty string, then only packages from the archive with suite or codename
-matching I<SUITE> will be considered for selection of C<Essential:yes>
-packages.
 
 =item B<mount>
 

tests/apt-patterns

@@ -5,4 +5,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 {{ CMD }} --mode={{ MODE }} --variant=essential \
 	--include '?or(?exact-name(dummy-does-not-exist),?exact-name(apt))' \
 	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

tests/apt-patterns-custom

@@ -6,4 +6,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 	--include '?narrow(?archive(^{{ DIST }}$),?essential)' \
 	--include apt \
 	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+{ tar -tf /tmp/debian-chroot.tar; echo ./var/lib/apt/extended_states; } | sort | diff -u tar1.txt -

tests/aspcud-apt-solver

@@ -6,6 +6,6 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
     --include "$(tr '\n' ',' < pkglist.txt)" \
     --aptopt='APT::Solver "aspcud"' \
     {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-tar -tf /tmp/debian-chroot.tar | sort \
+{ tar -tf /tmp/debian-chroot.tar; echo ./var/lib/apt/extended_states; } | sort \
     | grep -v '^./etc/apt/apt.conf.d/99mmdebstrap$' \
     | diff -u tar1.txt -

tests/essential-hook

@@ -17,5 +17,4 @@ tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort \
 	| grep -v '^./usr/share/lintian/overrides/tzdata' \
 	| grep -v '^./usr/share/zoneinfo' \
 	| grep -v '^./var/lib/dpkg/info/tzdata.' \
-	| grep -v '^./var/lib/apt/extended_states$' \
 	| diff -u tar1.txt -

tests/include

@@ -6,7 +6,6 @@ trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
 rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
 rm -r /tmp/debian-chroot/usr/share/doc/debian
 rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
-rm /tmp/debian-chroot/var/lib/apt/extended_states
 rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.list
 rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.md5sums
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

tests/include-foreign-libmagic-mgc

@@ -36,7 +36,6 @@ rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
 rm /tmp/debian-chroot/usr/share/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/misc/magic.mgc
-rm /tmp/debian-chroot/var/lib/apt/extended_states
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
 rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/

tests/include-foreign-libmagic-mgc-with-multiple-arch-options

@@ -37,7 +37,6 @@ rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
 rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
 rm /tmp/debian-chroot/usr/share/file/magic.mgc
 rm /tmp/debian-chroot/usr/share/misc/magic.mgc
-rm /tmp/debian-chroot/var/lib/apt/extended_states
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
 rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
 rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/

tests/logfile

@@ -16,6 +16,7 @@ I: running apt-get update...
 I: downloading packages with apt...
 I: extracting archives...
 I: installing essential packages...
+I: installing remaining packages inside the chroot...
 I: cleaning package lists and apt cache...
 LOG
 tail --lines=1 /tmp/log | grep '^I: success in .* seconds$'

tests/multiple-include

@@ -11,7 +11,6 @@ rm /tmp/debian-chroot/etc/localtime
 rm /tmp/debian-chroot/etc/timezone
 rm -r /tmp/debian-chroot/usr/share/doc/tzdata
 rm -r /tmp/debian-chroot/usr/share/zoneinfo
-rm /tmp/debian-chroot/var/lib/apt/extended_states
 for p in doc-debian tzdata; do
 	for f in list md5sums config postinst postrm templates preinst prerm; do
 		[ -e "/tmp/debian-chroot/var/lib/dpkg/info/$p.$f" ] || continue

tests/not-having-to-install-apt-in-include-because-a-hook-did-it-before

@@ -6,4 +6,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
 	--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get update' \
 	--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install apt' \
 	{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
-tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -