Ask apt to give us the deb filenames directly

Guessing filenames is boring. What if we could ask apt to tell us which
debs it downloaded (or found lying around elsewhere) directly? Turns out
we can rather easily avoiding a bunch of guesswork.

coverage.sh

@@ -3072,7 +3072,7 @@ $CMD \$include --mode=$defaultmode --variant=$variant \
 	--setup-hook='sync-in "'"\$tmpdir"'" /var/cache/apt/archives/partial' \
 	$DEFAULT_DIST - $mirror > test1.tar
 cmp orig.tar test1.tar
-$CMD \$include --mode=$defaultmode --variant=$variant --skip=download/empty \
+$CMD \$include --mode=$defaultmode --variant=$variant \
 	--customize-hook='touch "\$1"/var/cache/apt/archives/partial' \
 	--setup-hook='mkdir -p "\$1"/var/cache/apt/archives/' \
 	--setup-hook='sync-in "'"\$tmpdir"'" /var/cache/apt/archives/' \

mmdebstrap

@@ -874,20 +874,9 @@ sub run_dpkg_progress {
 sub run_apt_progress {
     my $options  = shift;
     my @debs     = @{ $options->{PKGS} // [] };
-    my $tmpeipp;
-    if (exists $options->{EIPP_RES}) {
-        (undef, $tmpeipp) = tempfile(
-            "mmdebstrap.eipp.XXXXXXXXXXXX",
-            OPEN   => 0,
-            TMPDIR => 1
-        );
-    }
     my $get_exec = sub {
         my @prefix = ();
         my @opts   = ();
-        if (exists $options->{EIPP_RES}) {
-            push @opts, "-oDir::Log::Planner=$tmpeipp";
-        }
         return (
             @prefix,
             @{ $options->{ARGV} },
@@ -942,60 +931,44 @@ sub run_apt_progress {
         }
     };
     run_progress $get_exec, $line_handler, $line_has_error, $options->{CHDIR};
-    if (exists $options->{EIPP_RES}) {
-        info "parsing EIPP results...";
-        open my $fh, '<', $tmpeipp
-          or error "failed to open $tmpeipp for reading: $!";
-        my $inst = 0;
-        my $pkg;
-        my $arch;
-        my $ver;
-        while (my $line = <$fh>) {
-            chomp $line;
-            if ($line ne "") {
-                if ($line =~ /^Status: .+/) {
-                    $inst = 1;
-                } elsif ($line =~ /^Package: (.*)/) {
-                    $pkg = $1;
-                } elsif ($line =~ /^Architecture: (.*)/) {
-                    $arch = $1;
-                } elsif ($line =~ /^Version: (.*)/) {
-                    $ver = $1;
-                }
-                next;
-            }
-            if ($inst == 0 && defined $pkg && defined $arch && defined $ver) {
-                push @{ $options->{EIPP_RES} }, ["$pkg:$arch", $ver];
-            }
-            $inst = 0;
-            undef $pkg;
-            undef $ver;
-        }
-        close $fh;
-        unlink $tmpeipp;
-    }
     return;
 }
 
 sub run_apt_download_progress {
     my $options = shift;
-    my %result  = shift;
+    my $tmplistofdebs;
     if ($options->{dryrun}) {
         info "simulate downloading packages with apt...";
     } else {
         info "downloading packages with apt...";
+        (undef, $tmplistofdebs) = tempfile(
+            "mmdebstrap.listofdebs.XXXXXXXXXXXX",
+            OPEN   => 0,
+            TMPDIR => 1
+        );
     }
-    return run_apt_progress({
+    run_apt_progress({
             ARGV => [
                 'apt-get',
                 '--yes',
                 '-oDebug::pkgDpkgPm=1',
                 '-oDir::Log=/dev/null',
-                $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
+                $options->{dryrun}
+                ? '-oAPT::Get::Simulate=true'
+                : "-oDpkg::Pre-Install-Pkgs::=cat > $tmplistofdebs",
                 @{ $options->{APT_ARGV} },
             ],
-            %result
         });
+    if ($tmplistofdebs) {
+        open my $fh, '<', $tmplistofdebs
+          or error "failed to open $tmplistofdebs for reading: $!";
+        my @listofdebs = <$fh>;
+        close $fh;
+        unlink $tmplistofdebs;
+        chomp(@listofdebs);
+        return @listofdebs;
+    }
+    return [];
 }
 
 sub run_chroot {
@@ -2050,21 +2023,14 @@ sub run_update() {
 sub run_download() {
     my $options = shift;
 
-    # We use /var/cache/apt/archives/ to figure out which packages apt chooses
-    # to install. That's why the directory must be empty if:
-    #  - /var/cache/apt/archives exists, and
-    #  - no simulation run is done, and
-    #  - the variant is not extract or custom or the number to be
-    #    installed packages not zero
-    #
     # In the future we want to replace downloading packages with "apt-get
-    # install --download-only" and installing them with dpkg by just installing
+    # install" and installing them with dpkg by just installing the essential
-    # the essential packages with apt from the outside with
+    # packages with apt from the outside with DPkg::Chroot-Directory.
-    # DPkg::Chroot-Directory. We are not doing that because then the preinst
+    # We are not doing that because then the preinst script of base-passwd will
-    # script of base-passwd will not be called early enough and packages will
+    # not be called early enough and packages will fail to install because they
-    # fail to install because they are missing /etc/passwd.
+    # are missing /etc/passwd.
     my @cached_debs = ();
-    my @dl_debs     = ();
+    my @dl_debs;
     if (
         !$options->{dryrun}
         && ((none { $_ eq $options->{variant} } ('extract', 'custom'))
@@ -2084,14 +2050,6 @@ sub run_download() {
             push @cached_debs, $deb;
         }
         closedir $dh;
-        if (scalar @cached_debs > 0) {
-            if (any { $_ eq 'download/empty' } @{ $options->{skip} }) {
-                info "skipping download/empty as requested";
-            } else {
-                error("/var/cache/apt/archives/ inside the chroot contains: "
-                      . (join ', ', (sort @cached_debs)));
-            }
-        }
     }
 
     # To figure out the right package set for the apt variant we can use:
@@ -2100,14 +2058,6 @@ sub run_download() {
     # apt and libapt treats apt as essential. If we want to install less
     # (essential variant) then we have to compute the package set ourselves.
     # Same if we want to install priority based variants.
-    my %result = ();
-    if (not $options->{dryrun}) {
-        # if there are already packages in /var/cache/apt/archives/,
-        # we need to know which are part of the solution by apt
-        if (scalar @cached_debs > 0) {
-            $result{EIPP_RES} = \@dl_debs;
-        }
-    }
     if (any { $_ eq $options->{variant} } ('extract', 'custom')) {
         if (scalar @{ $options->{include} } == 0) {
             info "nothing to download -- skipping...";
@@ -2126,11 +2076,10 @@ sub run_download() {
             }
         }
 
-        run_apt_download_progress({
+        @dl_debs = run_apt_download_progress({
                 APT_ARGV => @apt_argv,
                 dryrun   => $options->{dryrun},
             },
-            %result
         );
     } elsif ($options->{variant} eq 'apt') {
         # if we just want to install Essential:yes packages, apt and their
@@ -2146,11 +2095,10 @@ sub run_download() {
         #                  remind me in 5+ years that I said that after I wrote
         #                  in the bugreport: "Are you crazy?!? Nobody in his
         #                  right mind would even suggest depending on it!")
-        run_apt_download_progress({
+        @dl_debs = run_apt_download_progress({
                 APT_ARGV => ['dist-upgrade'],
                 dryrun   => $options->{dryrun},
             },
-            %result
         );
     } elsif (
         any { $_ eq $options->{variant} }
@@ -2160,7 +2108,7 @@ sub run_download() {
         # 17:27 < DonKult> (?essential includes 'apt' through)
         # 17:30 < josch> DonKult: no, because pkgCacheGen::ForceEssential ",";
         # 17:32 < DonKult> touché
-        run_apt_download_progress({
+        @dl_debs = run_apt_download_progress({
                 APT_ARGV => [
                     'install',
                     '?narrow('
@@ -2178,75 +2126,26 @@ sub run_download() {
                 ],
                 dryrun => $options->{dryrun},
             },
-            %result
         );
     } else {
         error "unknown variant: $options->{variant}";
     }
 
     my @essential_pkgs;
-    if (scalar @cached_debs > 0 && scalar @dl_debs > 0) {
+    # strip the the chroot directory from the filenames
-        my $archives = "/var/cache/apt/archives/";
+    foreach my $deb (@dl_debs) {
-        # for each package in @dl_debs, check if it's in
+        if (rindex $deb, $options->{root}, 0) {
-        # /var/cache/apt/archives/ and add it to @essential_pkgs
+            error "package file $deb not in chroot directory"
-        foreach my $p (@dl_debs) {
+              . " -- use copy:// instead of file://";
-            my ($pkg, $ver_epoch) = @{$p};
+        }
-            # apt appends the architecture at the end of the package name
+        if (-e $deb) {
-            ($pkg, my $arch) = split ':', $pkg, 2;
+            push @essential_pkgs, substr($deb, length($options->{root}));
-            # apt replaces the colon by its percent encoding %3a
-            my $ver = $ver_epoch;
-            $ver =~ s/:/%3a/;
-            # the architecture returned by apt is the native architecture.
-            # Since we don't know whether the package is architecture
-            # independent or not, we first try with the native arch and then
-            # with "all" and only error out if neither exists.
-            if (-e "$options->{root}/$archives/${pkg}_${ver}_$arch.deb") {
-                push @essential_pkgs, "$archives/${pkg}_${ver}_$arch.deb";
-            } elsif (-e "$options->{root}/$archives/${pkg}_${ver}_all.deb") {
-                push @essential_pkgs, "$archives/${pkg}_${ver}_all.deb";
         } else {
-                error(  "cannot find package for $pkg:$arch (= $ver_epoch) "
+            error "cannot find package file $deb";
-                      . "in /var/cache/apt/archives/");
         }
     }
-    } else {
-        # collect the .deb files that were downloaded by apt from the content
-        # of /var/cache/apt/archives/
-        if (!$options->{dryrun}) {
-            my $apt_archives = "/var/cache/apt/archives/";
-            opendir my $dh, "$options->{root}/$apt_archives"
-              or error "cannot read $apt_archives";
-            while (my $deb = readdir $dh) {
-                if ($deb !~ /\.deb$/) {
-                    next;
-                }
-                $deb = "$apt_archives/$deb";
-                if (!-f "$options->{root}/$deb") {
-                    next;
-                }
-                push @essential_pkgs, $deb;
-            }
-            closedir $dh;
 
-            if (scalar @essential_pkgs == 0) {
+    # Unpack order matters for e.g. timestamps on directories
-                # check if a file:// URI was used
-                open(my $pipe_apt, '-|', 'apt-get', 'indextargets', '--format',
-                    '$(URI)', 'Created-By: Packages')
-                  or error "cannot start apt-get indextargets: $!";
-                while (my $uri = <$pipe_apt>) {
-                    if ($uri =~ /^file:\/\//) {
-                        error
-                          "nothing got downloaded -- use copy:// instead of"
-                          . " file://";
-                    }
-                }
-                error "nothing got downloaded";
-            }
-        }
-    }
-    # Unpack order matters. Since we create this list using two different
-    # methods but we want both methods to have the same result, we sort the
-    # list before returning it.
     @essential_pkgs = sort @essential_pkgs;
 
     return (\@essential_pkgs, \@cached_debs);
@@ -6678,15 +6577,13 @@ the B<setup> step. This can be disabled using B<--skip=update>.
 
 =item B<download>
 
-Checks whether F</var/cache/apt/archives/> is empty. This can be disabled with
+In the B<extract> and B<custom> variants, C<apt-get install> is used to
-B<--skip=download/empty>. In the B<extract> and B<custom> variants, C<apt-get
+download all the packages requested via the B<--include> option. The B<apt>
---download-only install> is used to download all the packages requested via the
+variant uses the fact that libapt treats the C<apt> packages as implicitly
-B<--include> option. The B<apt> variant uses the fact that libapt treats the
+essential to download only all C<Essential:yes> packages plus apt using
-C<apt> packages as implicitly essential to download only all C<Essential:yes>
+C<apt-get dist-upgrade>. In the remaining variants, all Packages files
-packages plus apt using C<apt-get --download-only dist-upgrade>. In the
+downloaded by the B<update> step are inspected to find the C<Essential:yes>
-remaining variants, all Packages files downloaded by the B<update> step are
+package set as well as all packages of the required priority.
-inspected to find the C<Essential:yes> package set as well as all packages of
-the required priority.
 
 =item B<extract>
 
@@ -6932,7 +6829,7 @@ apt-cacher-ng, you can use the B<sync-in> and B<sync-out> special hooks to
 synchronize a directory outside the chroot with F</var/cache/apt/archives>
 inside the chroot.
 
-    $ mmdebstrap --variant=apt --skip=download/empty --skip=essential/unlink \
+    $ mmdebstrap --variant=apt --skip=essential/unlink \
         --setup-hook='mkdir -p ./cache "$1"/var/cache/apt/archives/' \
         --setup-hook='sync-in ./cache /var/cache/apt/archives/' \
         --customize-hook='sync-out /var/cache/apt/archives ./cache' \