Replace EDSP with EIPP usage obsoleting proxysolver

EIPP stands for "External Installation Planner Protocol" and is rather
similar to EDSP but with the clear advantage that we can extract the
information we need more easily as we can tell apt to write the file for
us rather than playing solver-in-the-middle and the problem space is
much smaller meaning less data for apt to generate and to pass through
our hands.

The idea here is simply that every package which doesn't have a Status
field in EIPP has the uninstalled status and the only reason its is part
of the EIPP request is that we want to change this by installing it.
That could be verified via the Install header at the start of the
request, but this commit doesn't implement that.

Note that this means we need "more" than the download-only mode can
provide: Either a simulation or "the real deal". Except we modify the
later to be a fancy no op.

coverage.sh

@@ -89,13 +89,6 @@ if [ ! -e shared/tarfilter ] || [ tarfilter -nt shared/tarfilter ]; then
 		cp -a /usr/bin/mmtarfilter shared/tarfilter
 	fi
 fi
-if [ ! -e shared/proxysolver ] || [ proxysolver -nt shared/proxysolver ]; then
-	if [ -e ./proxysolver ]; then
-		cp -a proxysolver shared
-	else
-		cp -a /usr/lib/apt/solvers/mmdebstrap-dump-solution shared/proxysolver
-	fi
-fi
 if [ ! -e shared/ldconfig.fakechroot ] || [ ldconfig.fakechroot -nt shared/ldconfig.fakechroot ]; then
 	if [ -e ./ldconfig.fakechroot ]; then
 		cp -a ldconfig.fakechroot shared
@@ -3760,4 +3753,4 @@ if [ "$((skipped+runtests))" -ne "$total" ]; then
 	exit 1
 fi
 
-rm shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/taridshift shared/tarfilter shared/proxysolver
+rm shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/taridshift shared/tarfilter

mmdebstrap

@@ -874,10 +874,10 @@ sub run_dpkg_progress {
 sub run_apt_progress {
     my $options = shift;
     my @debs    = @{ $options->{PKGS} // [] };
-    my $tmpedsp;
+    my $tmpeipp;
-    if (exists $options->{EDSP_RES}) {
+    if (exists $options->{EIPP_RES}) {
-        (undef, $tmpedsp) = tempfile(
+        (undef, $tmpeipp) = tempfile(
-            "mmdebstrap.edsp.XXXXXXXXXXXX",
+            "mmdebstrap.eipp.XXXXXXXXXXXX",
             OPEN   => 0,
             TMPDIR => 1
         );
@@ -885,16 +885,8 @@ sub run_apt_progress {
     my $get_exec = sub {
         my @prefix = ();
         my @opts   = ();
-        if (exists $options->{EDSP_RES}) {
+        if (exists $options->{EIPP_RES}) {
-            push @prefix, 'env', "APT_EDSP_DUMP_FILENAME=$tmpedsp";
+            push @opts, "-oDir::Log::Planner=$tmpeipp";
-            if (-e "./proxysolver") {
-                # for development purposes, use the current directory if it
-                # contains a file called proxysolver
-                push @opts, ("-oDir::Bin::solvers=" . getcwd()),
-                  '--solver=proxysolver';
-            } else {
-                push @opts, '--solver=mmdebstrap-dump-solution';
-            }
         }
         return (
             @prefix,
@@ -950,34 +942,37 @@ sub run_apt_progress {
         }
     };
     run_progress $get_exec, $line_handler, $line_has_error, $options->{CHDIR};
-    if (exists $options->{EDSP_RES}) {
+    if (exists $options->{EIPP_RES}) {
-        info "parsing EDSP results...";
+        info "parsing EIPP results...";
-        open my $fh, '<', $tmpedsp
+        open my $fh, '<', $tmpeipp
-          or error "failed to open $tmpedsp for reading: $!";
+          or error "failed to open $tmpeipp for reading: $!";
         my $inst = 0;
         my $pkg;
+        my $arch;
         my $ver;
         while (my $line = <$fh>) {
             chomp $line;
             if ($line ne "") {
-                if ($line =~ /^Install: \d+/) {
+                if ($line =~ /^Status: .+/) {
                     $inst = 1;
                 } elsif ($line =~ /^Package: (.*)/) {
                     $pkg = $1;
+                } elsif ($line =~ /^Architecture: (.*)/) {
+                    $arch = $1;
                 } elsif ($line =~ /^Version: (.*)/) {
                     $ver = $1;
                 }
                 next;
             }
-            if ($inst == 1 && defined $pkg && defined $ver) {
+            if ($inst == 0 && defined $pkg && defined $arch && defined $ver) {
-                push @{ $options->{EDSP_RES} }, [$pkg, $ver];
+                push @{ $options->{EIPP_RES} }, ["$pkg:$arch", $ver];
             }
             $inst = 0;
             undef $pkg;
             undef $ver;
         }
         close $fh;
-        unlink $tmpedsp;
+        unlink $tmpeipp;
     }
     return;
 }
@@ -2041,11 +2036,6 @@ sub run_download() {
     #  - the variant is not extract or custom or the number to be
     #    installed packages not zero
     #
-    # We could also unconditionally use the proxysolver and then "apt-get
-    # download" any missing packages but using the proxysolver requires
-    # /usr/lib/apt/solvers/apt from the apt-utils package and we want to avoid
-    # that dependency.
-    #
     # In the future we want to replace downloading packages with "apt-get
     # install --download-only" and installing them with dpkg by just installing
     # the essential packages with apt from the outside with
@@ -2112,9 +2102,9 @@ sub run_download() {
             info "simulate downloading packages with apt...";
         } else {
             # if there are already packages in /var/cache/apt/archives/, we
-            # need to use our proxysolver to obtain the solution chosen by apt
+            # need to know which are part of the solution by apt
             if (scalar @cached_debs > 0) {
-                $result{EDSP_RES} = \@dl_debs;
+                $result{EIPP_RES} = \@dl_debs;
             }
             info "downloading packages with apt...";
         }
@@ -2122,7 +2112,8 @@ sub run_download() {
                 ARGV => [
                     'apt-get',
                     '--yes',
-                    '-oApt::Get::Download-Only=true',
+                    '-oDebug::pkgDpkgPm=1',
+                    '-oDir::Log=/dev/null',
                     $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
                     'install'
                 ],
@@ -2148,9 +2139,9 @@ sub run_download() {
             info "simulate downloading packages with apt...";
         } else {
             # if there are already packages in /var/cache/apt/archives/, we
-            # need to use our proxysolver to obtain the solution chosen by apt
+            # need to know which are part of the solution by apt
             if (scalar @cached_debs > 0) {
-                $result{EDSP_RES} = \@dl_debs;
+                $result{EIPP_RES} = \@dl_debs;
             }
             info "downloading packages with apt...";
         }
@@ -2158,7 +2149,8 @@ sub run_download() {
                 ARGV => [
                     'apt-get',
                     '--yes',
-                    '-oApt::Get::Download-Only=true',
+                    '-oDebug::pkgDpkgPm=1',
+                    '-oDir::Log=/dev/null',
                     $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
                     'dist-upgrade'
                 ],
@@ -2177,9 +2169,9 @@ sub run_download() {
             info "simulate downloading packages with apt...";
         } else {
             # if there are already packages in /var/cache/apt/archives/, we
-            # need to use our proxysolver to obtain the solution chosen by apt
+            # need to know which are part of the solution by apt
             if (scalar @cached_debs > 0) {
-                $result{EDSP_RES} = \@dl_debs;
+                $result{EIPP_RES} = \@dl_debs;
             }
             info "downloading packages with apt...";
         }
@@ -2187,7 +2179,8 @@ sub run_download() {
                 ARGV => [
                     'apt-get',
                     '--yes',
-                    '-oApt::Get::Download-Only=true',
+                    '-oDebug::pkgDpkgPm=1',
+                    '-oDir::Log=/dev/null',
                     $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
                     'install',
                     '?narrow('

proxysolver

@@ -1,56 +0,0 @@
-#!/usr/bin/env python3
-#
-# This script is in the public domain
-#
-# Author: Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
-#
-# thin layer around /usr/lib/apt/solvers/apt, so that we can capture the solver
-# result
-#
-# we set Debug::EDSP::WriteSolution=yes so that Install stanzas also come with
-# Package and Version fields. That way, we do not also have to parse the EDSP
-# request and spend time matching ID numbers
-
-import subprocess
-import sys
-import os
-import getpass
-
-if not os.path.exists("/usr/lib/apt/solvers/apt"):
-    print(
-        """Error: ERR_NO_SOLVER
-Message: The external apt solver doesn't exist. You must install the apt-utils package.
-"""
-    )
-    exit()
-
-fname = os.environ.get("APT_EDSP_DUMP_FILENAME")
-if fname is None:
-    print(
-        """Error: ERR_NO_FILENAME
-Message: You have to set the environment variable APT_EDSP_DUMP_FILENAME
- to a valid filename to store the dump of EDSP solver input in.
- For example with: export APT_EDSP_DUMP_FILENAME=/tmp/dump.edsp
-"""
-    )
-    exit()
-
-try:
-    with open(fname, "w") as f:
-        with subprocess.Popen(
-            ["/usr/lib/apt/solvers/apt", "-oDebug::EDSP::WriteSolution=yes"],
-            stdin=sys.stdin.fileno(),
-            stdout=subprocess.PIPE,
-            bufsize=0,  # unbuffered
-            text=True,  # open in text mode
-        ) as p:
-            for line in p.stdout:
-                print(line, end="")
-                f.write(line)
-except (FileNotFoundError, PermissionError) as e:
-    print(
-        """Error: ERR_CREATE_FILE
-Message: Writing EDSP solver input to file '%s' failed as it couldn't be created!
-"""
-        % fname
-    )