From 8b58dc583e589d4165445ed06930161f071f553d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 22 Apr 2022 22:08:27 +0200 Subject: [PATCH] Replace EDSP with EIPP usage obsoleting proxysolver EIPP stands for "External Installation Planner Protocol" and is rather similar to EDSP but with the clear advantage that we can extract the information we need more easily as we can tell apt to write the file for us rather than playing solver-in-the-middle and the problem space is much smaller meaning less data for apt to generate and to pass through our hands. The idea here is simply that every package which doesn't have a Status field in EIPP has the uninstalled status and the only reason its is part of the EIPP request is that we want to change this by installing it. That could be verified via the Install header at the start of the request, but this commit doesn't implement that. Note that this means we need "more" than the download-only mode can provide: Either a simulation or "the real deal". Except we modify the later to be a fancy no op. --- coverage.sh | 9 +------- mmdebstrap | 65 ++++++++++++++++++++++++----------------------------- proxysolver | 56 --------------------------------------------- 3 files changed, 30 insertions(+), 100 deletions(-) delete mode 100755 proxysolver diff --git a/coverage.sh b/coverage.sh index a05b75a..673723d 100755 --- a/coverage.sh +++ b/coverage.sh @@ -89,13 +89,6 @@ if [ ! -e shared/tarfilter ] || [ tarfilter -nt shared/tarfilter ]; then cp -a /usr/bin/mmtarfilter shared/tarfilter fi fi -if [ ! -e shared/proxysolver ] || [ proxysolver -nt shared/proxysolver ]; then - if [ -e ./proxysolver ]; then - cp -a proxysolver shared - else - cp -a /usr/lib/apt/solvers/mmdebstrap-dump-solution shared/proxysolver - fi -fi if [ ! -e shared/ldconfig.fakechroot ] || [ ldconfig.fakechroot -nt shared/ldconfig.fakechroot ]; then if [ -e ./ldconfig.fakechroot ]; then cp -a ldconfig.fakechroot shared @@ -3760,4 +3753,4 @@ if [ "$((skipped+runtests))" -ne "$total" ]; then exit 1 fi -rm shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/taridshift shared/tarfilter shared/proxysolver +rm shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/taridshift shared/tarfilter diff --git a/mmdebstrap b/mmdebstrap index f5430ed..7094a51 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -874,10 +874,10 @@ sub run_dpkg_progress { sub run_apt_progress { my $options = shift; my @debs = @{ $options->{PKGS} // [] }; - my $tmpedsp; - if (exists $options->{EDSP_RES}) { - (undef, $tmpedsp) = tempfile( - "mmdebstrap.edsp.XXXXXXXXXXXX", + my $tmpeipp; + if (exists $options->{EIPP_RES}) { + (undef, $tmpeipp) = tempfile( + "mmdebstrap.eipp.XXXXXXXXXXXX", OPEN => 0, TMPDIR => 1 ); @@ -885,16 +885,8 @@ sub run_apt_progress { my $get_exec = sub { my @prefix = (); my @opts = (); - if (exists $options->{EDSP_RES}) { - push @prefix, 'env', "APT_EDSP_DUMP_FILENAME=$tmpedsp"; - if (-e "./proxysolver") { - # for development purposes, use the current directory if it - # contains a file called proxysolver - push @opts, ("-oDir::Bin::solvers=" . getcwd()), - '--solver=proxysolver'; - } else { - push @opts, '--solver=mmdebstrap-dump-solution'; - } + if (exists $options->{EIPP_RES}) { + push @opts, "-oDir::Log::Planner=$tmpeipp"; } return ( @prefix, @@ -950,34 +942,37 @@ sub run_apt_progress { } }; run_progress $get_exec, $line_handler, $line_has_error, $options->{CHDIR}; - if (exists $options->{EDSP_RES}) { - info "parsing EDSP results..."; - open my $fh, '<', $tmpedsp - or error "failed to open $tmpedsp for reading: $!"; + if (exists $options->{EIPP_RES}) { + info "parsing EIPP results..."; + open my $fh, '<', $tmpeipp + or error "failed to open $tmpeipp for reading: $!"; my $inst = 0; my $pkg; + my $arch; my $ver; while (my $line = <$fh>) { chomp $line; if ($line ne "") { - if ($line =~ /^Install: \d+/) { + if ($line =~ /^Status: .+/) { $inst = 1; } elsif ($line =~ /^Package: (.*)/) { $pkg = $1; + } elsif ($line =~ /^Architecture: (.*)/) { + $arch = $1; } elsif ($line =~ /^Version: (.*)/) { $ver = $1; } next; } - if ($inst == 1 && defined $pkg && defined $ver) { - push @{ $options->{EDSP_RES} }, [$pkg, $ver]; + if ($inst == 0 && defined $pkg && defined $arch && defined $ver) { + push @{ $options->{EIPP_RES} }, ["$pkg:$arch", $ver]; } $inst = 0; undef $pkg; undef $ver; } close $fh; - unlink $tmpedsp; + unlink $tmpeipp; } return; } @@ -2041,11 +2036,6 @@ sub run_download() { # - the variant is not extract or custom or the number to be # installed packages not zero # - # We could also unconditionally use the proxysolver and then "apt-get - # download" any missing packages but using the proxysolver requires - # /usr/lib/apt/solvers/apt from the apt-utils package and we want to avoid - # that dependency. - # # In the future we want to replace downloading packages with "apt-get # install --download-only" and installing them with dpkg by just installing # the essential packages with apt from the outside with @@ -2112,9 +2102,9 @@ sub run_download() { info "simulate downloading packages with apt..."; } else { # if there are already packages in /var/cache/apt/archives/, we - # need to use our proxysolver to obtain the solution chosen by apt + # need to know which are part of the solution by apt if (scalar @cached_debs > 0) { - $result{EDSP_RES} = \@dl_debs; + $result{EIPP_RES} = \@dl_debs; } info "downloading packages with apt..."; } @@ -2122,7 +2112,8 @@ sub run_download() { ARGV => [ 'apt-get', '--yes', - '-oApt::Get::Download-Only=true', + '-oDebug::pkgDpkgPm=1', + '-oDir::Log=/dev/null', $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), 'install' ], @@ -2148,9 +2139,9 @@ sub run_download() { info "simulate downloading packages with apt..."; } else { # if there are already packages in /var/cache/apt/archives/, we - # need to use our proxysolver to obtain the solution chosen by apt + # need to know which are part of the solution by apt if (scalar @cached_debs > 0) { - $result{EDSP_RES} = \@dl_debs; + $result{EIPP_RES} = \@dl_debs; } info "downloading packages with apt..."; } @@ -2158,7 +2149,8 @@ sub run_download() { ARGV => [ 'apt-get', '--yes', - '-oApt::Get::Download-Only=true', + '-oDebug::pkgDpkgPm=1', + '-oDir::Log=/dev/null', $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), 'dist-upgrade' ], @@ -2177,9 +2169,9 @@ sub run_download() { info "simulate downloading packages with apt..."; } else { # if there are already packages in /var/cache/apt/archives/, we - # need to use our proxysolver to obtain the solution chosen by apt + # need to know which are part of the solution by apt if (scalar @cached_debs > 0) { - $result{EDSP_RES} = \@dl_debs; + $result{EIPP_RES} = \@dl_debs; } info "downloading packages with apt..."; } @@ -2187,7 +2179,8 @@ sub run_download() { ARGV => [ 'apt-get', '--yes', - '-oApt::Get::Download-Only=true', + '-oDebug::pkgDpkgPm=1', + '-oDir::Log=/dev/null', $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), 'install', '?narrow(' diff --git a/proxysolver b/proxysolver deleted file mode 100755 index 5cd51fa..0000000 --- a/proxysolver +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -# -# This script is in the public domain -# -# Author: Johannes Schauer Marin Rodrigues -# -# thin layer around /usr/lib/apt/solvers/apt, so that we can capture the solver -# result -# -# we set Debug::EDSP::WriteSolution=yes so that Install stanzas also come with -# Package and Version fields. That way, we do not also have to parse the EDSP -# request and spend time matching ID numbers - -import subprocess -import sys -import os -import getpass - -if not os.path.exists("/usr/lib/apt/solvers/apt"): - print( - """Error: ERR_NO_SOLVER -Message: The external apt solver doesn't exist. You must install the apt-utils package. -""" - ) - exit() - -fname = os.environ.get("APT_EDSP_DUMP_FILENAME") -if fname is None: - print( - """Error: ERR_NO_FILENAME -Message: You have to set the environment variable APT_EDSP_DUMP_FILENAME - to a valid filename to store the dump of EDSP solver input in. - For example with: export APT_EDSP_DUMP_FILENAME=/tmp/dump.edsp -""" - ) - exit() - -try: - with open(fname, "w") as f: - with subprocess.Popen( - ["/usr/lib/apt/solvers/apt", "-oDebug::EDSP::WriteSolution=yes"], - stdin=sys.stdin.fileno(), - stdout=subprocess.PIPE, - bufsize=0, # unbuffered - text=True, # open in text mode - ) as p: - for line in p.stdout: - print(line, end="") - f.write(line) -except (FileNotFoundError, PermissionError) as e: - print( - """Error: ERR_CREATE_FILE -Message: Writing EDSP solver input to file '%s' failed as it couldn't be created! -""" - % fname - )