From ee142d52a536bae813b3828ab14ec886bb785a4f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 22 Apr 2022 22:08:27 +0200 Subject: [PATCH 1/6] Replace EDSP with EIPP usage obsoleting proxysolver EIPP stands for "External Installation Planner Protocol" and is rather similar to EDSP but with the clear advantage that we can extract the information we need more easily as we can tell apt to write the file for us rather than playing solver-in-the-middle and the problem space is much smaller meaning less data for apt to generate and to pass through our hands. The idea here is simply that every package which doesn't have a Status field in EIPP has the uninstalled status and the only reason its is part of the EIPP request is that we want to change this by installing it. That could be verified via the Install header at the start of the request, but this commit doesn't implement that. Note that this means we need "more" than the download-only mode can provide: Either a simulation or "the real deal". Except we modify the later to be a fancy no op. --- coverage.sh | 9 +------- mmdebstrap | 65 ++++++++++++++++++++++++----------------------------- proxysolver | 56 --------------------------------------------- 3 files changed, 30 insertions(+), 100 deletions(-) delete mode 100755 proxysolver diff --git a/coverage.sh b/coverage.sh index a05b75a..673723d 100755 --- a/coverage.sh +++ b/coverage.sh @@ -89,13 +89,6 @@ if [ ! -e shared/tarfilter ] || [ tarfilter -nt shared/tarfilter ]; then cp -a /usr/bin/mmtarfilter shared/tarfilter fi fi -if [ ! -e shared/proxysolver ] || [ proxysolver -nt shared/proxysolver ]; then - if [ -e ./proxysolver ]; then - cp -a proxysolver shared - else - cp -a /usr/lib/apt/solvers/mmdebstrap-dump-solution shared/proxysolver - fi -fi if [ ! -e shared/ldconfig.fakechroot ] || [ ldconfig.fakechroot -nt shared/ldconfig.fakechroot ]; then if [ -e ./ldconfig.fakechroot ]; then cp -a ldconfig.fakechroot shared @@ -3760,4 +3753,4 @@ if [ "$((skipped+runtests))" -ne "$total" ]; then exit 1 fi -rm shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/taridshift shared/tarfilter shared/proxysolver +rm shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/taridshift shared/tarfilter diff --git a/mmdebstrap b/mmdebstrap index 5c12763..7c2467c 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -874,10 +874,10 @@ sub run_dpkg_progress { sub run_apt_progress { my $options = shift; my @debs = @{ $options->{PKGS} // [] }; - my $tmpedsp; - if (exists $options->{EDSP_RES}) { - (undef, $tmpedsp) = tempfile( - "mmdebstrap.edsp.XXXXXXXXXXXX", + my $tmpeipp; + if (exists $options->{EIPP_RES}) { + (undef, $tmpeipp) = tempfile( + "mmdebstrap.eipp.XXXXXXXXXXXX", OPEN => 0, TMPDIR => 1 ); @@ -885,16 +885,8 @@ sub run_apt_progress { my $get_exec = sub { my @prefix = (); my @opts = (); - if (exists $options->{EDSP_RES}) { - push @prefix, 'env', "APT_EDSP_DUMP_FILENAME=$tmpedsp"; - if (-e "./proxysolver") { - # for development purposes, use the current directory if it - # contains a file called proxysolver - push @opts, ("-oDir::Bin::solvers=" . getcwd()), - '--solver=proxysolver'; - } else { - push @opts, '--solver=mmdebstrap-dump-solution'; - } + if (exists $options->{EIPP_RES}) { + push @opts, "-oDir::Log::Planner=$tmpeipp"; } return ( @prefix, @@ -950,34 +942,37 @@ sub run_apt_progress { } }; run_progress $get_exec, $line_handler, $line_has_error, $options->{CHDIR}; - if (exists $options->{EDSP_RES}) { - info "parsing EDSP results..."; - open my $fh, '<', $tmpedsp - or error "failed to open $tmpedsp for reading: $!"; + if (exists $options->{EIPP_RES}) { + info "parsing EIPP results..."; + open my $fh, '<', $tmpeipp + or error "failed to open $tmpeipp for reading: $!"; my $inst = 0; my $pkg; + my $arch; my $ver; while (my $line = <$fh>) { chomp $line; if ($line ne "") { - if ($line =~ /^Install: \d+/) { + if ($line =~ /^Status: .+/) { $inst = 1; } elsif ($line =~ /^Package: (.*)/) { $pkg = $1; + } elsif ($line =~ /^Architecture: (.*)/) { + $arch = $1; } elsif ($line =~ /^Version: (.*)/) { $ver = $1; } next; } - if ($inst == 1 && defined $pkg && defined $ver) { - push @{ $options->{EDSP_RES} }, [$pkg, $ver]; + if ($inst == 0 && defined $pkg && defined $arch && defined $ver) { + push @{ $options->{EIPP_RES} }, ["$pkg:$arch", $ver]; } $inst = 0; undef $pkg; undef $ver; } close $fh; - unlink $tmpedsp; + unlink $tmpeipp; } return; } @@ -2041,11 +2036,6 @@ sub run_download() { # - the variant is not extract or custom or the number to be # installed packages not zero # - # We could also unconditionally use the proxysolver and then "apt-get - # download" any missing packages but using the proxysolver requires - # /usr/lib/apt/solvers/apt from the apt-utils package and we want to avoid - # that dependency. - # # In the future we want to replace downloading packages with "apt-get # install --download-only" and installing them with dpkg by just installing # the essential packages with apt from the outside with @@ -2112,9 +2102,9 @@ sub run_download() { info "simulate downloading packages with apt..."; } else { # if there are already packages in /var/cache/apt/archives/, we - # need to use our proxysolver to obtain the solution chosen by apt + # need to know which are part of the solution by apt if (scalar @cached_debs > 0) { - $result{EDSP_RES} = \@dl_debs; + $result{EIPP_RES} = \@dl_debs; } info "downloading packages with apt..."; } @@ -2122,7 +2112,8 @@ sub run_download() { ARGV => [ 'apt-get', '--yes', - '-oApt::Get::Download-Only=true', + '-oDebug::pkgDpkgPm=1', + '-oDir::Log=/dev/null', $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), 'install' ], @@ -2148,9 +2139,9 @@ sub run_download() { info "simulate downloading packages with apt..."; } else { # if there are already packages in /var/cache/apt/archives/, we - # need to use our proxysolver to obtain the solution chosen by apt + # need to know which are part of the solution by apt if (scalar @cached_debs > 0) { - $result{EDSP_RES} = \@dl_debs; + $result{EIPP_RES} = \@dl_debs; } info "downloading packages with apt..."; } @@ -2158,7 +2149,8 @@ sub run_download() { ARGV => [ 'apt-get', '--yes', - '-oApt::Get::Download-Only=true', + '-oDebug::pkgDpkgPm=1', + '-oDir::Log=/dev/null', $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), 'dist-upgrade' ], @@ -2177,9 +2169,9 @@ sub run_download() { info "simulate downloading packages with apt..."; } else { # if there are already packages in /var/cache/apt/archives/, we - # need to use our proxysolver to obtain the solution chosen by apt + # need to know which are part of the solution by apt if (scalar @cached_debs > 0) { - $result{EDSP_RES} = \@dl_debs; + $result{EIPP_RES} = \@dl_debs; } info "downloading packages with apt..."; } @@ -2187,7 +2179,8 @@ sub run_download() { ARGV => [ 'apt-get', '--yes', - '-oApt::Get::Download-Only=true', + '-oDebug::pkgDpkgPm=1', + '-oDir::Log=/dev/null', $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), 'install', '?narrow(' diff --git a/proxysolver b/proxysolver deleted file mode 100755 index 5cd51fa..0000000 --- a/proxysolver +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -# -# This script is in the public domain -# -# Author: Johannes Schauer Marin Rodrigues -# -# thin layer around /usr/lib/apt/solvers/apt, so that we can capture the solver -# result -# -# we set Debug::EDSP::WriteSolution=yes so that Install stanzas also come with -# Package and Version fields. That way, we do not also have to parse the EDSP -# request and spend time matching ID numbers - -import subprocess -import sys -import os -import getpass - -if not os.path.exists("/usr/lib/apt/solvers/apt"): - print( - """Error: ERR_NO_SOLVER -Message: The external apt solver doesn't exist. You must install the apt-utils package. -""" - ) - exit() - -fname = os.environ.get("APT_EDSP_DUMP_FILENAME") -if fname is None: - print( - """Error: ERR_NO_FILENAME -Message: You have to set the environment variable APT_EDSP_DUMP_FILENAME - to a valid filename to store the dump of EDSP solver input in. - For example with: export APT_EDSP_DUMP_FILENAME=/tmp/dump.edsp -""" - ) - exit() - -try: - with open(fname, "w") as f: - with subprocess.Popen( - ["/usr/lib/apt/solvers/apt", "-oDebug::EDSP::WriteSolution=yes"], - stdin=sys.stdin.fileno(), - stdout=subprocess.PIPE, - bufsize=0, # unbuffered - text=True, # open in text mode - ) as p: - for line in p.stdout: - print(line, end="") - f.write(line) -except (FileNotFoundError, PermissionError) as e: - print( - """Error: ERR_CREATE_FILE -Message: Writing EDSP solver input to file '%s' failed as it couldn't be created! -""" - % fname - ) -- 2.39.5 From 75e5a14e6d3791a87ac071a67b0823eebe3129bb Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 23 Apr 2022 13:55:54 +0200 Subject: [PATCH 2/6] Factor out downloading packages with apt --- mmdebstrap | 116 ++++++++++++++++++++++------------------------------- 1 file changed, 49 insertions(+), 67 deletions(-) diff --git a/mmdebstrap b/mmdebstrap index 7c2467c..a57239d 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -977,6 +977,27 @@ sub run_apt_progress { return; } +sub run_apt_download_progress { + my $options = shift; + my %result = shift; + if ($options->{dryrun}) { + info "simulate downloading packages with apt..."; + } else { + info "downloading packages with apt..."; + } + return run_apt_progress({ + ARGV => [ + 'apt-get', + '--yes', + '-oDebug::pkgDpkgPm=1', + '-oDir::Log=/dev/null', + $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), + @{ $options->{APT_ARGV} }, + ], + %result + }); +} + sub run_chroot { my $cmd = shift; my $options = shift; @@ -2079,12 +2100,20 @@ sub run_download() { # apt and libapt treats apt as essential. If we want to install less # (essential variant) then we have to compute the package set ourselves. # Same if we want to install priority based variants. + my %result = (); + if (not $options->{dryrun}) { + # if there are already packages in /var/cache/apt/archives/, + # we need to know which are part of the solution by apt + if (scalar @cached_debs > 0) { + $result{EIPP_RES} = \@dl_debs; + } + } if (any { $_ eq $options->{variant} } ('extract', 'custom')) { if (scalar @{ $options->{include} } == 0) { info "nothing to download -- skipping..."; return ([], []); } - my %pkgs_to_install; + my @apt_argv = ['install']; for my $incl (@{ $options->{include} }) { for my $pkg (split /[,\s]+/, $incl) { # strip leading and trailing whitespace @@ -2093,33 +2122,16 @@ sub run_download() { if ($pkg eq '') { next; } - $pkgs_to_install{$pkg} = (); + push @apt_argv, $pkg; } } - my %result = (); - if ($options->{dryrun}) { - info "simulate downloading packages with apt..."; - } else { - # if there are already packages in /var/cache/apt/archives/, we - # need to know which are part of the solution by apt - if (scalar @cached_debs > 0) { - $result{EIPP_RES} = \@dl_debs; - } - info "downloading packages with apt..."; - } - run_apt_progress({ - ARGV => [ - 'apt-get', - '--yes', - '-oDebug::pkgDpkgPm=1', - '-oDir::Log=/dev/null', - $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), - 'install' - ], - PKGS => [keys %pkgs_to_install], - %result - }); + run_apt_download_progress({ + APT_ARGV => @apt_argv, + dryrun => $options->{dryrun}, + }, + %result + ); } elsif ($options->{variant} eq 'apt') { # if we just want to install Essential:yes packages, apt and their # dependencies then we can make use of libapt treating apt as @@ -2134,28 +2146,12 @@ sub run_download() { # remind me in 5+ years that I said that after I wrote # in the bugreport: "Are you crazy?!? Nobody in his # right mind would even suggest depending on it!") - my %result = (); - if ($options->{dryrun}) { - info "simulate downloading packages with apt..."; - } else { - # if there are already packages in /var/cache/apt/archives/, we - # need to know which are part of the solution by apt - if (scalar @cached_debs > 0) { - $result{EIPP_RES} = \@dl_debs; - } - info "downloading packages with apt..."; - } - run_apt_progress({ - ARGV => [ - 'apt-get', - '--yes', - '-oDebug::pkgDpkgPm=1', - '-oDir::Log=/dev/null', - $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), - 'dist-upgrade' - ], - %result - }); + run_apt_download_progress({ + APT_ARGV => ['dist-upgrade'], + dryrun => $options->{dryrun}, + }, + %result + ); } elsif ( any { $_ eq $options->{variant} } ('essential', 'standard', 'important', 'required', 'buildd') @@ -2164,24 +2160,8 @@ sub run_download() { # 17:27 < DonKult> (?essential includes 'apt' through) # 17:30 < josch> DonKult: no, because pkgCacheGen::ForceEssential ","; # 17:32 < DonKult> touché - my %result = (); - if ($options->{dryrun}) { - info "simulate downloading packages with apt..."; - } else { - # if there are already packages in /var/cache/apt/archives/, we - # need to know which are part of the solution by apt - if (scalar @cached_debs > 0) { - $result{EIPP_RES} = \@dl_debs; - } - info "downloading packages with apt..."; - } - run_apt_progress({ - ARGV => [ - 'apt-get', - '--yes', - '-oDebug::pkgDpkgPm=1', - '-oDir::Log=/dev/null', - $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), + run_apt_download_progress({ + APT_ARGV => [ 'install', '?narrow(' . ( @@ -2196,8 +2176,10 @@ sub run_download() { . $options->{nativearch} . '),?essential)' ], - %result - }); + dryrun => $options->{dryrun}, + }, + %result + ); } else { error "unknown variant: $options->{variant}"; } -- 2.39.5 From 3ceebe7e07cf6ef482c996bd4d8e45f7545acf80 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 23 Apr 2022 16:48:31 +0200 Subject: [PATCH 3/6] Ask apt to give us the deb filenames directly Guessing filenames is boring. What if we could ask apt to tell us which debs it downloaded (or found lying around elsewhere) directly? Turns out we can rather easily avoiding a bunch of guesswork. --- coverage.sh | 2 +- mmdebstrap | 203 +++++++++++++--------------------------------------- 2 files changed, 51 insertions(+), 154 deletions(-) diff --git a/coverage.sh b/coverage.sh index 673723d..dbcf26f 100755 --- a/coverage.sh +++ b/coverage.sh @@ -3072,7 +3072,7 @@ $CMD \$include --mode=$defaultmode --variant=$variant \ --setup-hook='sync-in "'"\$tmpdir"'" /var/cache/apt/archives/partial' \ $DEFAULT_DIST - $mirror > test1.tar cmp orig.tar test1.tar -$CMD \$include --mode=$defaultmode --variant=$variant --skip=download/empty \ +$CMD \$include --mode=$defaultmode --variant=$variant \ --customize-hook='touch "\$1"/var/cache/apt/archives/partial' \ --setup-hook='mkdir -p "\$1"/var/cache/apt/archives/' \ --setup-hook='sync-in "'"\$tmpdir"'" /var/cache/apt/archives/' \ diff --git a/mmdebstrap b/mmdebstrap index a57239d..8765b15 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -872,22 +872,11 @@ sub run_dpkg_progress { } sub run_apt_progress { - my $options = shift; - my @debs = @{ $options->{PKGS} // [] }; - my $tmpeipp; - if (exists $options->{EIPP_RES}) { - (undef, $tmpeipp) = tempfile( - "mmdebstrap.eipp.XXXXXXXXXXXX", - OPEN => 0, - TMPDIR => 1 - ); - } + my $options = shift; + my @debs = @{ $options->{PKGS} // [] }; my $get_exec = sub { my @prefix = (); my @opts = (); - if (exists $options->{EIPP_RES}) { - push @opts, "-oDir::Log::Planner=$tmpeipp"; - } return ( @prefix, @{ $options->{ARGV} }, @@ -942,60 +931,44 @@ sub run_apt_progress { } }; run_progress $get_exec, $line_handler, $line_has_error, $options->{CHDIR}; - if (exists $options->{EIPP_RES}) { - info "parsing EIPP results..."; - open my $fh, '<', $tmpeipp - or error "failed to open $tmpeipp for reading: $!"; - my $inst = 0; - my $pkg; - my $arch; - my $ver; - while (my $line = <$fh>) { - chomp $line; - if ($line ne "") { - if ($line =~ /^Status: .+/) { - $inst = 1; - } elsif ($line =~ /^Package: (.*)/) { - $pkg = $1; - } elsif ($line =~ /^Architecture: (.*)/) { - $arch = $1; - } elsif ($line =~ /^Version: (.*)/) { - $ver = $1; - } - next; - } - if ($inst == 0 && defined $pkg && defined $arch && defined $ver) { - push @{ $options->{EIPP_RES} }, ["$pkg:$arch", $ver]; - } - $inst = 0; - undef $pkg; - undef $ver; - } - close $fh; - unlink $tmpeipp; - } return; } sub run_apt_download_progress { my $options = shift; - my %result = shift; + my $tmplistofdebs; if ($options->{dryrun}) { info "simulate downloading packages with apt..."; } else { info "downloading packages with apt..."; + (undef, $tmplistofdebs) = tempfile( + "mmdebstrap.listofdebs.XXXXXXXXXXXX", + OPEN => 0, + TMPDIR => 1 + ); } - return run_apt_progress({ + run_apt_progress({ ARGV => [ 'apt-get', '--yes', '-oDebug::pkgDpkgPm=1', '-oDir::Log=/dev/null', - $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), + $options->{dryrun} + ? '-oAPT::Get::Simulate=true' + : "-oDpkg::Pre-Install-Pkgs::=cat > $tmplistofdebs", @{ $options->{APT_ARGV} }, ], - %result }); + if ($tmplistofdebs) { + open my $fh, '<', $tmplistofdebs + or error "failed to open $tmplistofdebs for reading: $!"; + my @listofdebs = <$fh>; + close $fh; + unlink $tmplistofdebs; + chomp(@listofdebs); + return @listofdebs; + } + return []; } sub run_chroot { @@ -2050,21 +2023,14 @@ sub run_update() { sub run_download() { my $options = shift; - # We use /var/cache/apt/archives/ to figure out which packages apt chooses - # to install. That's why the directory must be empty if: - # - /var/cache/apt/archives exists, and - # - no simulation run is done, and - # - the variant is not extract or custom or the number to be - # installed packages not zero - # # In the future we want to replace downloading packages with "apt-get - # install --download-only" and installing them with dpkg by just installing - # the essential packages with apt from the outside with - # DPkg::Chroot-Directory. We are not doing that because then the preinst - # script of base-passwd will not be called early enough and packages will - # fail to install because they are missing /etc/passwd. + # install" and installing them with dpkg by just installing the essential + # packages with apt from the outside with DPkg::Chroot-Directory. + # We are not doing that because then the preinst script of base-passwd will + # not be called early enough and packages will fail to install because they + # are missing /etc/passwd. my @cached_debs = (); - my @dl_debs = (); + my @dl_debs; if ( !$options->{dryrun} && ((none { $_ eq $options->{variant} } ('extract', 'custom')) @@ -2084,14 +2050,6 @@ sub run_download() { push @cached_debs, $deb; } closedir $dh; - if (scalar @cached_debs > 0) { - if (any { $_ eq 'download/empty' } @{ $options->{skip} }) { - info "skipping download/empty as requested"; - } else { - error("/var/cache/apt/archives/ inside the chroot contains: " - . (join ', ', (sort @cached_debs))); - } - } } # To figure out the right package set for the apt variant we can use: @@ -2100,14 +2058,6 @@ sub run_download() { # apt and libapt treats apt as essential. If we want to install less # (essential variant) then we have to compute the package set ourselves. # Same if we want to install priority based variants. - my %result = (); - if (not $options->{dryrun}) { - # if there are already packages in /var/cache/apt/archives/, - # we need to know which are part of the solution by apt - if (scalar @cached_debs > 0) { - $result{EIPP_RES} = \@dl_debs; - } - } if (any { $_ eq $options->{variant} } ('extract', 'custom')) { if (scalar @{ $options->{include} } == 0) { info "nothing to download -- skipping..."; @@ -2126,11 +2076,10 @@ sub run_download() { } } - run_apt_download_progress({ + @dl_debs = run_apt_download_progress({ APT_ARGV => @apt_argv, dryrun => $options->{dryrun}, }, - %result ); } elsif ($options->{variant} eq 'apt') { # if we just want to install Essential:yes packages, apt and their @@ -2146,11 +2095,10 @@ sub run_download() { # remind me in 5+ years that I said that after I wrote # in the bugreport: "Are you crazy?!? Nobody in his # right mind would even suggest depending on it!") - run_apt_download_progress({ + @dl_debs = run_apt_download_progress({ APT_ARGV => ['dist-upgrade'], dryrun => $options->{dryrun}, }, - %result ); } elsif ( any { $_ eq $options->{variant} } @@ -2160,7 +2108,7 @@ sub run_download() { # 17:27 < DonKult> (?essential includes 'apt' through) # 17:30 < josch> DonKult: no, because pkgCacheGen::ForceEssential ","; # 17:32 < DonKult> touché - run_apt_download_progress({ + @dl_debs = run_apt_download_progress({ APT_ARGV => [ 'install', '?narrow(' @@ -2178,75 +2126,26 @@ sub run_download() { ], dryrun => $options->{dryrun}, }, - %result ); } else { error "unknown variant: $options->{variant}"; } my @essential_pkgs; - if (scalar @cached_debs > 0 && scalar @dl_debs > 0) { - my $archives = "/var/cache/apt/archives/"; - # for each package in @dl_debs, check if it's in - # /var/cache/apt/archives/ and add it to @essential_pkgs - foreach my $p (@dl_debs) { - my ($pkg, $ver_epoch) = @{$p}; - # apt appends the architecture at the end of the package name - ($pkg, my $arch) = split ':', $pkg, 2; - # apt replaces the colon by its percent encoding %3a - my $ver = $ver_epoch; - $ver =~ s/:/%3a/; - # the architecture returned by apt is the native architecture. - # Since we don't know whether the package is architecture - # independent or not, we first try with the native arch and then - # with "all" and only error out if neither exists. - if (-e "$options->{root}/$archives/${pkg}_${ver}_$arch.deb") { - push @essential_pkgs, "$archives/${pkg}_${ver}_$arch.deb"; - } elsif (-e "$options->{root}/$archives/${pkg}_${ver}_all.deb") { - push @essential_pkgs, "$archives/${pkg}_${ver}_all.deb"; - } else { - error( "cannot find package for $pkg:$arch (= $ver_epoch) " - . "in /var/cache/apt/archives/"); - } + # strip the the chroot directory from the filenames + foreach my $deb (@dl_debs) { + if (rindex $deb, $options->{root}, 0) { + error "package file $deb not in chroot directory" + . " -- use copy:// instead of file://"; } - } else { - # collect the .deb files that were downloaded by apt from the content - # of /var/cache/apt/archives/ - if (!$options->{dryrun}) { - my $apt_archives = "/var/cache/apt/archives/"; - opendir my $dh, "$options->{root}/$apt_archives" - or error "cannot read $apt_archives"; - while (my $deb = readdir $dh) { - if ($deb !~ /\.deb$/) { - next; - } - $deb = "$apt_archives/$deb"; - if (!-f "$options->{root}/$deb") { - next; - } - push @essential_pkgs, $deb; - } - closedir $dh; - - if (scalar @essential_pkgs == 0) { - # check if a file:// URI was used - open(my $pipe_apt, '-|', 'apt-get', 'indextargets', '--format', - '$(URI)', 'Created-By: Packages') - or error "cannot start apt-get indextargets: $!"; - while (my $uri = <$pipe_apt>) { - if ($uri =~ /^file:\/\//) { - error - "nothing got downloaded -- use copy:// instead of" - . " file://"; - } - } - error "nothing got downloaded"; - } + if (-e $deb) { + push @essential_pkgs, substr($deb, length($options->{root})); + } else { + error "cannot find package file $deb"; } } - # Unpack order matters. Since we create this list using two different - # methods but we want both methods to have the same result, we sort the - # list before returning it. + + # Unpack order matters for e.g. timestamps on directories @essential_pkgs = sort @essential_pkgs; return (\@essential_pkgs, \@cached_debs); @@ -6678,15 +6577,13 @@ the B step. This can be disabled using B<--skip=update>. =item B -Checks whether F is empty. This can be disabled with -B<--skip=download/empty>. In the B and B variants, C is used to download all the packages requested via the -B<--include> option. The B variant uses the fact that libapt treats the -C packages as implicitly essential to download only all C -packages plus apt using C. In the -remaining variants, all Packages files downloaded by the B step are -inspected to find the C package set as well as all packages of -the required priority. +In the B and B variants, C is used to +download all the packages requested via the B<--include> option. The B +variant uses the fact that libapt treats the C packages as implicitly +essential to download only all C packages plus apt using +C. In the remaining variants, all Packages files +downloaded by the B step are inspected to find the C +package set as well as all packages of the required priority. =item B @@ -6932,7 +6829,7 @@ apt-cacher-ng, you can use the B and B special hooks to synchronize a directory outside the chroot with F inside the chroot. - $ mmdebstrap --variant=apt --skip=download/empty --skip=essential/unlink \ + $ mmdebstrap --variant=apt --skip=essential/unlink \ --setup-hook='mkdir -p ./cache "$1"/var/cache/apt/archives/' \ --setup-hook='sync-in ./cache /var/cache/apt/archives/' \ --customize-hook='sync-out /var/cache/apt/archives ./cache' \ -- 2.39.5 From 317bfea7e50fb183fdbcd0a3eed2fb2307d6dcc9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 24 Apr 2022 03:09:03 +0200 Subject: [PATCH 4/6] Support file:// mirrors the same way copy:// is supported As long as you can make it so that the same path to the deb file works inside and outside of the chroot using file:// as a mirror is no longer a problem with the previous work. --- coverage.sh | 10 +++------- mmdebstrap | 15 +++++++-------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/coverage.sh b/coverage.sh index dbcf26f..23bf716 100755 --- a/coverage.sh +++ b/coverage.sh @@ -1526,7 +1526,7 @@ else skipped=$((skipped+1)) fi -print_header "mode=$defaultmode,variant=apt: fail with file:// mirror" +print_header "mode=$defaultmode,variant=apt: file:// mirror" cat << END > shared/test.sh #!/bin/sh set -eu @@ -1535,13 +1535,9 @@ if [ ! -e /mmdebstrap-testenv ]; then echo "this test requires the cache directory to be mounted on /mnt and should only be run inside a container" >&2 exit 1 fi -ret=0 -$CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST /tmp/debian-chroot.tar "deb file:///mnt/cache/debian unstable main" || ret=\$? +$CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST /tmp/debian-chroot.tar "deb file:///mnt/cache/debian $DEFAULT_DIST main" +tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt - rm /tmp/debian-chroot.tar -if [ "\$ret" = 0 ]; then - echo expected failure but got exit \$ret >&2 - exit 1 -fi END if [ "$HAVE_QEMU" = "yes" ]; then ./run_qemu.sh diff --git a/mmdebstrap b/mmdebstrap index 8765b15..a10ceff 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -2135,8 +2135,13 @@ sub run_download() { # strip the the chroot directory from the filenames foreach my $deb (@dl_debs) { if (rindex $deb, $options->{root}, 0) { - error "package file $deb not in chroot directory" - . " -- use copy:// instead of file://"; + if (-e "$options->{root}/$deb") { + push @essential_pkgs, $deb; + } else { + error "package file $deb not accessible from chroot directory" + . " -- use copy:// instead of file:// or a bind-mount"; + } + next; } if (-e $deb) { push @essential_pkgs, substr($deb, length($options->{root})); @@ -6983,12 +6988,6 @@ as the non-root user, then as a workaround you could run C so that the config files are only accessible by the root user. See Debian bug #808203. -The C URI type cannot be used to install the essential packages. This -is because B uses dpkg to install the packages that apt places into -F but with C apt will not copy the files even -with C<--download-only>. Use C instead, which is equivalent to -C but copies the archives into F. - With apt versions before 2.1.16, setting C<[trusted=yes]> or C to allow signed archives without a known public key or unsigned archives will fail because of a gpg warning in the -- 2.39.5 From c2cb442899079a61942acd228b9bbbdc9df0517d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 24 Apr 2022 03:16:00 +0200 Subject: [PATCH 5/6] Let apt decide unpack order instead of sorting filenames Now that the deb files can reside in different places sorting them leads to subtil differences in the order and hence the created chroot. apts unpack order on the other hand might not be a good order (but why would one sorted from a to z be one?), but it is far more stable as it is independent on the filenames. --- mmdebstrap | 3 --- 1 file changed, 3 deletions(-) diff --git a/mmdebstrap b/mmdebstrap index a10ceff..24f7250 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -2150,9 +2150,6 @@ sub run_download() { } } - # Unpack order matters for e.g. timestamps on directories - @essential_pkgs = sort @essential_pkgs; - return (\@essential_pkgs, \@cached_debs); } -- 2.39.5 From 6baf4151f9195f9aef8ec52ca4e69d6202969f98 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 13 May 2022 11:13:57 +0200 Subject: [PATCH 6/6] Set MMDEBSTRAP_VERBOSITY to inform hooks of verbosity level Hooks might want to be more or less verbose depending on how verbose mmdebstrap is configured to be. --- mmdebstrap | 1 + 1 file changed, 1 insertion(+) diff --git a/mmdebstrap b/mmdebstrap index 24f7250..84e7f66 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -1413,6 +1413,7 @@ sub run_hooks { # This is the file descriptor of the socket that the mmdebstrap # --hook-helper can write to and read from to communicate with the outside. push @env_opts, ("MMDEBSTRAP_HOOKSOCK=" . fileno($options->{hooksock})); + push @env_opts, ("MMDEBSTRAP_VERBOSITY=" . $verbosity_level); my $runner = sub { foreach my $script (@{ $options->{"${name}_hook"} }) { -- 2.39.5