From 71067316eef9f3c1cc04decd8823958238bcc46c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 23 Apr 2022 16:48:31 +0200 Subject: [PATCH] Ask apt to give us the deb filenames directly Guessing filenames is boring. What if we could ask apt to tell us which debs it downloaded (or found lying around elsewhere) directly? Turns out we can rather easily avoiding a bunch of guesswork. --- coverage.sh | 2 +- mmdebstrap | 205 +++++++++++++--------------------------------------- 2 files changed, 52 insertions(+), 155 deletions(-) diff --git a/coverage.sh b/coverage.sh index 673723d..dbcf26f 100755 --- a/coverage.sh +++ b/coverage.sh @@ -3072,7 +3072,7 @@ $CMD \$include --mode=$defaultmode --variant=$variant \ --setup-hook='sync-in "'"\$tmpdir"'" /var/cache/apt/archives/partial' \ $DEFAULT_DIST - $mirror > test1.tar cmp orig.tar test1.tar -$CMD \$include --mode=$defaultmode --variant=$variant --skip=download/empty \ +$CMD \$include --mode=$defaultmode --variant=$variant \ --customize-hook='touch "\$1"/var/cache/apt/archives/partial' \ --setup-hook='mkdir -p "\$1"/var/cache/apt/archives/' \ --setup-hook='sync-in "'"\$tmpdir"'" /var/cache/apt/archives/' \ diff --git a/mmdebstrap b/mmdebstrap index ba58be5..ea85a77 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -872,22 +872,11 @@ sub run_dpkg_progress { } sub run_apt_progress { - my $options = shift; - my @debs = @{ $options->{PKGS} // [] }; - my $tmpeipp; - if (exists $options->{EIPP_RES}) { - (undef, $tmpeipp) = tempfile( - "mmdebstrap.eipp.XXXXXXXXXXXX", - OPEN => 0, - TMPDIR => 1 - ); - } + my $options = shift; + my @debs = @{ $options->{PKGS} // [] }; my $get_exec = sub { my @prefix = (); my @opts = (); - if (exists $options->{EIPP_RES}) { - push @opts, "-oDir::Log::Planner=$tmpeipp"; - } return ( @prefix, @{ $options->{ARGV} }, @@ -942,60 +931,44 @@ sub run_apt_progress { } }; run_progress $get_exec, $line_handler, $line_has_error, $options->{CHDIR}; - if (exists $options->{EIPP_RES}) { - info "parsing EIPP results..."; - open my $fh, '<', $tmpeipp - or error "failed to open $tmpeipp for reading: $!"; - my $inst = 0; - my $pkg; - my $arch; - my $ver; - while (my $line = <$fh>) { - chomp $line; - if ($line ne "") { - if ($line =~ /^Status: .+/) { - $inst = 1; - } elsif ($line =~ /^Package: (.*)/) { - $pkg = $1; - } elsif ($line =~ /^Architecture: (.*)/) { - $arch = $1; - } elsif ($line =~ /^Version: (.*)/) { - $ver = $1; - } - next; - } - if ($inst == 0 && defined $pkg && defined $arch && defined $ver) { - push @{ $options->{EIPP_RES} }, ["$pkg:$arch", $ver]; - } - $inst = 0; - undef $pkg; - undef $ver; - } - close $fh; - unlink $tmpeipp; - } return; } sub run_apt_download_progress { my $options = shift; - my %result = shift; + my $tmplistofdebs; if ($options->{dryrun}) { info "simulate downloading packages with apt..."; } else { info "downloading packages with apt..."; + (undef, $tmplistofdebs) = tempfile( + "mmdebstrap.listofdebs.XXXXXXXXXXXX", + OPEN => 0, + TMPDIR => 1 + ); } - return run_apt_progress({ + run_apt_progress({ ARGV => [ 'apt-get', '--yes', '-oDebug::pkgDpkgPm=1', '-oDir::Log=/dev/null', - $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), + $options->{dryrun} + ? '-oAPT::Get::Simulate=true' + : "-oDpkg::Pre-Install-Pkgs::=cat > $tmplistofdebs", @{ $options->{APT_ARGV} }, ], - %result }); + if ($tmplistofdebs) { + open my $fh, '<', $tmplistofdebs + or error "failed to open $tmplistofdebs for reading: $!"; + my @listofdebs = <$fh>; + close $fh; + unlink $tmplistofdebs; + chomp(@listofdebs); + return @listofdebs; + } + return []; } sub run_chroot { @@ -2050,21 +2023,14 @@ sub run_update() { sub run_download() { my $options = shift; - # We use /var/cache/apt/archives/ to figure out which packages apt chooses - # to install. That's why the directory must be empty if: - # - /var/cache/apt/archives exists, and - # - no simulation run is done, and - # - the variant is not extract or custom or the number to be - # installed packages not zero - # # In the future we want to replace downloading packages with "apt-get - # install --download-only" and installing them with dpkg by just installing - # the essential packages with apt from the outside with - # DPkg::Chroot-Directory. We are not doing that because then the preinst - # script of base-passwd will not be called early enough and packages will - # fail to install because they are missing /etc/passwd. + # install" and installing them with dpkg by just installing the essential + # packages with apt from the outside with DPkg::Chroot-Directory. + # We are not doing that because then the preinst script of base-passwd will + # not be called early enough and packages will fail to install because they + # are missing /etc/passwd. my @cached_debs = (); - my @dl_debs = (); + my @dl_debs; if ( !$options->{dryrun} && ((none { $_ eq $options->{variant} } ('extract', 'custom')) @@ -2084,14 +2050,6 @@ sub run_download() { push @cached_debs, $deb; } closedir $dh; - if (scalar @cached_debs > 0) { - if (any { $_ eq 'download/empty' } @{ $options->{skip} }) { - info "skipping download/empty as requested"; - } else { - error("/var/cache/apt/archives/ inside the chroot contains: " - . (join ', ', (sort @cached_debs))); - } - } } # To figure out the right package set for the apt variant we can use: @@ -2100,14 +2058,6 @@ sub run_download() { # apt and libapt treats apt as essential. If we want to install less # (essential variant) then we have to compute the package set ourselves. # Same if we want to install priority based variants. - my %result = (); - if (not $options->{dryrun}) { - # if there are already packages in /var/cache/apt/archives/, - # we need to know which are part of the solution by apt - if (scalar @cached_debs > 0) { - $result{EIPP_RES} = \@dl_debs; - } - } if (any { $_ eq $options->{variant} } ('extract', 'custom')) { if (scalar @{ $options->{include} } == 0) { info "nothing to download -- skipping..."; @@ -2126,11 +2076,10 @@ sub run_download() { } } - run_apt_download_progress({ + @dl_debs = run_apt_download_progress({ APT_ARGV => @apt_argv, dryrun => $options->{dryrun}, }, - %result ); } elsif ($options->{variant} eq 'apt') { # if we just want to install Essential:yes packages, apt and their @@ -2146,11 +2095,10 @@ sub run_download() { # remind me in 5+ years that I said that after I wrote # in the bugreport: "Are you crazy?!? Nobody in his # right mind would even suggest depending on it!") - run_apt_download_progress({ + @dl_debs = run_apt_download_progress({ APT_ARGV => ['dist-upgrade'], dryrun => $options->{dryrun}, }, - %result ); } elsif ( any { $_ eq $options->{variant} } @@ -2160,7 +2108,7 @@ sub run_download() { # 17:27 < DonKult> (?essential includes 'apt' through) # 17:30 < josch> DonKult: no, because pkgCacheGen::ForceEssential ","; # 17:32 < DonKult> touché - run_apt_download_progress({ + @dl_debs = run_apt_download_progress({ APT_ARGV => [ 'install', '?narrow(' @@ -2178,75 +2126,26 @@ sub run_download() { ], dryrun => $options->{dryrun}, }, - %result ); } else { error "unknown variant: $options->{variant}"; } my @essential_pkgs; - if (scalar @cached_debs > 0 && scalar @dl_debs > 0) { - my $archives = "/var/cache/apt/archives/"; - # for each package in @dl_debs, check if it's in - # /var/cache/apt/archives/ and add it to @essential_pkgs - foreach my $p (@dl_debs) { - my ($pkg, $ver_epoch) = @{$p}; - # apt appends the architecture at the end of the package name - ($pkg, my $arch) = split ':', $pkg, 2; - # apt replaces the colon by its percent encoding %3a - my $ver = $ver_epoch; - $ver =~ s/:/%3a/; - # the architecture returned by apt is the native architecture. - # Since we don't know whether the package is architecture - # independent or not, we first try with the native arch and then - # with "all" and only error out if neither exists. - if (-e "$options->{root}/$archives/${pkg}_${ver}_$arch.deb") { - push @essential_pkgs, "$archives/${pkg}_${ver}_$arch.deb"; - } elsif (-e "$options->{root}/$archives/${pkg}_${ver}_all.deb") { - push @essential_pkgs, "$archives/${pkg}_${ver}_all.deb"; - } else { - error( "cannot find package for $pkg:$arch (= $ver_epoch) " - . "in /var/cache/apt/archives/"); - } - } - } else { - # collect the .deb files that were downloaded by apt from the content - # of /var/cache/apt/archives/ - if (!$options->{dryrun}) { - my $apt_archives = "/var/cache/apt/archives/"; - opendir my $dh, "$options->{root}/$apt_archives" - or error "cannot read $apt_archives"; - while (my $deb = readdir $dh) { - if ($deb !~ /\.deb$/) { - next; - } - $deb = "$apt_archives/$deb"; - if (!-f "$options->{root}/$deb") { - next; - } - push @essential_pkgs, $deb; - } - closedir $dh; - - if (scalar @essential_pkgs == 0) { - # check if a file:// URI was used - open(my $pipe_apt, '-|', 'apt-get', 'indextargets', '--format', - '$(URI)', 'Created-By: Packages') - or error "cannot start apt-get indextargets: $!"; - while (my $uri = <$pipe_apt>) { - if ($uri =~ /^file:\/\//) { - error - "nothing got downloaded -- use copy:// instead of" - . " file://"; - } - } - error "nothing got downloaded"; - } + # strip the the chroot directory from the filenames + foreach my $deb (@dl_debs) { + if (rindex $deb, $options->{root}, 0) { + error "package file $deb not in chroot directory" + . " -- use copy:// instead of file://"; + } + if (-e $deb) { + push @essential_pkgs, substr($deb, length($options->{root})); + } else { + error "cannot find package file $deb"; } } - # Unpack order matters. Since we create this list using two different - # methods but we want both methods to have the same result, we sort the - # list before returning it. + + # Unpack order matters for e.g. timestamps on directories @essential_pkgs = sort @essential_pkgs; return (\@essential_pkgs, \@cached_debs); @@ -6678,15 +6577,13 @@ the B step. This can be disabled using B<--skip=update>. =item B -Checks whether F is empty. This can be disabled with -B<--skip=download/empty>. In the B and B variants, C is used to download all the packages requested via the -B<--include> option. The B variant uses the fact that libapt treats the -C packages as implicitly essential to download only all C -packages plus apt using C. In the -remaining variants, all Packages files downloaded by the B step are -inspected to find the C package set as well as all packages of -the required priority. +In the B and B variants, C is used to +download all the packages requested via the B<--include> option. The B +variant uses the fact that libapt treats the C packages as implicitly +essential to download only all C packages plus apt using +C. In the remaining variants, all Packages files +downloaded by the B step are inspected to find the C +package set as well as all packages of the required priority. =item B @@ -6914,7 +6811,7 @@ apt-cacher-ng, you can use the B and B special hooks to synchronize a directory outside the chroot with F inside the chroot. - $ mmdebstrap --variant=apt --skip=download/empty --skip=essential/unlink \ + $ mmdebstrap --variant=apt --skip=essential/unlink \ --setup-hook='mkdir -p ./cache "$1"/var/cache/apt/archives/' \ --setup-hook='sync-in ./cache /var/cache/apt/archives/' \ --customize-hook='sync-out /var/cache/apt/archives ./cache' \