josch/mmdebstrap
2
3
Fork 6
Code Issues 8 Pull requests 1 Projects Releases Wiki Activity

Ask apt to give us the deb filenames directly

Browse source 
Guessing filenames is boring. What if we could ask apt to tell us which
debs it downloaded (or found lying around elsewhere) directly? Turns out
we can rather easily avoiding a bunch of guesswork.
This commit is contained in:
David Kalnischkies 2022-04-23 16:48:31 +02:00
parent 487237f9ae
commit 71067316ee
2 changed files with 51 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
coverage.sh
View file

@ -3072,7 +3072,7 @@ $CMD \$include --mode=$defaultmode --variant=$variant \
--setup-hook='sync-in "'"\$tmpdir"'" /var/cache/apt/archives/partial' \
$DEFAULT_DIST - $mirror > test1.tar
cmp orig.tar test1.tar
$CMD \$include --mode=$defaultmode --variant=$variant --skip=download/empty \
$CMD \$include --mode=$defaultmode --variant=$variant \
--customize-hook='touch "\$1"/var/cache/apt/archives/partial' \
--setup-hook='mkdir -p "\$1"/var/cache/apt/archives/' \
--setup-hook='sync-in "'"\$tmpdir"'" /var/cache/apt/archives/' \

203
mmdebstrap
View file

@ -872,22 +872,11 @@ sub run_dpkg_progress {
}
sub run_apt_progress {
my $options = shift;
my @debs = @{ $options->{PKGS} // [] };
my $tmpeipp;
if (exists $options->{EIPP_RES}) {
(undef, $tmpeipp) = tempfile(
"mmdebstrap.eipp.XXXXXXXXXXXX",
OPEN => 0,
TMPDIR => 1
);
}
my $options = shift;
my @debs = @{ $options->{PKGS} // [] };
my $get_exec = sub {
my @prefix = ();
my @opts = ();
if (exists $options->{EIPP_RES}) {
push @opts, "-oDir::Log::Planner=$tmpeipp";
}
return (
@prefix,
@{ $options->{ARGV} },
@ -942,60 +931,44 @@ sub run_apt_progress {
}
};
run_progress $get_exec, $line_handler, $line_has_error, $options->{CHDIR};
if (exists $options->{EIPP_RES}) {
info "parsing EIPP results...";
open my $fh, '<', $tmpeipp
or error "failed to open $tmpeipp for reading: $!";
my $inst = 0;
my $pkg;
my $arch;
my $ver;
while (my $line = <$fh>) {
chomp $line;
if ($line ne "") {
if ($line =~ /^Status: .+/) {
$inst = 1;
} elsif ($line =~ /^Package: (.*)/) {
$pkg = $1;
} elsif ($line =~ /^Architecture: (.*)/) {
$arch = $1;
} elsif ($line =~ /^Version: (.*)/) {
$ver = $1;
}
next;
}
if ($inst == 0 && defined $pkg && defined $arch && defined $ver) {
push @{ $options->{EIPP_RES} }, ["$pkg:$arch", $ver];
}
$inst = 0;
undef $pkg;
undef $ver;
}
close $fh;
unlink $tmpeipp;
}
return;
}
sub run_apt_download_progress {
my $options = shift;
my %result = shift;
my $tmplistofdebs;
if ($options->{dryrun}) {
info "simulate downloading packages with apt...";
} else {
info "downloading packages with apt...";
(undef, $tmplistofdebs) = tempfile(
"mmdebstrap.listofdebs.XXXXXXXXXXXX",
OPEN => 0,
TMPDIR => 1
);
}
return run_apt_progress({
run_apt_progress({
ARGV => [
'apt-get',
'--yes',
'-oDebug::pkgDpkgPm=1',
'-oDir::Log=/dev/null',
$options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
$options->{dryrun}
? '-oAPT::Get::Simulate=true'
: "-oDpkg::Pre-Install-Pkgs::=cat > $tmplistofdebs",
@{ $options->{APT_ARGV} },
],
%result
});
if ($tmplistofdebs) {
open my $fh, '<', $tmplistofdebs
or error "failed to open $tmplistofdebs for reading: $!";
my @listofdebs = <$fh>;
close $fh;
unlink $tmplistofdebs;
chomp(@listofdebs);
return @listofdebs;
}
return [];
}
sub run_chroot {
@ -2050,21 +2023,14 @@ sub run_update() {
sub run_download() {
my $options = shift;
# We use /var/cache/apt/archives/ to figure out which packages apt chooses
# to install. That's why the directory must be empty if:
# - /var/cache/apt/archives exists, and
# - no simulation run is done, and
# - the variant is not extract or custom or the number to be
# installed packages not zero
#
# In the future we want to replace downloading packages with "apt-get
# install --download-only" and installing them with dpkg by just installing
# the essential packages with apt from the outside with
# DPkg::Chroot-Directory. We are not doing that because then the preinst
# script of base-passwd will not be called early enough and packages will
# fail to install because they are missing /etc/passwd.
# install" and installing them with dpkg by just installing the essential
# packages with apt from the outside with DPkg::Chroot-Directory.
# We are not doing that because then the preinst script of base-passwd will
# not be called early enough and packages will fail to install because they
# are missing /etc/passwd.
my @cached_debs = ();
my @dl_debs = ();
my @dl_debs;
if (
!$options->{dryrun}
&& ((none { $_ eq $options->{variant} } ('extract', 'custom'))
@ -2084,14 +2050,6 @@ sub run_download() {
push @cached_debs, $deb;
}
closedir $dh;
if (scalar @cached_debs > 0) {
if (any { $_ eq 'download/empty' } @{ $options->{skip} }) {
info "skipping download/empty as requested";
} else {
error("/var/cache/apt/archives/ inside the chroot contains: "
. (join ', ', (sort @cached_debs)));
}
}
}
# To figure out the right package set for the apt variant we can use:
@ -2100,14 +2058,6 @@ sub run_download() {
# apt and libapt treats apt as essential. If we want to install less
# (essential variant) then we have to compute the package set ourselves.
# Same if we want to install priority based variants.
my %result = ();
if (not $options->{dryrun}) {
# if there are already packages in /var/cache/apt/archives/,
# we need to know which are part of the solution by apt
if (scalar @cached_debs > 0) {
$result{EIPP_RES} = \@dl_debs;
}
}
if (any { $_ eq $options->{variant} } ('extract', 'custom')) {
if (scalar @{ $options->{include} } == 0) {
info "nothing to download -- skipping...";
@ -2126,11 +2076,10 @@ sub run_download() {
}
}
run_apt_download_progress({
@dl_debs = run_apt_download_progress({
APT_ARGV => @apt_argv,
dryrun => $options->{dryrun},
},
%result
);
} elsif ($options->{variant} eq 'apt') {
# if we just want to install Essential:yes packages, apt and their
@ -2146,11 +2095,10 @@ sub run_download() {
# remind me in 5+ years that I said that after I wrote
# in the bugreport: "Are you crazy?!? Nobody in his
# right mind would even suggest depending on it!")
run_apt_download_progress({
@dl_debs = run_apt_download_progress({
APT_ARGV => ['dist-upgrade'],
dryrun => $options->{dryrun},
},
%result
);
} elsif (
any { $_ eq $options->{variant} }
@ -2160,7 +2108,7 @@ sub run_download() {
# 17:27 < DonKult> (?essential includes 'apt' through)
# 17:30 < josch> DonKult: no, because pkgCacheGen::ForceEssential ",";
# 17:32 < DonKult> touché
run_apt_download_progress({
@dl_debs = run_apt_download_progress({
APT_ARGV => [
'install',
'?narrow('
@ -2178,75 +2126,26 @@ sub run_download() {
],
dryrun => $options->{dryrun},
},
%result
);
} else {
error "unknown variant: $options->{variant}";
}
my @essential_pkgs;
if (scalar @cached_debs > 0 && scalar @dl_debs > 0) {
my $archives = "/var/cache/apt/archives/";
# for each package in @dl_debs, check if it's in
# /var/cache/apt/archives/ and add it to @essential_pkgs
foreach my $p (@dl_debs) {
my ($pkg, $ver_epoch) = @{$p};
# apt appends the architecture at the end of the package name
($pkg, my $arch) = split ':', $pkg, 2;
# apt replaces the colon by its percent encoding %3a
my $ver = $ver_epoch;
$ver =~ s/:/%3a/;
# the architecture returned by apt is the native architecture.
# Since we don't know whether the package is architecture
# independent or not, we first try with the native arch and then
# with "all" and only error out if neither exists.
if (-e "$options->{root}/$archives/${pkg}_${ver}_$arch.deb") {
push @essential_pkgs, "$archives/${pkg}_${ver}_$arch.deb";
} elsif (-e "$options->{root}/$archives/${pkg}_${ver}_all.deb") {
push @essential_pkgs, "$archives/${pkg}_${ver}_all.deb";
} else {
error( "cannot find package for $pkg:$arch (= $ver_epoch) "
. "in /var/cache/apt/archives/");
}
# strip the the chroot directory from the filenames
foreach my $deb (@dl_debs) {
if (rindex $deb, $options->{root}, 0) {
error "package file $deb not in chroot directory"
. " -- use copy:// instead of file://";
}
} else {
# collect the .deb files that were downloaded by apt from the content
# of /var/cache/apt/archives/
if (!$options->{dryrun}) {
my $apt_archives = "/var/cache/apt/archives/";
opendir my $dh, "$options->{root}/$apt_archives"
or error "cannot read $apt_archives";
while (my $deb = readdir $dh) {
if ($deb !~ /\.deb$/) {
next;
}
$deb = "$apt_archives/$deb";
if (!-f "$options->{root}/$deb") {
next;
}
push @essential_pkgs, $deb;
}
closedir $dh;
if (scalar @essential_pkgs == 0) {
# check if a file:// URI was used
open(my $pipe_apt, '-|', 'apt-get', 'indextargets', '--format',
'$(URI)', 'Created-By: Packages')
or error "cannot start apt-get indextargets: $!";
while (my $uri = <$pipe_apt>) {
if ($uri =~ /^file:\/\//) {
error
"nothing got downloaded -- use copy:// instead of"
. " file://";
}
}
error "nothing got downloaded";
}
if (-e $deb) {
push @essential_pkgs, substr($deb, length($options->{root}));
} else {
error "cannot find package file $deb";
}
}
# Unpack order matters. Since we create this list using two different
# methods but we want both methods to have the same result, we sort the
# list before returning it.
# Unpack order matters for e.g. timestamps on directories
@essential_pkgs = sort @essential_pkgs;
return (\@essential_pkgs, \@cached_debs);
@ -6678,15 +6577,13 @@ the B<setup> step. This can be disabled using B<--skip=update>.
=item B<download>
Checks whether F</var/cache/apt/archives/> is empty. This can be disabled with
B<--skip=download/empty>. In the B<extract> and B<custom> variants, C<apt-get
--download-only install> is used to download all the packages requested via the
B<--include> option. The B<apt> variant uses the fact that libapt treats the
C<apt> packages as implicitly essential to download only all C<Essential:yes>
packages plus apt using C<apt-get --download-only dist-upgrade>. In the
remaining variants, all Packages files downloaded by the B<update> step are
inspected to find the C<Essential:yes> package set as well as all packages of
the required priority.
In the B<extract> and B<custom> variants, C<apt-get install> is used to
download all the packages requested via the B<--include> option. The B<apt>
variant uses the fact that libapt treats the C<apt> packages as implicitly
essential to download only all C<Essential:yes> packages plus apt using
C<apt-get dist-upgrade>. In the remaining variants, all Packages files
downloaded by the B<update> step are inspected to find the C<Essential:yes>
package set as well as all packages of the required priority.
=item B<extract>
@ -6914,7 +6811,7 @@ apt-cacher-ng, you can use the B<sync-in> and B<sync-out> special hooks to
synchronize a directory outside the chroot with F</var/cache/apt/archives>
inside the chroot.
$ mmdebstrap --variant=apt --skip=download/empty --skip=essential/unlink \
$ mmdebstrap --variant=apt --skip=essential/unlink \
--setup-hook='mkdir -p ./cache "$1"/var/cache/apt/archives/' \
--setup-hook='sync-in ./cache /var/cache/apt/archives/' \
--customize-hook='sync-out /var/cache/apt/archives ./cache' \