josch/mmdebstrap
2
3
Fork 6
Code Issues 8 Pull requests 1 Projects Releases Wiki Activity

WIP: Support file:/ mirrors #25

Closed
DonKult wants to merge 6 commits from DonKult/mmdebstrap:feature/filemirror into main
pull from: DonKult/mmdebstrap:feature/filemirror
merge into: josch:main
Conversation 10 Commits 6 Files changed 3 +51 -154
2 changed files with 51 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 3ceebe7e07 - Show all commits

2
coverage.sh
View file

@ -3072,7 +3072,7 @@ $CMD \$include --mode=$defaultmode --variant=$variant \
--setup-hook='sync-in "'"\$tmpdir"'" /var/cache/apt/archives/partial' \
$DEFAULT_DIST - $mirror > test1.tar
cmp orig.tar test1.tar
$CMD \$include --mode=$defaultmode --variant=$variant --skip=download/empty \
$CMD \$include --mode=$defaultmode --variant=$variant \
--customize-hook='touch "\$1"/var/cache/apt/archives/partial' \
--setup-hook='mkdir -p "\$1"/var/cache/apt/archives/' \
--setup-hook='sync-in "'"\$tmpdir"'" /var/cache/apt/archives/' \

197
mmdebstrap
View file

@ -874,20 +874,9 @@ sub run_dpkg_progress {
sub run_apt_progress {
my $options = shift;
my @debs = @{ $options->{PKGS} // [] };
my $tmpeipp;
if (exists $options->{EIPP_RES}) {
(undef, $tmpeipp) = tempfile(
"mmdebstrap.eipp.XXXXXXXXXXXX",
OPEN => 0,
TMPDIR => 1
);
}
my $get_exec = sub {
my @prefix = ();
my @opts = ();
if (exists $options->{EIPP_RES}) {
push @opts, "-oDir::Log::Planner=$tmpeipp";
}
return (
@prefix,
@{ $options->{ARGV} },
@ -942,60 +931,44 @@ sub run_apt_progress {
}
};
run_progress $get_exec, $line_handler, $line_has_error, $options->{CHDIR};
if (exists $options->{EIPP_RES}) {
info "parsing EIPP results...";
open my $fh, '<', $tmpeipp
or error "failed to open $tmpeipp for reading: $!";
my $inst = 0;
my $pkg;
my $arch;
my $ver;
while (my $line = <$fh>) {
chomp $line;
if ($line ne "") {
if ($line =~ /^Status: .+/) {
$inst = 1;
} elsif ($line =~ /^Package: (.*)/) {
$pkg = $1;
} elsif ($line =~ /^Architecture: (.*)/) {
$arch = $1;
} elsif ($line =~ /^Version: (.*)/) {
$ver = $1;
}
next;
}
if ($inst == 0 && defined $pkg && defined $arch && defined $ver) {
push @{ $options->{EIPP_RES} }, ["$pkg:$arch", $ver];
}
$inst = 0;
undef $pkg;
undef $ver;
}
close $fh;
unlink $tmpeipp;
}
return;
}
sub run_apt_download_progress {
my $options = shift;
my %result = shift;
my $tmplistofdebs;
if ($options->{dryrun}) {
info "simulate downloading packages with apt...";
} else {
info "downloading packages with apt...";
(undef, $tmplistofdebs) = tempfile(
"mmdebstrap.listofdebs.XXXXXXXXXXXX",
OPEN => 0,
TMPDIR => 1
);
}
return run_apt_progress({
run_apt_progress({
ARGV => [
'apt-get',
'--yes',
'-oDebug::pkgDpkgPm=1',
'-oDir::Log=/dev/null',
$options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
$options->{dryrun}
? '-oAPT::Get::Simulate=true'
: "-oDpkg::Pre-Install-Pkgs::=cat > $tmplistofdebs",
@{ $options->{APT_ARGV} },
],
%result
});
if ($tmplistofdebs) {
open my $fh, '<', $tmplistofdebs
or error "failed to open $tmplistofdebs for reading: $!";
my @listofdebs = <$fh>;
close $fh;
unlink $tmplistofdebs;
chomp(@listofdebs);
return @listofdebs;
}
return [];
}
sub run_chroot {
@ -2050,21 +2023,14 @@ sub run_update() {
sub run_download() {
my $options = shift;
# We use /var/cache/apt/archives/ to figure out which packages apt chooses
# to install. That's why the directory must be empty if:
# - /var/cache/apt/archives exists, and
# - no simulation run is done, and
# - the variant is not extract or custom or the number to be
# installed packages not zero
#
# In the future we want to replace downloading packages with "apt-get
# install --download-only" and installing them with dpkg by just installing
# the essential packages with apt from the outside with
# DPkg::Chroot-Directory. We are not doing that because then the preinst
# script of base-passwd will not be called early enough and packages will
# fail to install because they are missing /etc/passwd.
# install" and installing them with dpkg by just installing the essential
# packages with apt from the outside with DPkg::Chroot-Directory.
# We are not doing that because then the preinst script of base-passwd will
# not be called early enough and packages will fail to install because they
# are missing /etc/passwd.
my @cached_debs = ();
my @dl_debs = ();
my @dl_debs;
if (
!$options->{dryrun}
&& ((none { $_ eq $options->{variant} } ('extract', 'custom'))
@ -2084,14 +2050,6 @@ sub run_download() {
push @cached_debs, $deb;
}
closedir $dh;
if (scalar @cached_debs > 0) {
if (any { $_ eq 'download/empty' } @{ $options->{skip} }) {
info "skipping download/empty as requested";
} else {
error("/var/cache/apt/archives/ inside the chroot contains: "
. (join ', ', (sort @cached_debs)));
}
}
}
# To figure out the right package set for the apt variant we can use:
@ -2100,14 +2058,6 @@ sub run_download() {
# apt and libapt treats apt as essential. If we want to install less
# (essential variant) then we have to compute the package set ourselves.
# Same if we want to install priority based variants.
my %result = ();
if (not $options->{dryrun}) {
# if there are already packages in /var/cache/apt/archives/,
# we need to know which are part of the solution by apt
if (scalar @cached_debs > 0) {
$result{EIPP_RES} = \@dl_debs;
}
}
if (any { $_ eq $options->{variant} } ('extract', 'custom')) {
if (scalar @{ $options->{include} } == 0) {
info "nothing to download -- skipping...";
@ -2126,11 +2076,10 @@ sub run_download() {
}
}
run_apt_download_progress({
@dl_debs = run_apt_download_progress({
APT_ARGV => @apt_argv,
dryrun => $options->{dryrun},
},
%result
);
} elsif ($options->{variant} eq 'apt') {
# if we just want to install Essential:yes packages, apt and their
@ -2146,11 +2095,10 @@ sub run_download() {
# remind me in 5+ years that I said that after I wrote
# in the bugreport: "Are you crazy?!? Nobody in his
# right mind would even suggest depending on it!")
run_apt_download_progress({
@dl_debs = run_apt_download_progress({
APT_ARGV => ['dist-upgrade'],
dryrun => $options->{dryrun},
},
%result
);
} elsif (
any { $_ eq $options->{variant} }
@ -2160,7 +2108,7 @@ sub run_download() {
# 17:27 < DonKult> (?essential includes 'apt' through)
# 17:30 < josch> DonKult: no, because pkgCacheGen::ForceEssential ",";
# 17:32 < DonKult> touché
run_apt_download_progress({
@dl_debs = run_apt_download_progress({
APT_ARGV => [
'install',
'?narrow('
@ -2178,75 +2126,26 @@ sub run_download() {
],
dryrun => $options->{dryrun},
},
%result
);
} else {
error "unknown variant: $options->{variant}";
}
my @essential_pkgs;
if (scalar @cached_debs > 0 && scalar @dl_debs > 0) {
my $archives = "/var/cache/apt/archives/";
# for each package in @dl_debs, check if it's in
# /var/cache/apt/archives/ and add it to @essential_pkgs
foreach my $p (@dl_debs) {
my ($pkg, $ver_epoch) = @{$p};
# apt appends the architecture at the end of the package name
($pkg, my $arch) = split ':', $pkg, 2;
# apt replaces the colon by its percent encoding %3a
my $ver = $ver_epoch;
$ver =~ s/:/%3a/;
# the architecture returned by apt is the native architecture.
# Since we don't know whether the package is architecture
# independent or not, we first try with the native arch and then
# with "all" and only error out if neither exists.
if (-e "$options->{root}/$archives/${pkg}_${ver}_$arch.deb") {
push @essential_pkgs, "$archives/${pkg}_${ver}_$arch.deb";
} elsif (-e "$options->{root}/$archives/${pkg}_${ver}_all.deb") {
push @essential_pkgs, "$archives/${pkg}_${ver}_all.deb";
# strip the the chroot directory from the filenames
foreach my $deb (@dl_debs) {
if (rindex $deb, $options->{root}, 0) {
error "package file $deb not in chroot directory"
. " -- use copy:// instead of file://";
}
if (-e $deb) {
push @essential_pkgs, substr($deb, length($options->{root}));
} else {
error( "cannot find package for $pkg:$arch (= $ver_epoch) "
. "in /var/cache/apt/archives/");
error "cannot find package file $deb";
}
}
} else {
# collect the .deb files that were downloaded by apt from the content
# of /var/cache/apt/archives/
if (!$options->{dryrun}) {
my $apt_archives = "/var/cache/apt/archives/";
opendir my $dh, "$options->{root}/$apt_archives"
or error "cannot read $apt_archives";
while (my $deb = readdir $dh) {
if ($deb !~ /\.deb$/) {
next;
}
$deb = "$apt_archives/$deb";
if (!-f "$options->{root}/$deb") {
next;
}
push @essential_pkgs, $deb;
}
closedir $dh;
if (scalar @essential_pkgs == 0) {
# check if a file:// URI was used
open(my $pipe_apt, '-|', 'apt-get', 'indextargets', '--format',
'$(URI)', 'Created-By: Packages')
or error "cannot start apt-get indextargets: $!";
while (my $uri = <$pipe_apt>) {
if ($uri =~ /^file:\/\//) {
error
"nothing got downloaded -- use copy:// instead of"
. " file://";
}
}
error "nothing got downloaded";
}
}
}
# Unpack order matters. Since we create this list using two different
# methods but we want both methods to have the same result, we sort the
# list before returning it.
# Unpack order matters for e.g. timestamps on directories
@essential_pkgs = sort @essential_pkgs;
return (\@essential_pkgs, \@cached_debs);
@ -6678,15 +6577,13 @@ the B<setup> step. This can be disabled using B<--skip=update>.
=item B<download>
Checks whether F</var/cache/apt/archives/> is empty. This can be disabled with
B<--skip=download/empty>. In the B<extract> and B<custom> variants, C<apt-get
--download-only install> is used to download all the packages requested via the
B<--include> option. The B<apt> variant uses the fact that libapt treats the
C<apt> packages as implicitly essential to download only all C<Essential:yes>
packages plus apt using C<apt-get --download-only dist-upgrade>. In the
remaining variants, all Packages files downloaded by the B<update> step are
inspected to find the C<Essential:yes> package set as well as all packages of
the required priority.
In the B<extract> and B<custom> variants, C<apt-get install> is used to
download all the packages requested via the B<--include> option. The B<apt>
variant uses the fact that libapt treats the C<apt> packages as implicitly
essential to download only all C<Essential:yes> packages plus apt using
C<apt-get dist-upgrade>. In the remaining variants, all Packages files
downloaded by the B<update> step are inspected to find the C<Essential:yes>
package set as well as all packages of the required priority.
=item B<extract>
@ -6932,7 +6829,7 @@ apt-cacher-ng, you can use the B<sync-in> and B<sync-out> special hooks to
synchronize a directory outside the chroot with F</var/cache/apt/archives>
inside the chroot.
$ mmdebstrap --variant=apt --skip=download/empty --skip=essential/unlink \
$ mmdebstrap --variant=apt --skip=essential/unlink \
--setup-hook='mkdir -p ./cache "$1"/var/cache/apt/archives/' \
--setup-hook='sync-in ./cache /var/cache/apt/archives/' \
--customize-hook='sync-out /var/cache/apt/archives ./cache' \