josch/mmdebstrap
2
3
Fork 6
Code Issues 8 Pull requests 1 Projects Releases Wiki Activity

WIP: Support file:/ mirrors #25

Closed
DonKult wants to merge 6 commits from DonKult/mmdebstrap:feature/filemirror into main
pull from: DonKult/mmdebstrap:feature/filemirror
merge into: josch:main
Conversation 10 Commits 6 Files changed 3 +30 -100
3 changed files with 30 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit ee142d52a5 - Show all commits

9
coverage.sh
View file

@ -89,13 +89,6 @@ if [ ! -e shared/tarfilter ] || [ tarfilter -nt shared/tarfilter ]; then
cp -a /usr/bin/mmtarfilter shared/tarfilter cp -a /usr/bin/mmtarfilter shared/tarfilter
fi fi
fi fi
if [ ! -e shared/proxysolver ] || [ proxysolver -nt shared/proxysolver ]; then
if [ -e ./proxysolver ]; then
cp -a proxysolver shared
else
cp -a /usr/lib/apt/solvers/mmdebstrap-dump-solution shared/proxysolver
fi
fi
if [ ! -e shared/ldconfig.fakechroot ] || [ ldconfig.fakechroot -nt shared/ldconfig.fakechroot ]; then if [ ! -e shared/ldconfig.fakechroot ] || [ ldconfig.fakechroot -nt shared/ldconfig.fakechroot ]; then
if [ -e ./ldconfig.fakechroot ]; then if [ -e ./ldconfig.fakechroot ]; then
cp -a ldconfig.fakechroot shared cp -a ldconfig.fakechroot shared
@ -3760,4 +3753,4 @@ if [ "$((skipped+runtests))" -ne "$total" ]; then
exit 1 exit 1
fi fi
rm shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/taridshift shared/tarfilter shared/proxysolver rm shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/taridshift shared/tarfilter

65
mmdebstrap
View file

@ -874,10 +874,10 @@ sub run_dpkg_progress {
sub run_apt_progress { sub run_apt_progress {
my $options = shift; my $options = shift;
my @debs = @{ $options->{PKGS} // [] }; my @debs = @{ $options->{PKGS} // [] };
my $tmpedsp; my $tmpeipp;
if (exists $options->{EDSP_RES}) { if (exists $options->{EIPP_RES}) {
(undef, $tmpedsp) = tempfile( (undef, $tmpeipp) = tempfile(
"mmdebstrap.edsp.XXXXXXXXXXXX", "mmdebstrap.eipp.XXXXXXXXXXXX",
OPEN => 0, OPEN => 0,
TMPDIR => 1 TMPDIR => 1
); );
@ -885,16 +885,8 @@ sub run_apt_progress {
my $get_exec = sub { my $get_exec = sub {
my @prefix = (); my @prefix = ();
my @opts = (); my @opts = ();
if (exists $options->{EDSP_RES}) { if (exists $options->{EIPP_RES}) {
push @prefix, 'env', "APT_EDSP_DUMP_FILENAME=$tmpedsp"; push @opts, "-oDir::Log::Planner=$tmpeipp";
if (-e "./proxysolver") {
# for development purposes, use the current directory if it
# contains a file called proxysolver
push @opts, ("-oDir::Bin::solvers=" . getcwd()),
'--solver=proxysolver';
} else {
push @opts, '--solver=mmdebstrap-dump-solution';
}
} }
return ( return (
@prefix, @prefix,
@ -950,34 +942,37 @@ sub run_apt_progress {
} }
}; };
run_progress $get_exec, $line_handler, $line_has_error, $options->{CHDIR}; run_progress $get_exec, $line_handler, $line_has_error, $options->{CHDIR};
if (exists $options->{EDSP_RES}) { if (exists $options->{EIPP_RES}) {
info "parsing EDSP results..."; info "parsing EIPP results...";
open my $fh, '<', $tmpedsp open my $fh, '<', $tmpeipp
or error "failed to open $tmpedsp for reading: $!"; or error "failed to open $tmpeipp for reading: $!";
my $inst = 0; my $inst = 0;
my $pkg; my $pkg;
my $arch;
my $ver; my $ver;
while (my $line = <$fh>) { while (my $line = <$fh>) {
chomp $line; chomp $line;
if ($line ne "") { if ($line ne "") {
if ($line =~ /^Install: \d+/) { if ($line =~ /^Status: .+/) {
$inst = 1; $inst = 1;
} elsif ($line =~ /^Package: (.*)/) { } elsif ($line =~ /^Package: (.*)/) {
$pkg = $1; $pkg = $1;
} elsif ($line =~ /^Architecture: (.*)/) {
$arch = $1;
} elsif ($line =~ /^Version: (.*)/) { } elsif ($line =~ /^Version: (.*)/) {
$ver = $1; $ver = $1;
} }
next; next;
} }
if ($inst == 1 && defined $pkg && defined $ver) { if ($inst == 0 && defined $pkg && defined $arch && defined $ver) {
push @{ $options->{EDSP_RES} }, [$pkg, $ver]; push @{ $options->{EIPP_RES} }, ["$pkg:$arch", $ver];
} }
$inst = 0; $inst = 0;
undef $pkg; undef $pkg;
undef $ver; undef $ver;
} }
close $fh; close $fh;
unlink $tmpedsp; unlink $tmpeipp;
} }
return; return;
} }
@ -2041,11 +2036,6 @@ sub run_download() {
# - the variant is not extract or custom or the number to be # - the variant is not extract or custom or the number to be
# installed packages not zero # installed packages not zero
# #
# We could also unconditionally use the proxysolver and then "apt-get
# download" any missing packages but using the proxysolver requires
# /usr/lib/apt/solvers/apt from the apt-utils package and we want to avoid
# that dependency.
#
# In the future we want to replace downloading packages with "apt-get # In the future we want to replace downloading packages with "apt-get
# install --download-only" and installing them with dpkg by just installing # install --download-only" and installing them with dpkg by just installing
# the essential packages with apt from the outside with # the essential packages with apt from the outside with
@ -2112,9 +2102,9 @@ sub run_download() {
info "simulate downloading packages with apt..."; info "simulate downloading packages with apt...";
} else { } else {
# if there are already packages in /var/cache/apt/archives/, we # if there are already packages in /var/cache/apt/archives/, we
# need to use our proxysolver to obtain the solution chosen by apt # need to know which are part of the solution by apt
if (scalar @cached_debs > 0) { if (scalar @cached_debs > 0) {
$result{EDSP_RES} = \@dl_debs; $result{EIPP_RES} = \@dl_debs;
} }
info "downloading packages with apt..."; info "downloading packages with apt...";
} }
@ -2122,7 +2112,8 @@ sub run_download() {
ARGV => [ ARGV => [
'apt-get', 'apt-get',
'--yes', '--yes',
'-oApt::Get::Download-Only=true', '-oDebug::pkgDpkgPm=1',
'-oDir::Log=/dev/null',
$options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
'install' 'install'
], ],
@ -2148,9 +2139,9 @@ sub run_download() {
info "simulate downloading packages with apt..."; info "simulate downloading packages with apt...";
} else { } else {
# if there are already packages in /var/cache/apt/archives/, we # if there are already packages in /var/cache/apt/archives/, we
# need to use our proxysolver to obtain the solution chosen by apt # need to know which are part of the solution by apt
if (scalar @cached_debs > 0) { if (scalar @cached_debs > 0) {
$result{EDSP_RES} = \@dl_debs; $result{EIPP_RES} = \@dl_debs;
} }
info "downloading packages with apt..."; info "downloading packages with apt...";
} }
@ -2158,7 +2149,8 @@ sub run_download() {
ARGV => [ ARGV => [
'apt-get', 'apt-get',
'--yes', '--yes',
'-oApt::Get::Download-Only=true', '-oDebug::pkgDpkgPm=1',
'-oDir::Log=/dev/null',
$options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
'dist-upgrade' 'dist-upgrade'
], ],
@ -2177,9 +2169,9 @@ sub run_download() {
info "simulate downloading packages with apt..."; info "simulate downloading packages with apt...";
} else { } else {
# if there are already packages in /var/cache/apt/archives/, we # if there are already packages in /var/cache/apt/archives/, we
# need to use our proxysolver to obtain the solution chosen by apt # need to know which are part of the solution by apt
if (scalar @cached_debs > 0) { if (scalar @cached_debs > 0) {
$result{EDSP_RES} = \@dl_debs; $result{EIPP_RES} = \@dl_debs;
} }
info "downloading packages with apt..."; info "downloading packages with apt...";
} }
@ -2187,7 +2179,8 @@ sub run_download() {
ARGV => [ ARGV => [
'apt-get', 'apt-get',
'--yes', '--yes',
'-oApt::Get::Download-Only=true', '-oDebug::pkgDpkgPm=1',
'-oDir::Log=/dev/null',
$options->{dryrun} ? '-oAPT::Get::Simulate=true' : (), $options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
'install', 'install',
'?narrow(' '?narrow('

56
proxysolver
View file

@ -1,56 +0,0 @@
#!/usr/bin/env python3
#
# This script is in the public domain
#
# Author: Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
#
# thin layer around /usr/lib/apt/solvers/apt, so that we can capture the solver
# result
#
# we set Debug::EDSP::WriteSolution=yes so that Install stanzas also come with
# Package and Version fields. That way, we do not also have to parse the EDSP
# request and spend time matching ID numbers
import subprocess
import sys
import os
import getpass
if not os.path.exists("/usr/lib/apt/solvers/apt"):
print(
"""Error: ERR_NO_SOLVER
Message: The external apt solver doesn't exist. You must install the apt-utils package.
"""
)
exit()
fname = os.environ.get("APT_EDSP_DUMP_FILENAME")
if fname is None:
print(
"""Error: ERR_NO_FILENAME
Message: You have to set the environment variable APT_EDSP_DUMP_FILENAME
to a valid filename to store the dump of EDSP solver input in.
For example with: export APT_EDSP_DUMP_FILENAME=/tmp/dump.edsp
"""
)
exit()
try:
with open(fname, "w") as f:
with subprocess.Popen(
["/usr/lib/apt/solvers/apt", "-oDebug::EDSP::WriteSolution=yes"],
stdin=sys.stdin.fileno(),
stdout=subprocess.PIPE,
bufsize=0, # unbuffered
text=True, # open in text mode
) as p:
for line in p.stdout:
print(line, end="")
f.write(line)
except (FileNotFoundError, PermissionError) as e:
print(
"""Error: ERR_CREATE_FILE
Message: Writing EDSP solver input to file '%s' failed as it couldn't be created!
"""
% fname
)