support deb822-style format apt sources

This commit is contained in:
Johannes 'josch' Schauer 2020-01-22 23:30:28 +01:00
parent 1579d06380
commit 62159d124a
Signed by untrusted user: josch
GPG key ID: F2CBA5C78FBD83E1
3 changed files with 336 additions and 15 deletions

View file

@ -72,7 +72,7 @@ if [ ! -e shared/taridshift ] || [ taridshift -nt shared/taridshift ]; then
fi fi
starttime= starttime=
total=138 total=140
skipped=0 skipped=0
runtests=0 runtests=0
i=1 i=1
@ -985,7 +985,9 @@ set -eu
export LC_ALL=C.UTF-8 export LC_ALL=C.UTF-8
echo "deb $mirror $DEFAULT_DIST main" > /tmp/sources.list echo "deb $mirror $DEFAULT_DIST main" > /tmp/sources.list
$CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST /tmp/debian-chroot.tar /tmp/sources.list $CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST /tmp/debian-chroot.tar /tmp/sources.list
tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt - tar -tf /tmp/debian-chroot.tar \
| sed 's#^./etc/apt/sources.list.d/0000sources.list\$#./etc/apt/sources.list#' \
| sort | diff -u tar1.txt -
rm /tmp/debian-chroot.tar /tmp/sources.list rm /tmp/debian-chroot.tar /tmp/sources.list
END END
if [ "$HAVE_QEMU" = "yes" ]; then if [ "$HAVE_QEMU" = "yes" ]; then
@ -999,6 +1001,125 @@ else
runtests=$((runtests+1)) runtests=$((runtests+1))
fi fi
print_header "mode=$defaultmode,variant=apt: test deb822 (1/2)"
cat << END > shared/test.sh
#!/bin/sh
set -eu
export LC_ALL=C.UTF-8
cat << SOURCES > /tmp/deb822.sources
Types: deb
URIs: ${mirror}1
Suites: $DEFAULT_DIST
Components: main
SOURCES
echo "deb ${mirror}2 $DEFAULT_DIST main" > /tmp/sources.list
echo "deb ${mirror}3 $DEFAULT_DIST main" \
| $CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST \
/tmp/debian-chroot \
/tmp/deb822.sources \
${mirror}4 \
- \
"deb ${mirror}5 $DEFAULT_DIST main" \
${mirror}6 \
/tmp/sources.list
test ! -e /tmp/debian-chroot/etc/apt/sources.list
cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0000deb822.sources -
Types: deb
URIs: ${mirror}1
Suites: $DEFAULT_DIST
Components: main
SOURCES
cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0001main.list -
deb ${mirror}4 $DEFAULT_DIST main
deb ${mirror}3 $DEFAULT_DIST main
deb ${mirror}5 $DEFAULT_DIST main
deb ${mirror}6 $DEFAULT_DIST main
SOURCES
echo "deb ${mirror}2 $DEFAULT_DIST main" | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0002sources.list -
tar -C /tmp/debian-chroot --one-file-system -c . \
| {
tar -t \
| grep -v "^./etc/apt/sources.list.d/0000deb822.sources$" \
| grep -v "^./etc/apt/sources.list.d/0001main.list$" \
| grep -v "^./etc/apt/sources.list.d/0002sources.list";
printf "./etc/apt/sources.list\n";
} | sort | diff -u tar1.txt -
rm -r /tmp/debian-chroot
rm /tmp/sources.list /tmp/deb822.sources
END
if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
runtests=$((runtests+1))
elif [ "$defaultmode" = "root" ]; then
./run_null.sh SUDO
runtests=$((runtests+1))
else
./run_null.sh
runtests=$((runtests+1))
fi
print_header "mode=$defaultmode,variant=apt: test deb822 (2/2)"
cat << END > shared/test.sh
#!/bin/sh
set -eu
export LC_ALL=C.UTF-8
cat << SOURCES > /tmp/deb822
Types: deb
URIs: ${mirror}1
Suites: $DEFAULT_DIST
Components: main
SOURCES
echo "deb ${mirror}2 $DEFAULT_DIST main" > /tmp/sources
cat << SOURCES | $CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST \
/tmp/debian-chroot \
/tmp/deb822 \
- \
/tmp/sources
Types: deb
URIs: ${mirror}3
Suites: $DEFAULT_DIST
Components: main
SOURCES
test ! -e /tmp/debian-chroot/etc/apt/sources.list
ls -lha /tmp/debian-chroot/etc/apt/sources.list.d/
cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0000deb822.sources -
Types: deb
URIs: ${mirror}1
Suites: $DEFAULT_DIST
Components: main
SOURCES
cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0001main.sources -
Types: deb
URIs: ${mirror}3
Suites: $DEFAULT_DIST
Components: main
SOURCES
echo "deb ${mirror}2 $DEFAULT_DIST main" | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0002sources.list -
tar -C /tmp/debian-chroot --one-file-system -c . \
| {
tar -t \
| grep -v "^./etc/apt/sources.list.d/0000deb822.sources$" \
| grep -v "^./etc/apt/sources.list.d/0001main.sources$" \
| grep -v "^./etc/apt/sources.list.d/0002sources.list$";
printf "./etc/apt/sources.list\n";
} | sort | diff -u tar1.txt -
rm -r /tmp/debian-chroot
rm /tmp/sources /tmp/deb822
END
if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
runtests=$((runtests+1))
elif [ "$defaultmode" = "root" ]; then
./run_null.sh SUDO
runtests=$((runtests+1))
else
./run_null.sh
runtests=$((runtests+1))
fi
print_header "mode=$defaultmode,variant=apt: automatic mirror from suite" print_header "mode=$defaultmode,variant=apt: automatic mirror from suite"
cat << END > shared/test.sh cat << END > shared/test.sh
#!/bin/sh #!/bin/sh

View file

@ -61,6 +61,12 @@ deletecache() {
else else
echo "does not exist: $dir/debian-security/pool/updates/main" >&2 echo "does not exist: $dir/debian-security/pool/updates/main" >&2
fi fi
for i in $(seq 1 6); do
if [ ! -e "$dir/debian$i" ]; then
continue
fi
rm "$dir/debian$i"
done
rm "$dir/mmdebstrapcache" rm "$dir/mmdebstrapcache"
# now the rest should only be empty directories # now the rest should only be empty directories
if [ -e "$dir" ]; then if [ -e "$dir" ]; then
@ -374,6 +380,14 @@ END
done done
done done
# Create some symlinks so that we can trick apt into accepting multiple apt
# lines that point to the same repository but look different. This is to
# avoid the warning:
# W: Target Packages (main/binary-all/Packages) is configured multiple times...
for i in $(seq 1 6); do
ln -s debian "$newcachedir/debian$i"
done
tmpdir="" tmpdir=""
cleanuptmpdir() { cleanuptmpdir() {

View file

@ -1371,12 +1371,70 @@ sub setup {
or error "cannot chmod fstab: $!"; or error "cannot chmod fstab: $!";
} }
# write /etc/apt/sources.list # write /etc/apt/sources.list and files in /etc/apt/sources.list.d/
{ {
open my $fh, '>', "$options->{root}/etc/apt/sources.list" my $firstentry = $options->{sourceslists}->[0];
or error "cannot open /etc/apt/sources.list: $!"; # if the first sources.list entry is of one-line type and without
print $fh $options->{sourceslist}; # explicit filename, then write out an actual /etc/apt/sources.list
# otherwise everything goes into /etc/apt/sources.list.d
my $fname;
if ($firstentry->{type} eq 'one-line'
&& !defined $firstentry->{fname}) {
$fname = "$options->{root}/etc/apt/sources.list";
} else {
$fname = "$options->{root}/etc/apt/sources.list.d/0000";
if (defined $firstentry->{fname}) {
$fname .= $firstentry->{fname};
if ( $firstentry->{fname} !~ /\.list/
&& $firstentry->{fname} !~ /\.sources/) {
if ($firstentry->{type} eq 'one-line') {
$fname .= '.list';
} elsif ($firstentry->{type} eq 'deb822') {
$fname .= '.sources';
} else {
error "invalid type: $firstentry->{type}";
}
}
} else {
# if no filename is given, then this must be a deb822 file
# because if it was a one-line type file, then it would've been
# written to /etc/apt/sources.list
$fname .= 'main.sources';
}
}
open my $fh, '>', "$fname" or error "cannot open $fname: $!";
print $fh $firstentry->{content};
close $fh; close $fh;
# everything else goes into /etc/apt/sources.list.d/
for (my $i = 1 ; $i < scalar @{ $options->{sourceslists} } ; $i++) {
my $entry = $options->{sourceslists}->[$i];
my $fname = "$options->{root}/etc/apt/sources.list.d/"
. sprintf("%04d", $i);
if (defined $entry->{fname}) {
$fname .= $entry->{fname};
if ( $entry->{fname} !~ /\.list/
&& $entry->{fname} !~ /\.sources/) {
if ($entry->{type} eq 'one-line') {
$fname .= '.list';
} elsif ($entry->{type} eq 'deb822') {
$fname .= '.sources';
} else {
error "invalid type: $entry->{type}";
}
}
} else {
if ($entry->{type} eq 'one-line') {
$fname .= 'main.list';
} elsif ($entry->{type} eq 'deb822') {
$fname .= 'main.sources';
} else {
error "invalid type: $entry->{type}";
}
}
open my $fh, '>', "$fname" or error "cannot open $fname: $!";
print $fh $entry->{content};
close $fh;
}
} }
# allow network access from within # allow network access from within
@ -2642,6 +2700,29 @@ sub hookhelper {
return; return;
} }
sub guess_sources_format {
my $content = shift;
my $is_deb822 = 0;
my $is_oneline = 0;
for my $line (split "\n", $content) {
if ($line =~ /^deb(-src)? /) {
$is_oneline = 1;
last;
}
if ($line =~ /^[^#:\s]+:/) {
$is_deb822 = 1;
last;
}
}
if ($is_deb822) {
return 'deb822';
}
if ($is_oneline) {
return 'one-line';
}
return;
}
sub main() { sub main() {
umask 022; umask 022;
@ -3116,12 +3197,27 @@ sub main() {
$options->{target} = '-'; $options->{target} = '-';
} }
my $sourceslist = ''; my $sourceslists = [];
if (!defined $suite) { if (!defined $suite) {
# If no suite was specified, then the whole sources.list has to # If no suite was specified, then the whole sources.list has to
# come from standard input # come from standard input
info "Reading sources.list from standard input..."; info "Reading sources.list from standard input...";
$sourceslist = do { local $/; <> }; my $content = do {
local $/;
## no critic (InputOutput::ProhibitExplicitStdin)
<STDIN>;
};
my $type = guess_sources_format($content);
if (!defined $type
|| ($type ne "deb822" and $type ne "one-line")) {
error "cannot determine sources.list format";
}
push @{$sourceslists},
{
type => $type,
fname => undef,
content => $content,
};
} else { } else {
my @components = (); my @components = ();
foreach my $comp (@{ $options->{components} }) { foreach my $comp (@{ $options->{components} }) {
@ -3282,18 +3378,90 @@ sub main() {
if (scalar @ARGV > 0) { if (scalar @ARGV > 0) {
for my $arg (@ARGV) { for my $arg (@ARGV) {
if ($arg eq '-') { if ($arg eq '-') {
info "Reading sources.list from standard input..."; info 'Reading sources.list from standard input...';
$sourceslist .= do { local $/; <> }; my $content = do {
local $/;
## no critic (InputOutput::ProhibitExplicitStdin)
<STDIN>;
};
my $type = guess_sources_format($content);
if (!defined $type
|| ($type ne 'deb822' and $type ne 'one-line')) {
error "cannot determine sources.list format";
}
# if last entry is of same type and without filename,
# then append
if ( scalar @{$sourceslists} > 0
&& $sourceslists->[-1]{type} eq $type
&& !defined $sourceslists->[-1]{fname}) {
$sourceslists->[-1]{content}
.= ($type eq 'one-line' ? "\n" : "\n\n")
. $content;
} else {
push @{$sourceslists},
{
type => $type,
fname => undef,
content => $content,
};
}
} elsif ($arg =~ /^deb(-src)? /) { } elsif ($arg =~ /^deb(-src)? /) {
$sourceslist .= "$arg\n"; my $content = "$arg\n";
# if last entry is of same type and without filename,
# then append
if ( scalar @{$sourceslists} > 0
&& $sourceslists->[-1]{type} eq 'one-line'
&& !defined $sourceslists->[-1]{fname}) {
$sourceslists->[-1]{content} .= "\n" . $content;
} else {
push @{$sourceslists},
{
type => 'one-line',
fname => undef,
content => $content,
};
}
} elsif ($arg =~ /:\/\//) { } elsif ($arg =~ /:\/\//) {
$sourceslist .= "deb$signedby $arg $suite $compstr\n"; my $content = "deb$signedby $arg $suite $compstr\n";
# if last entry is of same type and without filename,
# then append
if ( scalar @{$sourceslists} > 0
&& $sourceslists->[-1]{type} eq 'one-line'
&& !defined $sourceslists->[-1]{fname}) {
$sourceslists->[-1]{content} .= "\n" . $content;
} else {
push @{$sourceslists},
{
type => 'one-line',
fname => undef,
content => $content,
};
}
} elsif (-f $arg) { } elsif (-f $arg) {
my $content = '';
open my $fh, '<', $arg or error "cannot open $arg: $!"; open my $fh, '<', $arg or error "cannot open $arg: $!";
while (my $line = <$fh>) { while (my $line = <$fh>) {
$sourceslist .= $line; $content .= $line;
} }
close $fh; close $fh;
my $type = undef;
if ($arg =~ /\.list$/) {
$type = 'one-line';
} elsif ($arg =~ /\.sources$/) {
$type = 'deb822';
} else {
$type = guess_sources_format($content);
}
if (!defined $type
|| ($type ne 'deb822' and $type ne 'one-line')) {
error "cannot determine sources.list format";
}
push @{$sourceslists},
{
type => $type,
fname => basename($arg),
content => $content,
};
} else { } else {
error "invalid mirror: $arg"; error "invalid mirror: $arg";
} }
@ -3328,6 +3496,7 @@ sub main() {
} elsif (any { $_ eq $suite } @kali) { } elsif (any { $_ eq $suite } @kali) {
$mirror = 'https://http.kali.org/kali'; $mirror = 'https://http.kali.org/kali';
} }
my $sourceslist = '';
$sourceslist .= "deb$signedby $mirror $suite $compstr\n"; $sourceslist .= "deb$signedby $mirror $suite $compstr\n";
if (any { $_ eq $suite } @ubuntustable) { if (any { $_ eq $suite } @ubuntustable) {
$sourceslist $sourceslist
@ -3358,12 +3527,29 @@ sub main() {
. " $compstr\n"; . " $compstr\n";
} }
} }
push @{$sourceslists},
{
type => 'one-line',
fname => undef,
content => $sourceslist,
};
} }
} }
if ($sourceslist eq '') { if (scalar @{$sourceslists} == 0) {
error "empty apt sources.list"; error "empty apt sources.list";
} }
$options->{sourceslist} = $sourceslist; debug("sources list entries:");
for my $list (@{$sourceslists}) {
if (defined $list->{fname}) {
debug("fname: $list->{fname}");
}
debug("type: $list->{type}");
debug("content:");
for my $line (split "\n", $list->{content}) {
debug(" $line");
}
}
$options->{sourceslists} = $sourceslists;
} }
if ($options->{target} ne '-') { if ($options->{target} ne '-') {