From 62159d124a484fb5a4e8737dbe4070a6b3f6d1db Mon Sep 17 00:00:00 2001 From: Johannes 'josch' Schauer Date: Wed, 22 Jan 2020 23:30:28 +0100 Subject: [PATCH] support deb822-style format apt sources --- coverage.sh | 125 ++++++++++++++++++++++++++++- make_mirror.sh | 14 ++++ mmdebstrap | 212 ++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 336 insertions(+), 15 deletions(-) diff --git a/coverage.sh b/coverage.sh index 92c719d..b755ee2 100755 --- a/coverage.sh +++ b/coverage.sh @@ -72,7 +72,7 @@ if [ ! -e shared/taridshift ] || [ taridshift -nt shared/taridshift ]; then fi starttime= -total=138 +total=140 skipped=0 runtests=0 i=1 @@ -985,7 +985,9 @@ set -eu export LC_ALL=C.UTF-8 echo "deb $mirror $DEFAULT_DIST main" > /tmp/sources.list $CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST /tmp/debian-chroot.tar /tmp/sources.list -tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt - +tar -tf /tmp/debian-chroot.tar \ + | sed 's#^./etc/apt/sources.list.d/0000sources.list\$#./etc/apt/sources.list#' \ + | sort | diff -u tar1.txt - rm /tmp/debian-chroot.tar /tmp/sources.list END if [ "$HAVE_QEMU" = "yes" ]; then @@ -999,6 +1001,125 @@ else runtests=$((runtests+1)) fi +print_header "mode=$defaultmode,variant=apt: test deb822 (1/2)" +cat << END > shared/test.sh +#!/bin/sh +set -eu +export LC_ALL=C.UTF-8 +cat << SOURCES > /tmp/deb822.sources +Types: deb +URIs: ${mirror}1 +Suites: $DEFAULT_DIST +Components: main +SOURCES +echo "deb ${mirror}2 $DEFAULT_DIST main" > /tmp/sources.list +echo "deb ${mirror}3 $DEFAULT_DIST main" \ + | $CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST \ + /tmp/debian-chroot \ + /tmp/deb822.sources \ + ${mirror}4 \ + - \ + "deb ${mirror}5 $DEFAULT_DIST main" \ + ${mirror}6 \ + /tmp/sources.list +test ! -e /tmp/debian-chroot/etc/apt/sources.list +cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0000deb822.sources - +Types: deb +URIs: ${mirror}1 +Suites: $DEFAULT_DIST +Components: main +SOURCES +cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0001main.list - +deb ${mirror}4 $DEFAULT_DIST main + +deb ${mirror}3 $DEFAULT_DIST main + +deb ${mirror}5 $DEFAULT_DIST main + +deb ${mirror}6 $DEFAULT_DIST main +SOURCES +echo "deb ${mirror}2 $DEFAULT_DIST main" | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0002sources.list - +tar -C /tmp/debian-chroot --one-file-system -c . \ + | { + tar -t \ + | grep -v "^./etc/apt/sources.list.d/0000deb822.sources$" \ + | grep -v "^./etc/apt/sources.list.d/0001main.list$" \ + | grep -v "^./etc/apt/sources.list.d/0002sources.list"; + printf "./etc/apt/sources.list\n"; + } | sort | diff -u tar1.txt - +rm -r /tmp/debian-chroot +rm /tmp/sources.list /tmp/deb822.sources +END +if [ "$HAVE_QEMU" = "yes" ]; then + ./run_qemu.sh + runtests=$((runtests+1)) +elif [ "$defaultmode" = "root" ]; then + ./run_null.sh SUDO + runtests=$((runtests+1)) +else + ./run_null.sh + runtests=$((runtests+1)) +fi + +print_header "mode=$defaultmode,variant=apt: test deb822 (2/2)" +cat << END > shared/test.sh +#!/bin/sh +set -eu +export LC_ALL=C.UTF-8 +cat << SOURCES > /tmp/deb822 +Types: deb +URIs: ${mirror}1 +Suites: $DEFAULT_DIST +Components: main +SOURCES +echo "deb ${mirror}2 $DEFAULT_DIST main" > /tmp/sources +cat << SOURCES | $CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST \ + /tmp/debian-chroot \ + /tmp/deb822 \ + - \ + /tmp/sources +Types: deb +URIs: ${mirror}3 +Suites: $DEFAULT_DIST +Components: main +SOURCES +test ! -e /tmp/debian-chroot/etc/apt/sources.list +ls -lha /tmp/debian-chroot/etc/apt/sources.list.d/ +cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0000deb822.sources - +Types: deb +URIs: ${mirror}1 +Suites: $DEFAULT_DIST +Components: main +SOURCES +cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0001main.sources - +Types: deb +URIs: ${mirror}3 +Suites: $DEFAULT_DIST +Components: main +SOURCES +echo "deb ${mirror}2 $DEFAULT_DIST main" | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0002sources.list - +tar -C /tmp/debian-chroot --one-file-system -c . \ + | { + tar -t \ + | grep -v "^./etc/apt/sources.list.d/0000deb822.sources$" \ + | grep -v "^./etc/apt/sources.list.d/0001main.sources$" \ + | grep -v "^./etc/apt/sources.list.d/0002sources.list$"; + printf "./etc/apt/sources.list\n"; + } | sort | diff -u tar1.txt - +rm -r /tmp/debian-chroot +rm /tmp/sources /tmp/deb822 +END +if [ "$HAVE_QEMU" = "yes" ]; then + ./run_qemu.sh + runtests=$((runtests+1)) +elif [ "$defaultmode" = "root" ]; then + ./run_null.sh SUDO + runtests=$((runtests+1)) +else + ./run_null.sh + runtests=$((runtests+1)) +fi + print_header "mode=$defaultmode,variant=apt: automatic mirror from suite" cat << END > shared/test.sh #!/bin/sh diff --git a/make_mirror.sh b/make_mirror.sh index e24adfc..d7fc31a 100755 --- a/make_mirror.sh +++ b/make_mirror.sh @@ -61,6 +61,12 @@ deletecache() { else echo "does not exist: $dir/debian-security/pool/updates/main" >&2 fi + for i in $(seq 1 6); do + if [ ! -e "$dir/debian$i" ]; then + continue + fi + rm "$dir/debian$i" + done rm "$dir/mmdebstrapcache" # now the rest should only be empty directories if [ -e "$dir" ]; then @@ -374,6 +380,14 @@ END done done +# Create some symlinks so that we can trick apt into accepting multiple apt +# lines that point to the same repository but look different. This is to +# avoid the warning: +# W: Target Packages (main/binary-all/Packages) is configured multiple times... +for i in $(seq 1 6); do + ln -s debian "$newcachedir/debian$i" +done + tmpdir="" cleanuptmpdir() { diff --git a/mmdebstrap b/mmdebstrap index 52d386c..1e9c209 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -1371,12 +1371,70 @@ sub setup { or error "cannot chmod fstab: $!"; } - # write /etc/apt/sources.list + # write /etc/apt/sources.list and files in /etc/apt/sources.list.d/ { - open my $fh, '>', "$options->{root}/etc/apt/sources.list" - or error "cannot open /etc/apt/sources.list: $!"; - print $fh $options->{sourceslist}; + my $firstentry = $options->{sourceslists}->[0]; + # if the first sources.list entry is of one-line type and without + # explicit filename, then write out an actual /etc/apt/sources.list + # otherwise everything goes into /etc/apt/sources.list.d + my $fname; + if ($firstentry->{type} eq 'one-line' + && !defined $firstentry->{fname}) { + $fname = "$options->{root}/etc/apt/sources.list"; + } else { + $fname = "$options->{root}/etc/apt/sources.list.d/0000"; + if (defined $firstentry->{fname}) { + $fname .= $firstentry->{fname}; + if ( $firstentry->{fname} !~ /\.list/ + && $firstentry->{fname} !~ /\.sources/) { + if ($firstentry->{type} eq 'one-line') { + $fname .= '.list'; + } elsif ($firstentry->{type} eq 'deb822') { + $fname .= '.sources'; + } else { + error "invalid type: $firstentry->{type}"; + } + } + } else { + # if no filename is given, then this must be a deb822 file + # because if it was a one-line type file, then it would've been + # written to /etc/apt/sources.list + $fname .= 'main.sources'; + } + } + open my $fh, '>', "$fname" or error "cannot open $fname: $!"; + print $fh $firstentry->{content}; close $fh; + # everything else goes into /etc/apt/sources.list.d/ + for (my $i = 1 ; $i < scalar @{ $options->{sourceslists} } ; $i++) { + my $entry = $options->{sourceslists}->[$i]; + my $fname = "$options->{root}/etc/apt/sources.list.d/" + . sprintf("%04d", $i); + if (defined $entry->{fname}) { + $fname .= $entry->{fname}; + if ( $entry->{fname} !~ /\.list/ + && $entry->{fname} !~ /\.sources/) { + if ($entry->{type} eq 'one-line') { + $fname .= '.list'; + } elsif ($entry->{type} eq 'deb822') { + $fname .= '.sources'; + } else { + error "invalid type: $entry->{type}"; + } + } + } else { + if ($entry->{type} eq 'one-line') { + $fname .= 'main.list'; + } elsif ($entry->{type} eq 'deb822') { + $fname .= 'main.sources'; + } else { + error "invalid type: $entry->{type}"; + } + } + open my $fh, '>', "$fname" or error "cannot open $fname: $!"; + print $fh $entry->{content}; + close $fh; + } } # allow network access from within @@ -2642,6 +2700,29 @@ sub hookhelper { return; } +sub guess_sources_format { + my $content = shift; + my $is_deb822 = 0; + my $is_oneline = 0; + for my $line (split "\n", $content) { + if ($line =~ /^deb(-src)? /) { + $is_oneline = 1; + last; + } + if ($line =~ /^[^#:\s]+:/) { + $is_deb822 = 1; + last; + } + } + if ($is_deb822) { + return 'deb822'; + } + if ($is_oneline) { + return 'one-line'; + } + return; +} + sub main() { umask 022; @@ -3116,12 +3197,27 @@ sub main() { $options->{target} = '-'; } - my $sourceslist = ''; + my $sourceslists = []; if (!defined $suite) { # If no suite was specified, then the whole sources.list has to # come from standard input info "Reading sources.list from standard input..."; - $sourceslist = do { local $/; <> }; + my $content = do { + local $/; + ## no critic (InputOutput::ProhibitExplicitStdin) + ; + }; + my $type = guess_sources_format($content); + if (!defined $type + || ($type ne "deb822" and $type ne "one-line")) { + error "cannot determine sources.list format"; + } + push @{$sourceslists}, + { + type => $type, + fname => undef, + content => $content, + }; } else { my @components = (); foreach my $comp (@{ $options->{components} }) { @@ -3282,18 +3378,90 @@ sub main() { if (scalar @ARGV > 0) { for my $arg (@ARGV) { if ($arg eq '-') { - info "Reading sources.list from standard input..."; - $sourceslist .= do { local $/; <> }; + info 'Reading sources.list from standard input...'; + my $content = do { + local $/; + ## no critic (InputOutput::ProhibitExplicitStdin) + ; + }; + my $type = guess_sources_format($content); + if (!defined $type + || ($type ne 'deb822' and $type ne 'one-line')) { + error "cannot determine sources.list format"; + } + # if last entry is of same type and without filename, + # then append + if ( scalar @{$sourceslists} > 0 + && $sourceslists->[-1]{type} eq $type + && !defined $sourceslists->[-1]{fname}) { + $sourceslists->[-1]{content} + .= ($type eq 'one-line' ? "\n" : "\n\n") + . $content; + } else { + push @{$sourceslists}, + { + type => $type, + fname => undef, + content => $content, + }; + } } elsif ($arg =~ /^deb(-src)? /) { - $sourceslist .= "$arg\n"; + my $content = "$arg\n"; + # if last entry is of same type and without filename, + # then append + if ( scalar @{$sourceslists} > 0 + && $sourceslists->[-1]{type} eq 'one-line' + && !defined $sourceslists->[-1]{fname}) { + $sourceslists->[-1]{content} .= "\n" . $content; + } else { + push @{$sourceslists}, + { + type => 'one-line', + fname => undef, + content => $content, + }; + } } elsif ($arg =~ /:\/\//) { - $sourceslist .= "deb$signedby $arg $suite $compstr\n"; + my $content = "deb$signedby $arg $suite $compstr\n"; + # if last entry is of same type and without filename, + # then append + if ( scalar @{$sourceslists} > 0 + && $sourceslists->[-1]{type} eq 'one-line' + && !defined $sourceslists->[-1]{fname}) { + $sourceslists->[-1]{content} .= "\n" . $content; + } else { + push @{$sourceslists}, + { + type => 'one-line', + fname => undef, + content => $content, + }; + } } elsif (-f $arg) { + my $content = ''; open my $fh, '<', $arg or error "cannot open $arg: $!"; while (my $line = <$fh>) { - $sourceslist .= $line; + $content .= $line; } close $fh; + my $type = undef; + if ($arg =~ /\.list$/) { + $type = 'one-line'; + } elsif ($arg =~ /\.sources$/) { + $type = 'deb822'; + } else { + $type = guess_sources_format($content); + } + if (!defined $type + || ($type ne 'deb822' and $type ne 'one-line')) { + error "cannot determine sources.list format"; + } + push @{$sourceslists}, + { + type => $type, + fname => basename($arg), + content => $content, + }; } else { error "invalid mirror: $arg"; } @@ -3328,6 +3496,7 @@ sub main() { } elsif (any { $_ eq $suite } @kali) { $mirror = 'https://http.kali.org/kali'; } + my $sourceslist = ''; $sourceslist .= "deb$signedby $mirror $suite $compstr\n"; if (any { $_ eq $suite } @ubuntustable) { $sourceslist @@ -3358,12 +3527,29 @@ sub main() { . " $compstr\n"; } } + push @{$sourceslists}, + { + type => 'one-line', + fname => undef, + content => $sourceslist, + }; } } - if ($sourceslist eq '') { + if (scalar @{$sourceslists} == 0) { error "empty apt sources.list"; } - $options->{sourceslist} = $sourceslist; + debug("sources list entries:"); + for my $list (@{$sourceslists}) { + if (defined $list->{fname}) { + debug("fname: $list->{fname}"); + } + debug("type: $list->{type}"); + debug("content:"); + for my $line (split "\n", $list->{content}) { + debug(" $line"); + } + } + $options->{sourceslists} = $sourceslists; } if ($options->{target} ne '-') {