support deb822-style format apt sources

coverage.sh

@@ -72,7 +72,7 @@ if [ ! -e shared/taridshift ] || [ taridshift -nt shared/taridshift ]; then
 fi
 
 starttime=
-total=138
+total=140
 skipped=0
 runtests=0
 i=1
@@ -985,7 +985,9 @@ set -eu
 export LC_ALL=C.UTF-8
 echo "deb $mirror $DEFAULT_DIST main" > /tmp/sources.list
 $CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST /tmp/debian-chroot.tar /tmp/sources.list
-tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
+tar -tf /tmp/debian-chroot.tar \
+	| sed 's#^./etc/apt/sources.list.d/0000sources.list\$#./etc/apt/sources.list#' \
+	| sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar /tmp/sources.list
 END
 if [ "$HAVE_QEMU" = "yes" ]; then
@@ -999,6 +1001,125 @@ else
 	runtests=$((runtests+1))
 fi
 
+print_header "mode=$defaultmode,variant=apt: test deb822 (1/2)"
+cat << END > shared/test.sh
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+cat << SOURCES > /tmp/deb822.sources
+Types: deb
+URIs: ${mirror}1
+Suites: $DEFAULT_DIST
+Components: main
+SOURCES
+echo "deb ${mirror}2 $DEFAULT_DIST main" > /tmp/sources.list
+echo "deb ${mirror}3 $DEFAULT_DIST main" \
+	| $CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST \
+		/tmp/debian-chroot \
+		/tmp/deb822.sources \
+		${mirror}4 \
+		- \
+		"deb ${mirror}5 $DEFAULT_DIST main" \
+		${mirror}6 \
+		/tmp/sources.list
+test ! -e /tmp/debian-chroot/etc/apt/sources.list
+cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0000deb822.sources -
+Types: deb
+URIs: ${mirror}1
+Suites: $DEFAULT_DIST
+Components: main
+SOURCES
+cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0001main.list -
+deb ${mirror}4 $DEFAULT_DIST main
+
+deb ${mirror}3 $DEFAULT_DIST main
+
+deb ${mirror}5 $DEFAULT_DIST main
+
+deb ${mirror}6 $DEFAULT_DIST main
+SOURCES
+echo "deb ${mirror}2 $DEFAULT_DIST main" | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0002sources.list -
+tar -C /tmp/debian-chroot --one-file-system -c . \
+	| {
+		tar -t \
+			| grep -v "^./etc/apt/sources.list.d/0000deb822.sources$" \
+			| grep -v "^./etc/apt/sources.list.d/0001main.list$" \
+			| grep -v "^./etc/apt/sources.list.d/0002sources.list";
+		printf "./etc/apt/sources.list\n";
+	} | sort | diff -u tar1.txt -
+rm -r /tmp/debian-chroot
+rm /tmp/sources.list /tmp/deb822.sources
+END
+if [ "$HAVE_QEMU" = "yes" ]; then
+	./run_qemu.sh
+	runtests=$((runtests+1))
+elif [ "$defaultmode" = "root" ]; then
+	./run_null.sh SUDO
+	runtests=$((runtests+1))
+else
+	./run_null.sh
+	runtests=$((runtests+1))
+fi
+
+print_header "mode=$defaultmode,variant=apt: test deb822 (2/2)"
+cat << END > shared/test.sh
+#!/bin/sh
+set -eu
+export LC_ALL=C.UTF-8
+cat << SOURCES > /tmp/deb822
+Types: deb
+URIs: ${mirror}1
+Suites: $DEFAULT_DIST
+Components: main
+SOURCES
+echo "deb ${mirror}2 $DEFAULT_DIST main" > /tmp/sources
+cat << SOURCES | $CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST \
+		/tmp/debian-chroot \
+		/tmp/deb822 \
+		- \
+		/tmp/sources
+Types: deb
+URIs: ${mirror}3
+Suites: $DEFAULT_DIST
+Components: main
+SOURCES
+test ! -e /tmp/debian-chroot/etc/apt/sources.list
+ls -lha /tmp/debian-chroot/etc/apt/sources.list.d/
+cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0000deb822.sources -
+Types: deb
+URIs: ${mirror}1
+Suites: $DEFAULT_DIST
+Components: main
+SOURCES
+cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0001main.sources -
+Types: deb
+URIs: ${mirror}3
+Suites: $DEFAULT_DIST
+Components: main
+SOURCES
+echo "deb ${mirror}2 $DEFAULT_DIST main" | cmp /tmp/debian-chroot/etc/apt/sources.list.d/0002sources.list -
+tar -C /tmp/debian-chroot --one-file-system -c . \
+	| {
+		tar -t \
+			| grep -v "^./etc/apt/sources.list.d/0000deb822.sources$" \
+			| grep -v "^./etc/apt/sources.list.d/0001main.sources$" \
+			| grep -v "^./etc/apt/sources.list.d/0002sources.list$";
+		printf "./etc/apt/sources.list\n";
+	} | sort | diff -u tar1.txt -
+rm -r /tmp/debian-chroot
+rm /tmp/sources /tmp/deb822
+END
+if [ "$HAVE_QEMU" = "yes" ]; then
+	./run_qemu.sh
+	runtests=$((runtests+1))
+elif [ "$defaultmode" = "root" ]; then
+	./run_null.sh SUDO
+	runtests=$((runtests+1))
+else
+	./run_null.sh
+	runtests=$((runtests+1))
+fi
+
 print_header "mode=$defaultmode,variant=apt: automatic mirror from suite"
 cat << END > shared/test.sh
 #!/bin/sh

make_mirror.sh

@@ -61,6 +61,12 @@ deletecache() {
 	else
 		echo "does not exist: $dir/debian-security/pool/updates/main" >&2
 	fi
+	for i in $(seq 1 6); do
+		if [ ! -e "$dir/debian$i" ]; then
+			continue
+		fi
+		rm "$dir/debian$i"
+	done
 	rm "$dir/mmdebstrapcache"
 	# now the rest should only be empty directories
 	if [ -e "$dir" ]; then
@@ -374,6 +380,14 @@ END
 	done
 done
 
+# Create some symlinks so that we can trick apt into accepting multiple apt
+# lines that point to the same repository but look different. This is to
+# avoid the warning:
+# W: Target Packages (main/binary-all/Packages) is configured multiple times...
+for i in $(seq 1 6); do
+	ln -s debian "$newcachedir/debian$i"
+done
+
 tmpdir=""
 
 cleanuptmpdir() {

mmdebstrap

@@ -1371,12 +1371,70 @@ sub setup {
           or error "cannot chmod fstab: $!";
     }
 
-    # write /etc/apt/sources.list
+    # write /etc/apt/sources.list and files in /etc/apt/sources.list.d/
     {
-        open my $fh, '>', "$options->{root}/etc/apt/sources.list"
+        my $firstentry = $options->{sourceslists}->[0];
-          or error "cannot open /etc/apt/sources.list: $!";
+        # if the first sources.list entry is of one-line type and without
-        print $fh $options->{sourceslist};
+        # explicit filename, then write out an actual /etc/apt/sources.list
+        # otherwise everything goes into /etc/apt/sources.list.d
+        my $fname;
+        if ($firstentry->{type} eq 'one-line'
+            && !defined $firstentry->{fname}) {
+            $fname = "$options->{root}/etc/apt/sources.list";
+        } else {
+            $fname = "$options->{root}/etc/apt/sources.list.d/0000";
+            if (defined $firstentry->{fname}) {
+                $fname .= $firstentry->{fname};
+                if (   $firstentry->{fname} !~ /\.list/
+                    && $firstentry->{fname} !~ /\.sources/) {
+                    if ($firstentry->{type} eq 'one-line') {
+                        $fname .= '.list';
+                    } elsif ($firstentry->{type} eq 'deb822') {
+                        $fname .= '.sources';
+                    } else {
+                        error "invalid type: $firstentry->{type}";
+                    }
+                }
+            } else {
+                # if no filename is given, then this must be a deb822 file
+                # because if it was a one-line type file, then it would've been
+                # written to /etc/apt/sources.list
+                $fname .= 'main.sources';
+            }
+        }
+        open my $fh, '>', "$fname" or error "cannot open $fname: $!";
+        print $fh $firstentry->{content};
         close $fh;
+        # everything else goes into /etc/apt/sources.list.d/
+        for (my $i = 1 ; $i < scalar @{ $options->{sourceslists} } ; $i++) {
+            my $entry = $options->{sourceslists}->[$i];
+            my $fname = "$options->{root}/etc/apt/sources.list.d/"
+              . sprintf("%04d", $i);
+            if (defined $entry->{fname}) {
+                $fname .= $entry->{fname};
+                if (   $entry->{fname} !~ /\.list/
+                    && $entry->{fname} !~ /\.sources/) {
+                    if ($entry->{type} eq 'one-line') {
+                        $fname .= '.list';
+                    } elsif ($entry->{type} eq 'deb822') {
+                        $fname .= '.sources';
+                    } else {
+                        error "invalid type: $entry->{type}";
+                    }
+                }
+            } else {
+                if ($entry->{type} eq 'one-line') {
+                    $fname .= 'main.list';
+                } elsif ($entry->{type} eq 'deb822') {
+                    $fname .= 'main.sources';
+                } else {
+                    error "invalid type: $entry->{type}";
+                }
+            }
+            open my $fh, '>', "$fname" or error "cannot open $fname: $!";
+            print $fh $entry->{content};
+            close $fh;
+        }
     }
 
     # allow network access from within
@@ -2642,6 +2700,29 @@ sub hookhelper {
     return;
 }
 
+sub guess_sources_format {
+    my $content    = shift;
+    my $is_deb822  = 0;
+    my $is_oneline = 0;
+    for my $line (split "\n", $content) {
+        if ($line =~ /^deb(-src)? /) {
+            $is_oneline = 1;
+            last;
+        }
+        if ($line =~ /^[^#:\s]+:/) {
+            $is_deb822 = 1;
+            last;
+        }
+    }
+    if ($is_deb822) {
+        return 'deb822';
+    }
+    if ($is_oneline) {
+        return 'one-line';
+    }
+    return;
+}
+
 sub main() {
     umask 022;
 
@@ -3116,12 +3197,27 @@ sub main() {
             $options->{target} = '-';
         }
 
-        my $sourceslist = '';
+        my $sourceslists = [];
         if (!defined $suite) {
             # If no suite was specified, then the whole sources.list has to
             # come from standard input
             info "Reading sources.list from standard input...";
-            $sourceslist = do { local $/; <> };
+            my $content = do {
+                local $/;
+                ## no critic (InputOutput::ProhibitExplicitStdin)
+                <STDIN>;
+            };
+            my $type = guess_sources_format($content);
+            if (!defined $type
+                || ($type ne "deb822" and $type ne "one-line")) {
+                error "cannot determine sources.list format";
+            }
+            push @{$sourceslists},
+              {
+                type    => $type,
+                fname   => undef,
+                content => $content,
+              };
         } else {
             my @components = ();
             foreach my $comp (@{ $options->{components} }) {
@@ -3282,18 +3378,90 @@ sub main() {
             if (scalar @ARGV > 0) {
                 for my $arg (@ARGV) {
                     if ($arg eq '-') {
-                        info "Reading sources.list from standard input...";
+                        info 'Reading sources.list from standard input...';
-                        $sourceslist .= do { local $/; <> };
+                        my $content = do {
+                            local $/;
+                            ## no critic (InputOutput::ProhibitExplicitStdin)
+                            <STDIN>;
+                        };
+                        my $type = guess_sources_format($content);
+                        if (!defined $type
+                            || ($type ne 'deb822' and $type ne 'one-line')) {
+                            error "cannot determine sources.list format";
+                        }
+                        # if last entry is of same type and without filename,
+                        # then append
+                        if (   scalar @{$sourceslists} > 0
+                            && $sourceslists->[-1]{type} eq $type
+                            && !defined $sourceslists->[-1]{fname}) {
+                            $sourceslists->[-1]{content}
+                              .= ($type eq 'one-line' ? "\n" : "\n\n")
+                              . $content;
+                        } else {
+                            push @{$sourceslists},
+                              {
+                                type    => $type,
+                                fname   => undef,
+                                content => $content,
+                              };
+                        }
                     } elsif ($arg =~ /^deb(-src)? /) {
-                        $sourceslist .= "$arg\n";
+                        my $content = "$arg\n";
+                        # if last entry is of same type and without filename,
+                        # then append
+                        if (   scalar @{$sourceslists} > 0
+                            && $sourceslists->[-1]{type} eq 'one-line'
+                            && !defined $sourceslists->[-1]{fname}) {
+                            $sourceslists->[-1]{content} .= "\n" . $content;
+                        } else {
+                            push @{$sourceslists},
+                              {
+                                type    => 'one-line',
+                                fname   => undef,
+                                content => $content,
+                              };
+                        }
                     } elsif ($arg =~ /:\/\//) {
-                        $sourceslist .= "deb$signedby $arg $suite $compstr\n";
+                        my $content = "deb$signedby $arg $suite $compstr\n";
+                        # if last entry is of same type and without filename,
+                        # then append
+                        if (   scalar @{$sourceslists} > 0
+                            && $sourceslists->[-1]{type} eq 'one-line'
+                            && !defined $sourceslists->[-1]{fname}) {
+                            $sourceslists->[-1]{content} .= "\n" . $content;
+                        } else {
+                            push @{$sourceslists},
+                              {
+                                type    => 'one-line',
+                                fname   => undef,
+                                content => $content,
+                              };
+                        }
                     } elsif (-f $arg) {
+                        my $content = '';
                         open my $fh, '<', $arg or error "cannot open $arg: $!";
                         while (my $line = <$fh>) {
-                            $sourceslist .= $line;
+                            $content .= $line;
                         }
                         close $fh;
+                        my $type = undef;
+                        if ($arg =~ /\.list$/) {
+                            $type = 'one-line';
+                        } elsif ($arg =~ /\.sources$/) {
+                            $type = 'deb822';
+                        } else {
+                            $type = guess_sources_format($content);
+                        }
+                        if (!defined $type
+                            || ($type ne 'deb822' and $type ne 'one-line')) {
+                            error "cannot determine sources.list format";
+                        }
+                        push @{$sourceslists},
+                          {
+                            type    => $type,
+                            fname   => basename($arg),
+                            content => $content,
+                          };
                     } else {
                         error "invalid mirror: $arg";
                     }
@@ -3328,6 +3496,7 @@ sub main() {
                 } elsif (any { $_ eq $suite } @kali) {
                     $mirror = 'https://http.kali.org/kali';
                 }
+                my $sourceslist = '';
                 $sourceslist .= "deb$signedby $mirror $suite $compstr\n";
                 if (any { $_ eq $suite } @ubuntustable) {
                     $sourceslist
@@ -3358,12 +3527,29 @@ sub main() {
                           . " $compstr\n";
                     }
                 }
+                push @{$sourceslists},
+                  {
+                    type    => 'one-line',
+                    fname   => undef,
+                    content => $sourceslist,
+                  };
             }
         }
-        if ($sourceslist eq '') {
+        if (scalar @{$sourceslists} == 0) {
             error "empty apt sources.list";
         }
-        $options->{sourceslist} = $sourceslist;
+        debug("sources list entries:");
+        for my $list (@{$sourceslists}) {
+            if (defined $list->{fname}) {
+                debug("fname: $list->{fname}");
+            }
+            debug("type: $list->{type}");
+            debug("content:");
+            for my $line (split "\n", $list->{content}) {
+                debug("  $line");
+            }
+        }
+        $options->{sourceslists} = $sourceslists;
     }
 
     if ($options->{target} ne '-') {