From c63ad873106697c4bdfd950eca16be844676a937 Mon Sep 17 00:00:00 2001 From: Johannes Schauer Marin Rodrigues Date: Mon, 16 Aug 2021 13:11:42 +0200 Subject: [PATCH] changes for release of Debian 11 Buster --- coverage.sh | 42 ++++++++-------- make_mirror.sh | 129 ++++++++++++++++++++++++++++++++++--------------- mmdebstrap | 15 +++++- 3 files changed, 124 insertions(+), 62 deletions(-) diff --git a/coverage.sh b/coverage.sh index 870892e..c375124 100755 --- a/coverage.sh +++ b/coverage.sh @@ -54,7 +54,7 @@ fi # check if all required debootstrap tarballs exist notfound=0 -for dist in stable testing unstable; do +for dist in oldstable stable testing unstable; do for variant in minbase buildd -; do if [ ! -e "shared/cache/debian-$dist-$variant.tar" ]; then echo "shared/cache/debian-$dist-$variant.tar does not exist" >&2 @@ -120,7 +120,7 @@ if [ ! -e shared/hooks/eatmydata/customize.sh ] || [ hooks/eatmydata/customize.s fi fi starttime= -total=193 +total=196 skipped=0 runtests=0 i=1 @@ -160,7 +160,7 @@ fi : "${CMD:=perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap}" mirror="http://127.0.0.1/debian" -for dist in stable testing unstable; do +for dist in oldstable stable testing unstable; do for variant in minbase buildd -; do print_header "mode=$defaultmode,variant=$variant: check against debootstrap $dist" cat << END > shared/test.sh @@ -265,7 +265,7 @@ if [ "$variant" = "-" ]; then cap=\$(chroot /tmp/debian-$dist-debootstrap /sbin/getcap /bin/ping) expected="/bin/ping cap_net_raw=ep" - if [ "$dist" = stable ]; then + if [ "$dist" = oldstable ]; then expected="/bin/ping = cap_net_raw+ep" fi if [ "\$cap" != "\$expected" ]; then @@ -678,18 +678,18 @@ for variant in essential apt minbase buildd important standard; do skipped=$((skipped+1)) continue fi - if [ "$variant" = "important" ] && [ "$DEFAULT_DIST" = "stable" ]; then - echo "skipping test on stable because /var/lib/systemd/catalog/database differs" >&2 + if [ "$variant" = "important" ] && [ "$DEFAULT_DIST" = "oldstable" ]; then + echo "skipping test on oldstable because /var/lib/systemd/catalog/database differs" >&2 skipped=$((skipped+1)) continue fi - if [ "$format" = "squashfs" ] && [ "$DEFAULT_DIST" = "stable" ]; then - echo "skipping test on stable because squashfs-tools-ng is not available" >&2 + if [ "$format" = "squashfs" ] && [ "$DEFAULT_DIST" = "oldstable" ]; then + echo "skipping test on oldstable because squashfs-tools-ng is not available" >&2 skipped=$((skipped+1)) continue fi - if [ "$format" = "ext2" ] && [ "$DEFAULT_DIST" = "stable" ]; then - echo "skipping test on stable because genext2fs does not support SOURCE_DATE_EPOCH" >&2 + if [ "$format" = "ext2" ] && [ "$DEFAULT_DIST" = "oldstable" ]; then + echo "skipping test on oldstable because genext2fs does not support SOURCE_DATE_EPOCH" >&2 skipped=$((skipped+1)) continue fi @@ -790,8 +790,8 @@ runuser -u user -- $CMD --unshare-helper /usr/sbin/chroot /tmp/debian-chroot get rm /tmp/debian-chroot.tar /tmp/debian-chroot-shifted.tar /tmp/debian-chroot.txt /tmp/debian-chroot-shiftedback.tar /tmp/expected rm -r /tmp/debian-chroot END -if [ "$DEFAULT_DIST" = "stable" ]; then - echo "the python3 tarfile module in stable does not preserve xattrs -- Skipping test..." >&2 +if [ "$DEFAULT_DIST" = "oldstable" ]; then + echo "the python3 tarfile module in oldstable does not preserve xattrs -- Skipping test..." >&2 skipped=$((skipped+1)) elif [ "$HAVE_QEMU" = "yes" ]; then ./run_qemu.sh @@ -1115,8 +1115,8 @@ sqfs2tar --no-skip --root-becomes . /tmp/debian-chroot.squashfs | tar -t \ | sort | diff -u /tmp/tar1noslash.txt - rm /tmp/debian-chroot.squashfs /tmp/tar1noslash.txt END -if [ "$DEFAULT_DIST" = "stable" ]; then - echo "skipping test on stable because squashfs-tools-ng is not available" >&2 +if [ "$DEFAULT_DIST" = "oldstable" ]; then + echo "skipping test on oldstable because squashfs-tools-ng is not available" >&2 skipped=$((skipped+1)) elif [ "$HAVE_QEMU" = "yes" ]; then ./run_qemu.sh @@ -1131,8 +1131,8 @@ fi for mode in root unshare fakechroot proot; do print_header "mode=$mode,variant=apt: test ext2 image" - if [ "$DEFAULT_DIST" = "stable" ]; then - echo "skipping test on stable because genext2fs does not support SOURCE_DATE_EPOCH" >&2 + if [ "$DEFAULT_DIST" = "oldstable" ]; then + echo "skipping test on oldstable because genext2fs does not support SOURCE_DATE_EPOCH" >&2 skipped=$((skipped+1)) continue fi @@ -1344,7 +1344,7 @@ $CMD --mode=root --variant=apt stable /tmp/debian-chroot cat << SOURCES | cmp /tmp/debian-chroot/etc/apt/sources.list deb http://deb.debian.org/debian stable main deb http://deb.debian.org/debian stable-updates main -deb http://security.debian.org/debian-security stable/updates main +deb http://security.debian.org/debian-security stable-security main SOURCES rm -r /tmp/debian-chroot END @@ -2933,8 +2933,8 @@ for variant in extract custom essential apt minbase buildd important standard; d skipped=$((skipped+1)) continue fi - if [ "$variant" = "important" ] && [ "$DEFAULT_DIST" = "stable" ]; then - echo "skipping test on stable because /var/lib/systemd/catalog/database differs" >&2 + if [ "$variant" = "important" ] && [ "$DEFAULT_DIST" = "oldstable" ]; then + echo "skipping test on oldstable because /var/lib/systemd/catalog/database differs" >&2 skipped=$((skipped+1)) continue fi @@ -3404,8 +3404,8 @@ prefix= [ "\$(id -u)" -eq 0 ] && prefix="runuser -u user --" \$prefix $CMD --mode=chrootless --variant=custom --include=bsdutils,coreutils,debianutils,diffutils,dpkg,findutils,grep,gzip,hostname,init-system-helpers,ncurses-base,ncurses-bin,perl-base,sed,sysvinit-utils,tar $DEFAULT_DIST /dev/null $mirror END -if [ "$DEFAULT_DIST" = "stable" ]; then - echo "chrootless doesn't work in stable -- Skipping test..." >&2 +if [ "$DEFAULT_DIST" = "oldstable" ]; then + echo "chrootless doesn't work in oldstable -- Skipping test..." >&2 skipped=$((skipped+1)) elif [ "$HAVE_QEMU" = "yes" ]; then ./run_qemu.sh diff --git a/make_mirror.sh b/make_mirror.sh index 8c54417..20268ff 100755 --- a/make_mirror.sh +++ b/make_mirror.sh @@ -20,7 +20,7 @@ deletecache() { return 1 fi # be very careful with removing the old directory - for dist in stable testing unstable; do + for dist in oldstable stable testing unstable; do for variant in minbase buildd -; do if [ -e "$dir/debian-$dist-$variant.tar" ]; then rm "$dir/debian-$dist-$variant.tar" @@ -33,18 +33,30 @@ deletecache() { else echo "does not exist: $dir/debian/dists/$dist" >&2 fi - if [ "$dist" = "stable" ]; then - if [ -e "$dir/debian/dists/stable-updates" ]; then - rm --one-file-system --recursive "$dir/debian/dists/stable-updates" + case "$dist" in oldstable|stable) + if [ -e "$dir/debian/dists/$dist-updates" ]; then + rm --one-file-system --recursive "$dir/debian/dists/$dist-updates" else - echo "does not exist: $dir/debian/dists/stable-updates" >&2 + echo "does not exist: $dir/debian/dists/$dist-updates" >&2 fi - if [ -e "$dir/debian-security/dists/stable/updates" ]; then - rm --one-file-system --recursive "$dir/debian-security/dists/stable/updates" - else - echo "does not exist: $dir/debian-security/dists/stable/updates" >&2 - fi - fi + ;; + esac + case "$dist" in + oldstable) + if [ -e "$dir/debian-security/dists/$dist/updates" ]; then + rm --one-file-system --recursive "$dir/debian-security/dists/$dist/updates" + else + echo "does not exist: $dir/debian-security/dists/$dist/updates" >&2 + fi + ;; + stable) + if [ -e "$dir/debian-security/dists/$dist-security" ]; then + rm --one-file-system --recursive "$dir/debian-security/dists/$dist-security" + else + echo "does not exist: $dir/debian-security/dists/$dist-security" >&2 + fi + ;; + esac done if [ -e $dir/debian-*.qcow ]; then rm --one-file-system "$dir"/debian-*.qcow @@ -216,7 +228,6 @@ END > "$rootdir/var/lib/dpkg/status" - APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update # before downloading packages and before replacing the old Packages @@ -225,10 +236,18 @@ END # packages that we already have { get_oldaptnames "$oldmirrordir" "dists/$dist/main/binary-$nativearch/Packages.gz" - if grep --quiet security.debian.org "$rootdir/etc/apt/sources.list"; then - get_oldaptnames "$oldmirrordir" "dists/stable-updates/main/binary-$nativearch/Packages.gz" - get_oldaptnames "$oldcachedir/debian-security" "dists/stable/updates/main/binary-$nativearch/Packages.gz" - fi + case "$dist" in oldstable|stable) + get_oldaptnames "$oldmirrordir" "dists/$dist-updates/main/binary-$nativearch/Packages.gz" + ;; + esac + case "$dist" in + oldstable) + get_oldaptnames "$oldcachedir/debian-security" "dists/$dist/updates/main/binary-$nativearch/Packages.gz" + ;; + stable) + get_oldaptnames "$oldcachedir/debian-security" "dists/$dist-security/main/binary-$nativearch/Packages.gz" + ;; + esac } | sort -u > "$rootdir/oldaptnames" pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \ @@ -248,16 +267,27 @@ END curl --location "$mirror/dists/$dist/Release" > "$newmirrordir/dists/$dist/Release" curl --location "$mirror/dists/$dist/Release.gpg" > "$newmirrordir/dists/$dist/Release.gpg" curl --location "$mirror/dists/$dist/main/binary-$nativearch/Packages.gz" > "$newmirrordir/dists/$dist/main/binary-$nativearch/Packages.gz" - if grep --quiet security.debian.org "$rootdir/etc/apt/sources.list"; then - mkdir -p "$newmirrordir/dists/stable-updates/main/binary-$nativearch/" - curl --location "$mirror/dists/stable-updates/Release" > "$newmirrordir/dists/stable-updates/Release" - curl --location "$mirror/dists/stable-updates/Release.gpg" > "$newmirrordir/dists/stable-updates/Release.gpg" - curl --location "$mirror/dists/stable-updates/main/binary-$nativearch/Packages.gz" > "$newmirrordir/dists/stable-updates/main/binary-$nativearch/Packages.gz" - mkdir -p "$newcachedir/debian-security/dists/stable/updates/main/binary-$nativearch/" - curl --location "$security_mirror/dists/stable/updates/Release" > "$newcachedir/debian-security/dists/stable/updates/Release" - curl --location "$security_mirror/dists/stable/updates/Release.gpg" > "$newcachedir/debian-security/dists/stable/updates/Release.gpg" - curl --location "$security_mirror/dists/stable/updates/main/binary-$nativearch/Packages.gz" > "$newcachedir/debian-security/dists/stable/updates/main/binary-$nativearch/Packages.gz" - fi + case "$dist" in oldstable|stable) + mkdir -p "$newmirrordir/dists/$dist-updates/main/binary-$nativearch/" + curl --location "$mirror/dists/$dist-updates/Release" > "$newmirrordir/dists/$dist-updates/Release" + curl --location "$mirror/dists/$dist-updates/Release.gpg" > "$newmirrordir/dists/$dist-updates/Release.gpg" + curl --location "$mirror/dists/$dist-updates/main/binary-$nativearch/Packages.gz" > "$newmirrordir/dists/$dist-updates/main/binary-$nativearch/Packages.gz" + ;; + esac + case "$dist" in + oldstable) + mkdir -p "$newcachedir/debian-security/dists/$dist/updates/main/binary-$nativearch/" + curl --location "$security_mirror/dists/$dist/updates/Release" > "$newcachedir/debian-security/dists/$dist/updates/Release" + curl --location "$security_mirror/dists/$dist/updates/Release.gpg" > "$newcachedir/debian-security/dists/$dist/updates/Release.gpg" + curl --location "$security_mirror/dists/$dist/updates/main/binary-$nativearch/Packages.gz" > "$newcachedir/debian-security/dists/$dist/updates/main/binary-$nativearch/Packages.gz" + ;; + stable) + mkdir -p "$newcachedir/debian-security/dists/$dist-security/main/binary-$nativearch/" + curl --location "$security_mirror/dists/$dist-security/Release" > "$newcachedir/debian-security/dists/$dist-security/Release" + curl --location "$security_mirror/dists/$dist-security/Release.gpg" > "$newcachedir/debian-security/dists/$dist-security/Release.gpg" + curl --location "$security_mirror/dists/$dist-security/main/binary-$nativearch/Packages.gz" > "$newcachedir/debian-security/dists/$dist-security/main/binary-$nativearch/Packages.gz" + ;; + esac # the deb files downloaded by apt must be moved to their right locations in the # pool directory @@ -269,10 +299,18 @@ END # This way, it doesn't matter where the mirror ends up storing the package. { get_newaptnames "$newmirrordir" "dists/$dist/main/binary-$nativearch/Packages.gz"; - if grep --quiet security.debian.org "$rootdir/etc/apt/sources.list"; then - get_newaptnames "$newmirrordir" "dists/stable-updates/main/binary-$nativearch/Packages.gz" - get_newaptnames "$newcachedir/debian-security" "dists/stable/updates/main/binary-$nativearch/Packages.gz" - fi + case "$dist" in oldstable|stable) + get_newaptnames "$newmirrordir" "dists/$dist-updates/main/binary-$nativearch/Packages.gz" + ;; + esac + case "$dist" in + oldstable) + get_newaptnames "$newcachedir/debian-security" "dists/$dist/updates/main/binary-$nativearch/Packages.gz" + ;; + stable) + get_newaptnames "$newcachedir/debian-security" "dists/$dist-security/main/binary-$nativearch/Packages.gz" + ;; + esac } | sort -u > "$rootdir/newaptnames" rm "$rootdir/var/cache/apt/archives/lock" @@ -363,22 +401,33 @@ else fi for nativearch in $arches; do - for dist in stable testing unstable; do + for dist in oldstable stable testing unstable; do # non-host architectures are only downloaded for $DEFAULT_DIST if [ $nativearch != $HOSTARCH ] && [ $DEFAULT_DIST != $dist ]; then continue fi - cat << END | update_cache "$dist" "$nativearch" + # we need a first pass without updates and security patches + # because otherwise, old package versions needed by + # debootstrap will not get included + echo "deb [arch=$nativearch] $mirror $dist $components" | update_cache "$dist" "$nativearch" + # we need to include the base mirror again or otherwise + # packages like build-essential will be missing + case "$dist" in + oldstable) + cat << END | update_cache "$dist" "$nativearch" deb [arch=$nativearch] $mirror $dist $components +deb [arch=$nativearch] $mirror $dist-updates main +deb [arch=$nativearch] $security_mirror $dist/updates main END - if [ "$dist" = "stable" ]; then - # starting wit bullseye, stable/updates becomes stable-security - cat << END | update_cache "$dist" "$nativearch" + ;; + stable) + cat << END | update_cache "$dist" "$nativearch" deb [arch=$nativearch] $mirror $dist $components -deb [arch=$nativearch] $mirror stable-updates main -deb [arch=$nativearch] $security_mirror stable/updates main +deb [arch=$nativearch] $mirror $dist-updates main +deb [arch=$nativearch] $security_mirror $dist-security main END - fi + ;; + esac done done @@ -427,7 +476,7 @@ if [ "$HAVE_QEMU" = "yes" ]; then trap "cleanuptmpdir; cleanup_newcachedir" EXIT INT TERM pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils - if [ "$DEFAULT_DIST" != "stable" ]; then + if [ "$DEFAULT_DIST" != "oldstable" ]; then pkgs="$pkgs,squashfs-tools-ng,genext2fs" fi if [ "$HAVE_PROOT" = "yes" ]; then @@ -579,7 +628,7 @@ END fi mirror="http://127.0.0.1/debian" -for dist in stable testing unstable; do +for dist in oldstable stable testing unstable; do for variant in minbase buildd -; do echo "running debootstrap --no-merged-usr --variant=$variant $dist \${TEMPDIR} $mirror" cat << END > shared/test.sh diff --git a/mmdebstrap b/mmdebstrap index 730bd1a..a0096f7 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -4163,6 +4163,8 @@ sub get_sourceslist_by_suite { or error "cannot open $distro_info: $!"; my $i = 0; my $matching_version; + my @releases; + my $today = POSIX::strftime "%Y-%m-%d", localtime; while (my $line = <$fh>) { chomp($line); $i++; @@ -4185,6 +4187,11 @@ sub get_sourceslist_by_suite { if ($i == 1) { next; } + if ( scalar @cells > 4 + and $cells[4] =~ m/^\d\d\d\d-\d\d-\d\d$/ + and $cells[4] lt $today) { + push @releases, $cells[0]; + } if (lc $cells[1] eq $suite or lc $cells[2] eq $suite) { $matching_version = $cells[0]; last; @@ -4194,9 +4201,15 @@ sub get_sourceslist_by_suite { if (defined $matching_version and $matching_version >= 11) { $bullseye_or_later = 1; } + if ($suite eq "stable" and $releases[-1] >= 11) { + $bullseye_or_later = 1; + } } else { # neither libdistro-info-perl nor distro-info-data is installed - if (any { $_ eq $suite } ('bullseye', 'bookworm', 'trixie')) { + if ( + any { $_ eq $suite } + ('stable', 'bullseye', 'bookworm', 'trixie') + ) { $bullseye_or_later = 1; } }