CHANGELOG.md

@@ -1,8 +1,3 @@
-1.3.6 (2023-06-16)
-------------------
-
- - bugfix release
-
 1.3.5 (2023-03-20)
 ------------------
 

caching_proxy.py

@@ -57,8 +57,6 @@ class ProxyRequestHandler(http.server.BaseHTTPRequestHandler):
             self.send_header("Content-Length", oldpath.stat().st_size)
             self.end_headers()
             with oldpath.open(mode="rb") as old, newpath.open(mode="wb") as new:
-                # we are not using shutil.copyfileobj() because we want to
-                # write to two file objects simultaneously
                 while True:
                     buf = old.read(64 * 1024)  # same as shutil uses
                     if not buf:
@@ -83,9 +81,6 @@ class ProxyRequestHandler(http.server.BaseHTTPRequestHandler):
                 self.send_header(k, v)
             self.end_headers()
             with newpath.open(mode="wb") as f:
-                # we are not using shutil.copyfileobj() because we want to
-                # write to two file objects simultaneously and throttle the
-                # writing speed to 1024 kB/s
                 while True:
                     buf = res.read(64 * 1024)  # same as shutil uses
                     if not buf:

coverage.txt

@@ -3,18 +3,21 @@ Dists: any
 Variants: minbase buildd -
 Needs-Root: true
 Needs-APT-Config: true
+Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: check-against-debootstrap-dist
 Dists: any
 Variants: minbase buildd -
 Needs-Root: true
 Needs-APT-Config: true
+Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: as-debootstrap-unshare-wrapper
 Modes: unshare
 Needs-Root: true
 Variants: minbase -
 Needs-APT-Config: true
+Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: help
 
@@ -62,7 +65,10 @@ Modes: root
 Formats: tar squashfs ext2
 Variants: essential apt minbase buildd - standard
 Skip-If:
- variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
+ variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
+ variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
+ fmt == "squashfs" and dist == "oldstable" # squashfs-tools-ng is not available
+ fmt == "ext2" and dist == "oldstable" # genext2fs does not support SOURCE_DATE_EPOCH
  mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
  variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
@@ -71,12 +77,16 @@ Modes: unshare fakechroot
 Formats: tar squashfs ext2
 Variants: essential apt minbase buildd - standard
 Skip-If:
- variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
+ variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
+ variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
+ fmt == "squashfs" and dist == "oldstable" # squashfs-tools-ng is not available
+ fmt == "ext2" and dist == "oldstable" # genext2fs does not support SOURCE_DATE_EPOCH
  mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
  variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
 Test: tarfilter-idshift
 Needs-QEMU: true
+Skip-If: dist == "oldstable" # python3 tarfile module does not preserve xattrs
 
 Test: progress-bars-on-fake-tty
 
@@ -311,7 +321,8 @@ Test: compare-output-with-pre-seeded-var-cache-apt-archives
 Needs-QEMU: true
 Variants: any
 Skip-If:
- variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
+ variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
+ variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
 
 Test: create-directory-dry-run
 Modes: root
@@ -335,20 +346,20 @@ Variants: essential
 Modes: chrootless
 Needs-Root: true
 Skip-If:
- dist == "oldstable"
+ dist in ["oldstable", "stable"]
 
 Test: chrootless-fakeroot
 Variants: essential
 Modes: chrootless
 Skip-If:
- dist == "oldstable"
+ dist in ["oldstable", "stable"]
  hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
 
 Test: chrootless-foreign
 Variants: essential
 Modes: chrootless
 Skip-If:
- dist == "oldstable"
+ dist in ["oldstable", "stable"]
  hostarch not in ["amd64", "arm64"]
  not run_ma_same_tests
 Needs-QEMU: true

make_mirror.sh

@@ -56,7 +56,15 @@ deletecache() {
 			fi
 			;;
 		esac
-		case "$dist" in oldstable|stable)
+		case "$dist" in
+			oldstable)
+				if [ -e "$dir/debian-security/dists/$dist/updates" ]; then
+					rm --one-file-system --recursive "$dir/debian-security/dists/$dist/updates"
+				else
+					echo "does not exist: $dir/debian-security/dists/$dist/updates" >&2
+				fi
+				;;
+			stable)
 				if [ -e "$dir/debian-security/dists/$dist-security" ]; then
 					rm --one-file-system --recursive "$dir/debian-security/dists/$dist-security"
 				else
@@ -104,6 +112,7 @@ deletecache() {
 }
 
 cleanup_newcachedir() {
+	kill "$PROXYPID" || :
 	echo "running cleanup_newcachedir"
 	deletecache "$newcachedir"
 }
@@ -154,7 +163,7 @@ update_cache() (
 	# we only set this trap here and overwrite the previous trap, because
 	# the update_cache function is run as part of a pipe and thus in its
 	# own process which will EXIT after it finished
-	trap 'kill "$PROXYPID" || :;cleanupapt' EXIT INT TERM
+	trap "cleanupapt" EXIT INT TERM
 
 	for p in /etc/apt/apt.conf.d /etc/apt/sources.list.d /etc/apt/preferences.d /var/cache/apt/archives /var/lib/apt/lists/partial /var/lib/dpkg; do
 		mkdir -p "$rootdir/$p"
@@ -216,7 +225,7 @@ END
 		cat "$rootdir/$f" >&2
 	done
 
-	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update --error-on=any
+	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update
 
 	pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \
 		--format '$(FILENAME)' 'Created-By: Packages' "Architecture: $nativearch" \
@@ -231,7 +240,7 @@ END
 	# we need usr-is-merged to simulate debootstrap behaviour for all dists
 	# starting from Debian 12 (Bullseye)
 	case "$dist" in
-		oldstable) : ;;
+		oldstable|stable) : ;;
 		*) pkgs="$pkgs usr-is-merged usrmerge" ;;
 	esac
 
@@ -313,7 +322,6 @@ fi
 
 ./caching_proxy.py "$oldcachedir" "$newcachedir" &
 PROXYPID=$!
-trap 'kill "$PROXYPID" || :' EXIT INT TERM
 
 for i in $(seq 10); do
 	curl --proxy "http://127.0.0.1:8080/" --silent -o /dev/null "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" && break
@@ -325,7 +333,7 @@ if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
 	exit 1
 fi
 
-trap 'kill "$PROXYPID" || :;cleanup_newcachedir' EXIT INT TERM
+trap "cleanup_newcachedir" EXIT INT TERM
 
 mkdir -p "$newcachedir"
 touch "$newcachedir/mmdebstrapcache"
@@ -361,7 +369,15 @@ for dist in oldstable stable testing unstable; do
 		echo "deb [arch=$nativearch] $mirror $dist $components" | update_cache "$dist" "$nativearch"
 		# we need to include the base mirror again or otherwise
 		# packages like build-essential will be missing
-		case "$dist" in oldstable|stable)
+		case "$dist" in
+			oldstable)
+				cat << END | update_cache "$dist" "$nativearch"
+deb [arch=$nativearch] $mirror $dist $components
+deb [arch=$nativearch] $mirror $dist-updates main
+deb [arch=$nativearch] $security_mirror $dist/updates main
+END
+				;;
+			stable)
 				cat << END | update_cache "$dist" "$nativearch"
 deb [arch=$nativearch] $mirror $dist $components
 deb [arch=$nativearch] $mirror $dist-updates main
@@ -455,9 +471,12 @@ if [ "$HAVE_QEMU" = "yes" ]; then
 	#   - it doesn't matter if the base system is from a different mirror timestamp
 	# procps is needed for /sbin/sysctl
 	tmpdir="$(mktemp -d)"
-	trap 'kill "$PROXYPID" || :;cleanuptmpdir; cleanup_newcachedir' EXIT INT TERM
+	trap "cleanuptmpdir; cleanup_newcachedir" EXIT INT TERM
 
-	pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,grub-efi,disorderfs,squashfs-tools-ng,genext2fs
+	pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,grub-efi
+	if [ "$DEFAULT_DIST" != "oldstable" ]; then
+		pkgs="$pkgs,squashfs-tools-ng,genext2fs"
+	fi
 	if [ ! -e ./mmdebstrap ]; then
 		pkgs="$pkgs,mmdebstrap"
 	fi

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '1.3.6';
+our $VERSION = '1.3.5';
 
 use English;
 use Getopt::Long;
@@ -7630,27 +7630,6 @@ To change compression specific options, either use the respecitve environment
 variables like B<XZ_OPT> or send B<mmdebstrap> output to your compressor of
 choice with a pipe.
 
-=head1 WRAPPERS
-
-=head2 debvm
-
-B<debvm> helps create and run virtual machines for various Debian releases and
-architectures.  The tool B<debvm-create> can be used to create a virtual
-machine image and the tool B<debvm-run> can be used to run such a machine
-image. Their purpose primarily is testing software using qemu as a containment
-technology.  These are relatively thin wrappers around B<mmdebstrap> and
-B<qemu>.
-
-=head2 bdebstrap
-
-B<bdebstrap> is a YAML config based multi-mirror Debian chroot creation tool.
-B<bdebstrap> is an alternative to B<debootstrap> and a wrapper around
-B<mmdebstrap> to support YAML based configuration files. It inherits all
-benefits from B<mmdebstrap>. The support for configuration allows storing all
-customization in a YAML file instead of having to use a very long one-liner
-call to B<mmdebstrap>. It also layering multiple customizations on top of each
-other, e.g. to support flavors of an image.
-
 =head1 BUGS
 
 https://gitlab.mister-muffin.de/josch/mmdebstrap/issues
@@ -7675,7 +7654,7 @@ update" as an error.
 
 =head1 SEE ALSO
 
-    debootstrap(8), debvm(1), bdebstrap(1)
+    debootstrap(8)
 
 =cut
 

tests/check-against-debootstrap-dist

@@ -11,8 +11,8 @@ echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
 # https://bugs.debian.org/969631
 # we cannot use useradd because passwd is not Essential:yes
 {{ CMD }} --variant={{ VARIANT }} --mode={{ MODE }} \
-	--essential-hook='[ {{ DIST }} = oldstable ] && [ {{ VARIANT }} = - ] && echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd || :' \
+	--essential-hook='case {{ DIST }} in oldstable|stable) if [ {{ VARIANT }} = - ]; then echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd; fi;; esac' \
-	"$(if [ {{ DIST }} = oldstable ]; then echo --merged-usr; else echo --hook-dir=./hooks/merged-usr; fi)" \
+	"$(case {{ DIST }} in oldstable|stable) echo --merged-usr ;; *) echo --hook-dir=./hooks/merged-usr ;; esac)" \
 	{{ DIST }} /tmp/debian-{{ DIST }}-mm.tar {{ MIRROR }}
 
 mkdir /tmp/debian-{{ DIST }}-mm
@@ -108,6 +108,9 @@ if [ "{{ VARIANT }}" = "-" ]; then
 
 	cap=$(chroot /tmp/debian-{{ DIST }}-debootstrap /sbin/getcap /bin/ping)
 	expected="/bin/ping cap_net_raw=ep"
+	if [ "{{ DIST }}" = oldstable ]; then
+		expected="/bin/ping = cap_net_raw+ep"
+	fi
 	if [ "$cap" != "$expected" ]; then
 		echo "expected bin/ping to have capabilities $expected" >&2
 		echo "but debootstrap produced: $cap" >&2
@@ -145,12 +148,14 @@ done
 # Because of unreproducible uids (#969631) we created the _apt user ourselves
 # and because passwd is not Essential:yes we didn't use useradd. But newer
 # versions of adduser and shadow will create a different /etc/shadow
-if [ "{{ VARIANT }}" = "-" ] && [ "{{ DIST}}" = oldstable ]; then
+if [ "{{ VARIANT }}" = "-" ]; then
+case {{ DIST }} in oldstable|stable)
 for f in shadow shadow-; do
 	if grep -q '^_apt:!:' /tmp/debian-{{ DIST }}-debootstrap/etc/$f; then
 		sed -i 's/^_apt:\*:\([^:]\+\):0:99999:7:::$/_apt:!:\1::::::/' /tmp/debian-{{ DIST }}-mm/etc/$f
 	fi
-	done
+done;;
+esac
 fi
 
 for log in faillog lastlog; do
@@ -166,12 +171,16 @@ done
 # the order in which systemd and cron get installed differ and thus the order
 # of lines in /etc/group and /etc/gshadow differs
 if [ "{{ VARIANT }}" = "-" ]; then
+	case {{ DIST }} in testing|unstable)
 		for f in group group- gshadow gshadow-; do
+			cmp /tmp/debian-{{ DIST }}-mm/etc/$f /tmp/debian-{{ DIST }}-debootstrap/etc/$f 2>/dev/null && exit 1
 			for d in mm debootstrap; do
 				sort /tmp/debian-{{ DIST }}-$d/etc/$f > /tmp/debian-{{ DIST }}-$d/etc/$f.bak
 				mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
 			done
 		done
+		;;
+	esac
 fi
 
 # check if the file content differs

tests/debootstrap

@@ -5,6 +5,13 @@ export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
 tmpdir="$(mktemp -d)"
 chmod 755 "$tmpdir"
-debootstrap "$([ "{{ DIST }}" = oldstable ] && echo --no-merged-usr || echo --merged-usr)" --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
+case "{{ DIST }}" in
+	oldstable|stable)
+		debootstrap --no-merged-usr --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
+		;;
+	*)
+		debootstrap --merged-usr --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
+		;;
+esac
 tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system --xattrs -C "$tmpdir" -c . > "./cache/debian-{{ DIST }}-{{ VARIANT }}.tar"
 rm -r "$tmpdir"