release 1.3.6

CHANGELOG.md

@@ -1,3 +1,8 @@
+1.3.6 (2023-06-16)
+------------------
+
+ - bugfix release
+
 1.3.5 (2023-03-20)
 ------------------
 

caching_proxy.py

@@ -57,6 +57,8 @@ class ProxyRequestHandler(http.server.BaseHTTPRequestHandler):
             self.send_header("Content-Length", oldpath.stat().st_size)
             self.end_headers()
             with oldpath.open(mode="rb") as old, newpath.open(mode="wb") as new:
+                # we are not using shutil.copyfileobj() because we want to
+                # write to two file objects simultaneously
                 while True:
                     buf = old.read(64 * 1024)  # same as shutil uses
                     if not buf:
@@ -81,6 +83,9 @@ class ProxyRequestHandler(http.server.BaseHTTPRequestHandler):
                 self.send_header(k, v)
             self.end_headers()
             with newpath.open(mode="wb") as f:
+                # we are not using shutil.copyfileobj() because we want to
+                # write to two file objects simultaneously and throttle the
+                # writing speed to 1024 kB/s
                 while True:
                     buf = res.read(64 * 1024)  # same as shutil uses
                     if not buf:

coverage.txt

@@ -3,21 +3,18 @@ Dists: any
 Variants: minbase buildd -
 Needs-Root: true
 Needs-APT-Config: true
-Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: check-against-debootstrap-dist
 Dists: any
 Variants: minbase buildd -
 Needs-Root: true
 Needs-APT-Config: true
-Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: as-debootstrap-unshare-wrapper
 Modes: unshare
 Needs-Root: true
 Variants: minbase -
 Needs-APT-Config: true
-Skip-If: variant == "-" and dist == "oldstable" #917773
 
 Test: help
 
@@ -65,10 +62,7 @@ Modes: root
 Formats: tar squashfs ext2
 Variants: essential apt minbase buildd - standard
 Skip-If:
- variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
+ variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
- variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
- fmt == "squashfs" and dist == "oldstable" # squashfs-tools-ng is not available
- fmt == "ext2" and dist == "oldstable" # genext2fs does not support SOURCE_DATE_EPOCH
  mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
  variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
@@ -77,16 +71,12 @@ Modes: unshare fakechroot
 Formats: tar squashfs ext2
 Variants: essential apt minbase buildd - standard
 Skip-If:
- variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
+ variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
- variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
- fmt == "squashfs" and dist == "oldstable" # squashfs-tools-ng is not available
- fmt == "ext2" and dist == "oldstable" # genext2fs does not support SOURCE_DATE_EPOCH
  mode == "fakechroot" and variant in ["-", "standard"] # no extended attributes
  variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
 Test: tarfilter-idshift
 Needs-QEMU: true
-Skip-If: dist == "oldstable" # python3 tarfile module does not preserve xattrs
 
 Test: progress-bars-on-fake-tty
 
@@ -321,8 +311,7 @@ Test: compare-output-with-pre-seeded-var-cache-apt-archives
 Needs-QEMU: true
 Variants: any
 Skip-If:
- variant == "standard" and dist in ["oldstable", "stable"] # #864082, #1004557, #1004558
+ variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
- variant == "important" and dist == "oldstable" # /var/lib/systemd/catalog/database differs
 
 Test: create-directory-dry-run
 Modes: root
@@ -346,20 +335,20 @@ Variants: essential
 Modes: chrootless
 Needs-Root: true
 Skip-If:
- dist in ["oldstable", "stable"]
+ dist == "oldstable"
 
 Test: chrootless-fakeroot
 Variants: essential
 Modes: chrootless
 Skip-If:
- dist in ["oldstable", "stable"]
+ dist == "oldstable"
  hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
 
 Test: chrootless-foreign
 Variants: essential
 Modes: chrootless
 Skip-If:
- dist in ["oldstable", "stable"]
+ dist == "oldstable"
  hostarch not in ["amd64", "arm64"]
  not run_ma_same_tests
 Needs-QEMU: true

make_mirror.sh

@@ -56,21 +56,13 @@ deletecache() {
 			fi
 			;;
 		esac
-		case "$dist" in
+		case "$dist" in oldstable|stable)
-			oldstable)
+			if [ -e "$dir/debian-security/dists/$dist-security" ]; then
-				if [ -e "$dir/debian-security/dists/$dist/updates" ]; then
+				rm --one-file-system --recursive "$dir/debian-security/dists/$dist-security"
-					rm --one-file-system --recursive "$dir/debian-security/dists/$dist/updates"
+			else
-				else
+				echo "does not exist: $dir/debian-security/dists/$dist-security" >&2
-					echo "does not exist: $dir/debian-security/dists/$dist/updates" >&2
+			fi
-				fi
+			;;
-				;;
-			stable)
-				if [ -e "$dir/debian-security/dists/$dist-security" ]; then
-					rm --one-file-system --recursive "$dir/debian-security/dists/$dist-security"
-				else
-					echo "does not exist: $dir/debian-security/dists/$dist-security" >&2
-				fi
-				;;
 		esac
 	done
 	for f in "$dir/debian-"*.qcow; do
@@ -112,7 +104,6 @@ deletecache() {
 }
 
 cleanup_newcachedir() {
-	kill "$PROXYPID" || :
 	echo "running cleanup_newcachedir"
 	deletecache "$newcachedir"
 }
@@ -163,7 +154,7 @@ update_cache() (
 	# we only set this trap here and overwrite the previous trap, because
 	# the update_cache function is run as part of a pipe and thus in its
 	# own process which will EXIT after it finished
-	trap "cleanupapt" EXIT INT TERM
+	trap 'kill "$PROXYPID" || :;cleanupapt' EXIT INT TERM
 
 	for p in /etc/apt/apt.conf.d /etc/apt/sources.list.d /etc/apt/preferences.d /var/cache/apt/archives /var/lib/apt/lists/partial /var/lib/dpkg; do
 		mkdir -p "$rootdir/$p"
@@ -225,7 +216,7 @@ END
 		cat "$rootdir/$f" >&2
 	done
 
-	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update
+	APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update --error-on=any
 
 	pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \
 		--format '$(FILENAME)' 'Created-By: Packages' "Architecture: $nativearch" \
@@ -240,7 +231,7 @@ END
 	# we need usr-is-merged to simulate debootstrap behaviour for all dists
 	# starting from Debian 12 (Bullseye)
 	case "$dist" in
-		oldstable|stable) : ;;
+		oldstable) : ;;
 		*) pkgs="$pkgs usr-is-merged usrmerge" ;;
 	esac
 
@@ -322,6 +313,7 @@ fi
 
 ./caching_proxy.py "$oldcachedir" "$newcachedir" &
 PROXYPID=$!
+trap 'kill "$PROXYPID" || :' EXIT INT TERM
 
 for i in $(seq 10); do
 	curl --proxy "http://127.0.0.1:8080/" --silent -o /dev/null "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" && break
@@ -333,7 +325,7 @@ if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
 	exit 1
 fi
 
-trap "cleanup_newcachedir" EXIT INT TERM
+trap 'kill "$PROXYPID" || :;cleanup_newcachedir' EXIT INT TERM
 
 mkdir -p "$newcachedir"
 touch "$newcachedir/mmdebstrapcache"
@@ -369,16 +361,8 @@ for dist in oldstable stable testing unstable; do
 		echo "deb [arch=$nativearch] $mirror $dist $components" | update_cache "$dist" "$nativearch"
 		# we need to include the base mirror again or otherwise
 		# packages like build-essential will be missing
-		case "$dist" in
+		case "$dist" in oldstable|stable)
-			oldstable)
+			cat << END | update_cache "$dist" "$nativearch"
-				cat << END | update_cache "$dist" "$nativearch"
-deb [arch=$nativearch] $mirror $dist $components
-deb [arch=$nativearch] $mirror $dist-updates main
-deb [arch=$nativearch] $security_mirror $dist/updates main
-END
-				;;
-			stable)
-				cat << END | update_cache "$dist" "$nativearch"
 deb [arch=$nativearch] $mirror $dist $components
 deb [arch=$nativearch] $mirror $dist-updates main
 deb [arch=$nativearch] $security_mirror $dist-security main
@@ -471,12 +455,9 @@ if [ "$HAVE_QEMU" = "yes" ]; then
 	#   - it doesn't matter if the base system is from a different mirror timestamp
 	# procps is needed for /sbin/sysctl
 	tmpdir="$(mktemp -d)"
-	trap "cleanuptmpdir; cleanup_newcachedir" EXIT INT TERM
+	trap 'kill "$PROXYPID" || :;cleanuptmpdir; cleanup_newcachedir' EXIT INT TERM
 
-	pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,grub-efi
+	pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,grub-efi,disorderfs,squashfs-tools-ng,genext2fs
-	if [ "$DEFAULT_DIST" != "oldstable" ]; then
-		pkgs="$pkgs,squashfs-tools-ng,genext2fs"
-	fi
 	if [ ! -e ./mmdebstrap ]; then
 		pkgs="$pkgs,mmdebstrap"
 	fi

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '1.3.5';
+our $VERSION = '1.3.6';
 
 use English;
 use Getopt::Long;
@@ -7630,6 +7630,27 @@ To change compression specific options, either use the respecitve environment
 variables like B<XZ_OPT> or send B<mmdebstrap> output to your compressor of
 choice with a pipe.
 
+=head1 WRAPPERS
+
+=head2 debvm
+
+B<debvm> helps create and run virtual machines for various Debian releases and
+architectures.  The tool B<debvm-create> can be used to create a virtual
+machine image and the tool B<debvm-run> can be used to run such a machine
+image. Their purpose primarily is testing software using qemu as a containment
+technology.  These are relatively thin wrappers around B<mmdebstrap> and
+B<qemu>.
+
+=head2 bdebstrap
+
+B<bdebstrap> is a YAML config based multi-mirror Debian chroot creation tool.
+B<bdebstrap> is an alternative to B<debootstrap> and a wrapper around
+B<mmdebstrap> to support YAML based configuration files. It inherits all
+benefits from B<mmdebstrap>. The support for configuration allows storing all
+customization in a YAML file instead of having to use a very long one-liner
+call to B<mmdebstrap>. It also layering multiple customizations on top of each
+other, e.g. to support flavors of an image.
+
 =head1 BUGS
 
 https://gitlab.mister-muffin.de/josch/mmdebstrap/issues
@@ -7654,7 +7675,7 @@ update" as an error.
 
 =head1 SEE ALSO
 
-    debootstrap(8)
+    debootstrap(8), debvm(1), bdebstrap(1)
 
 =cut
 

tests/check-against-debootstrap-dist

@@ -11,8 +11,8 @@ echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
 # https://bugs.debian.org/969631
 # we cannot use useradd because passwd is not Essential:yes
 {{ CMD }} --variant={{ VARIANT }} --mode={{ MODE }} \
-	--essential-hook='case {{ DIST }} in oldstable|stable) if [ {{ VARIANT }} = - ]; then echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd; fi;; esac' \
+	--essential-hook='[ {{ DIST }} = oldstable ] && [ {{ VARIANT }} = - ] && echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "$1"/etc/passwd || :' \
-	"$(case {{ DIST }} in oldstable|stable) echo --merged-usr ;; *) echo --hook-dir=./hooks/merged-usr ;; esac)" \
+	"$(if [ {{ DIST }} = oldstable ]; then echo --merged-usr; else echo --hook-dir=./hooks/merged-usr; fi)" \
 	{{ DIST }} /tmp/debian-{{ DIST }}-mm.tar {{ MIRROR }}
 
 mkdir /tmp/debian-{{ DIST }}-mm
@@ -108,9 +108,6 @@ if [ "{{ VARIANT }}" = "-" ]; then
 
 	cap=$(chroot /tmp/debian-{{ DIST }}-debootstrap /sbin/getcap /bin/ping)
 	expected="/bin/ping cap_net_raw=ep"
-	if [ "{{ DIST }}" = oldstable ]; then
-		expected="/bin/ping = cap_net_raw+ep"
-	fi
 	if [ "$cap" != "$expected" ]; then
 		echo "expected bin/ping to have capabilities $expected" >&2
 		echo "but debootstrap produced: $cap" >&2
@@ -148,14 +145,12 @@ done
 # Because of unreproducible uids (#969631) we created the _apt user ourselves
 # and because passwd is not Essential:yes we didn't use useradd. But newer
 # versions of adduser and shadow will create a different /etc/shadow
-if [ "{{ VARIANT }}" = "-" ]; then
+if [ "{{ VARIANT }}" = "-" ] && [ "{{ DIST}}" = oldstable ]; then
-case {{ DIST }} in oldstable|stable)
+	for f in shadow shadow-; do
-for f in shadow shadow-; do
+		if grep -q '^_apt:!:' /tmp/debian-{{ DIST }}-debootstrap/etc/$f; then
-	if grep -q '^_apt:!:' /tmp/debian-{{ DIST }}-debootstrap/etc/$f; then
+			sed -i 's/^_apt:\*:\([^:]\+\):0:99999:7:::$/_apt:!:\1::::::/' /tmp/debian-{{ DIST }}-mm/etc/$f
-		sed -i 's/^_apt:\*:\([^:]\+\):0:99999:7:::$/_apt:!:\1::::::/' /tmp/debian-{{ DIST }}-mm/etc/$f
+		fi
-	fi
+	done
-done;;
-esac
 fi
 
 for log in faillog lastlog; do
@@ -171,16 +166,12 @@ done
 # the order in which systemd and cron get installed differ and thus the order
 # of lines in /etc/group and /etc/gshadow differs
 if [ "{{ VARIANT }}" = "-" ]; then
-	case {{ DIST }} in testing|unstable)
+	for f in group group- gshadow gshadow-; do
-		for f in group group- gshadow gshadow-; do
+		for d in mm debootstrap; do
-			cmp /tmp/debian-{{ DIST }}-mm/etc/$f /tmp/debian-{{ DIST }}-debootstrap/etc/$f 2>/dev/null && exit 1
+			sort /tmp/debian-{{ DIST }}-$d/etc/$f > /tmp/debian-{{ DIST }}-$d/etc/$f.bak
-			for d in mm debootstrap; do
+			mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
-				sort /tmp/debian-{{ DIST }}-$d/etc/$f > /tmp/debian-{{ DIST }}-$d/etc/$f.bak
-				mv /tmp/debian-{{ DIST }}-$d/etc/$f.bak /tmp/debian-{{ DIST }}-$d/etc/$f
-			done
 		done
-		;;
+	done
-	esac
 fi
 
 # check if the file content differs

tests/debootstrap

@@ -5,13 +5,6 @@ export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
 tmpdir="$(mktemp -d)"
 chmod 755 "$tmpdir"
-case "{{ DIST }}" in
+debootstrap "$([ "{{ DIST }}" = oldstable ] && echo --no-merged-usr || echo --merged-usr)" --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
-	oldstable|stable)
-		debootstrap --no-merged-usr --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
-		;;
-	*)
-		debootstrap --merged-usr --variant={{ VARIANT }} {{ DIST }} "$tmpdir" {{ MIRROR }}
-		;;
-esac
 tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system --xattrs -C "$tmpdir" -c . > "./cache/debian-{{ DIST }}-{{ VARIANT }}.tar"
 rm -r "$tmpdir"