release 0.8.0

- use /etc/ld.so.conf from the chroot instead of the host
 - parse /etc/ld.so.conf instead of blindly accessing /etc/ld.so.conf.d
 - add libraries from the chroot instead of the host

CHANGELOG.md

@@ -1,3 +1,34 @@
+0.8.0 (2021-09-21)
+------------------
+
+ - allow running inside chroot in root mode
+ - allow running without /dev, /sys or /proc
+ - new --format=null which gets automatically selected if the output is
+   /dev/null and doesn't produce a tarball or other permanent output
+ - allow ASCII-armored keyrings (requires gnupg >= 2.2.8)
+ - run zstd with --threads=0
+ - tarfilter: add --pax-exclude and --pax-include to strip extended attributes
+ - add --skip=setup, --skip=update and --skip=cleanup
+ - add --skip=cleanup/apt/lists and --skip=cleanup/apt/cache
+ - pass extended attributes (excluding system) to tar2sqfs
+ - use apt-get update -error-on=any (requires apt >= 2.1.16)
+ - support Debian 11 Buster
+ - use apt from outside using DPkg::Chroot-Directory (requires apt >= 2.3.7)
+    * build chroots without apt (for example from buildinfo files)
+    * no need to install additional packages like apt-transport-* or
+      ca-certificates inside the chroot
+    * no need for additional key material inside the chroot
+    * possible use of file:// and copy://
+ - use apt pattern to select essential set
+ - write 'uninitialized' to /etc/machine-id
+ - allow running in root mode without mount working, either because of missing
+   CAP_SYS_ADMIN or missing /usr/bin/mount
+ - make /etc/ld.so.cache under fakechroot mode bit-by-bit identical to root
+   and unshare mode
+ - move hooks/setup00-merged-usr.sh to hooks/merged-usr/setup00.sh
+ - add gpgvnoexpkeysig script for very old snapshot.d.o timestamps with expired
+   signature
+
 0.7.5 (2021-02-06)
 ------------------
 

README.md

@@ -104,6 +104,9 @@ There is no `SCRIPT` argument.
 The following options, don't exist: `--second-stage`, `--exclude`,
 `--resolve-deps`, `--force-check-gpg`, `--merged-usr` and `--no-merged-usr`.
 
+The quirks from debootstrap are needed to create chroots of Debian unstable
+from snapshot.d.o before timestamp 20141107T220431Z or Debian 8 (Jessie) or
+later.
 
 Tests
 =====

coverage.sh

@@ -96,6 +96,13 @@ if [ ! -e shared/proxysolver ] || [ proxysolver -nt shared/proxysolver ]; then
 		cp -a /usr/lib/apt/solvers/mmdebstrap-dump-solution shared/proxysolver
 	fi
 fi
+if [ ! -e shared/ldconfig.fakechroot ] || [ ldconfig.fakechroot -nt shared/ldconfig.fakechroot ]; then
+	if [ -e ./ldconfig.fakechroot ]; then
+		cp -a ldconfig.fakechroot shared
+	else
+		cp -a /usr/libexec/mmdebstrap/ldconfig.fakechroot shared/ldconfig.fakechroot
+	fi
+fi
 mkdir -p shared/hooks
 if [ ! -e shared/hooks/setup00-merged-usr.sh ] || [ hooks/setup00-merged-usr.sh -nt shared/hooks/setup00-merged-usr.sh ]; then
 	if [ -e hooks/setup00-merged-usr.sh ]; then
@@ -120,7 +127,7 @@ if [ ! -e shared/hooks/eatmydata/customize.sh ] || [ hooks/eatmydata/customize.s
 	fi
 fi
 starttime=
-total=217
+total=212
 skipped=0
 runtests=0
 i=1
@@ -706,7 +713,7 @@ fi
 
 for variant in essential apt minbase buildd important standard; do
 	for format in tar squashfs ext2; do
-		print_header "mode=unshare/root,variant=$variant: check for bit-by-bit identical $format output"
+		print_header "mode=root/unshare/fakechroot,variant=$variant: check for bit-by-bit identical $format output"
 		# fontconfig doesn't install reproducibly because differences
 		# in /var/cache/fontconfig/. See
 		# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=864082
@@ -753,7 +760,17 @@ else
 fi
 runuser -u user -- $CMD --mode=unshare --variant=$variant $DEFAULT_DIST /tmp/debian-chroot-unshare.$format $mirror
 cmp /tmp/debian-chroot-root.$format /tmp/debian-chroot-unshare.$format
-rm /tmp/debian-chroot-root.$format /tmp/debian-chroot-unshare.$format
+rm /tmp/debian-chroot-unshare.$format
+case $variant in essential|apt|minbase|buildd)
+	# variants important and standard differ because permissions drwxr-sr-x
+	# and extended attributes of ./var/log/journal/ cannot be preserved
+	# in fakechroot mode
+	runuser -u user -- $CMD --mode=fakechroot --variant=$variant $DEFAULT_DIST /tmp/debian-chroot-fakechroot.$format $mirror
+	cmp /tmp/debian-chroot-root.$format /tmp/debian-chroot-fakechroot.$format
+	rm /tmp/debian-chroot-fakechroot.$format
+	;;
+esac
+rm /tmp/debian-chroot-root.$format
 END
 		if [ "$HAVE_QEMU" = "yes" ]; then
 			./run_qemu.sh
@@ -1137,96 +1154,6 @@ else
 	runtests=$((runtests+1))
 fi
 
-print_header "mode=$defaultmode,variant=apt: test squashfs image"
-cat << END > shared/test.sh
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-$CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST /tmp/debian-chroot.squashfs $mirror
-printf 'hsqs' | cmp --bytes=4 /tmp/debian-chroot.squashfs -
-# workaround for https://github.com/AgentD/squashfs-tools-ng/issues/37
-sed 's#\\([^.]\\)/\$#\\1#' tar1.txt | sort > /tmp/tar1noslash.txt
-# workaround for https://github.com/AgentD/squashfs-tools-ng/issues/42
-sqfs2tar --no-skip --root-becomes . /tmp/debian-chroot.squashfs | tar -t \
-	| sed 's#\\([^.]\\)/\$#\\1#' \
-	| sort | diff -u /tmp/tar1noslash.txt -
-rm /tmp/debian-chroot.squashfs /tmp/tar1noslash.txt
-END
-if [ "$DEFAULT_DIST" = "oldstable" ]; then
-	echo "skipping test on oldstable because squashfs-tools-ng is not available" >&2
-	skipped=$((skipped+1))
-elif [ "$HAVE_QEMU" = "yes" ]; then
-	./run_qemu.sh
-	runtests=$((runtests+1))
-elif [ "$defaultmode" = "root" ]; then
-	./run_null.sh SUDO
-	runtests=$((runtests+1))
-else
-	./run_null.sh
-	runtests=$((runtests+1))
-fi
-
-for mode in root unshare fakechroot proot; do
-	print_header "mode=$mode,variant=apt: test ext2 image"
-	if [ "$DEFAULT_DIST" = "oldstable" ]; then
-		echo "skipping test on oldstable because genext2fs does not support SOURCE_DATE_EPOCH" >&2
-		skipped=$((skipped+1))
-		continue
-	fi
-	if [ "$mode" = "unshare" ] && [ "$HAVE_UNSHARE" != "yes" ]; then
-		echo "HAVE_UNSHARE != yes -- Skipping test..." >&2
-		skipped=$((skipped+1))
-		continue
-	fi
-	if [ "$mode" = "proot" ] && [ "$HAVE_PROOT" != "yes" ]; then
-		echo "HAVE_PROOT != yes -- Skipping test..." >&2
-		skipped=$((skipped+1))
-		continue
-	fi
-	cat << END > shared/test.sh
-#!/bin/sh
-set -eu
-export LC_ALL=C.UTF-8
-if [ "\$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1; then
-	if [ ! -e /mmdebstrap-testenv ]; then
-		echo "this test modifies the system and should only be run inside a container" >&2
-		exit 1
-	fi
-	adduser --gecos user --disabled-password user
-fi
-if [ "$mode" = unshare ]; then
-	if [ ! -e /mmdebstrap-testenv ]; then
-		echo "this test modifies the system and should only be run inside a container" >&2
-		exit 1
-	fi
-	sysctl -w kernel.unprivileged_userns_clone=1
-fi
-prefix=
-[ "\$(id -u)" -eq 0 ] && [ "$mode" != "root" ] && prefix="runuser -u user --"
-[ "$mode" = "fakechroot" ] && prefix="\$prefix fakechroot fakeroot"
-\$prefix $CMD --mode=$mode --variant=apt $DEFAULT_DIST /tmp/debian-chroot.ext2 $mirror
-mkdir /tmp/mnt
-mount /tmp/debian-chroot.ext2 /tmp/mnt
-rmdir /tmp/mnt/lost+found
-# in fakechroot mode, we use a fake ldconfig, so we have to
-# artificially add some files
-{ tar -C /tmp/mnt -c . | tar -t;
-  [ "$mode" = "fakechroot" ] && printf "./etc/ld.so.cache\n./var/cache/ldconfig/\n";
-  [ "$mode" = "fakechroot" ] && printf "./etc/.pwd.lock\n";
-} | sort | diff -u tar1.txt -
-umount /tmp/mnt
-rmdir /tmp/mnt
-rm /tmp/debian-chroot.ext2
-END
-	if [ "$HAVE_QEMU" = "yes" ]; then
-		./run_qemu.sh
-		runtests=$((runtests+1))
-	else
-		echo "HAVE_QEMU != yes -- Skipping test..." >&2
-		skipped=$((skipped+1))
-	fi
-done
-
 print_header "mode=auto,variant=apt: test auto-mode without unshare capabilities"
 cat << END > shared/test.sh
 #!/bin/sh
@@ -1239,9 +1166,7 @@ fi
 adduser --gecos user --disabled-password user
 sysctl -w kernel.unprivileged_userns_clone=0
 runuser -u user -- $CMD --mode=auto --variant=apt $DEFAULT_DIST /tmp/debian-chroot.tar.gz $mirror
-{ tar -tf /tmp/debian-chroot.tar.gz;
+tar -tf /tmp/debian-chroot.tar.gz | sort | diff -u tar1.txt -
-  printf "./etc/ld.so.cache\n./var/cache/ldconfig/\n./etc/.pwd.lock\n";
-} | sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar.gz
 END
 if [ "$HAVE_QEMU" = "yes" ]; then
@@ -2651,12 +2576,7 @@ echo upload-customize | cmp /tmp/download-customize -
 echo sync-in-setup | cmp /tmp/sync-out-setup/file -
 echo sync-in-essential | cmp /tmp/sync-out-essential/file -
 echo sync-in-customize | cmp /tmp/sync-out-customize/file -
-# in fakechroot mode, we use a fake ldconfig, so we have to
+tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
-# artificially add some files
-{ tar -tf /tmp/debian-chroot.tar;
-  [ "$mode" = "fakechroot" ] && printf "./etc/ld.so.cache\n./var/cache/ldconfig/\n";
-  [ "$mode" = "fakechroot" ] && [ "$variant" != "essential" ] && printf "./etc/.pwd.lock\n";
-} | sort | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar \
 	/tmp/copy-in-setup /tmp/copy-in-essential /tmp/copy-in-customize \
 	/tmp/copy-out-setup /tmp/copy-out-essential /tmp/copy-out-customize \
@@ -3192,12 +3112,7 @@ fi
 prefix=
 [ "\$(id -u)" -eq 0 ] && prefix="runuser -u user --"
 \$prefix $CMD --mode=$mode --variant=$variant $DEFAULT_DIST /tmp/debian-chroot.tar $mirror
-# in fakechroot mode, we use a fake ldconfig, so we have to
+tar -tf /tmp/debian-chroot.tar | sort | diff -u "./$variant.txt" -
-# artificially add some files
-{ tar -tf /tmp/debian-chroot.tar;
-  [ "$mode" = "fakechroot" ] && printf "./etc/ld.so.cache\n./var/cache/ldconfig/\n";
-  [ "$mode" = "fakechroot" ] && [ "$variant" != "essential" ] && printf "./etc/.pwd.lock\n";
-} | sort | diff -u "./$variant.txt" -
 rm /tmp/debian-chroot.tar
 END
 		if [ "$HAVE_QEMU" = "yes" ]; then
@@ -3446,6 +3361,10 @@ END
 if [ "$DEFAULT_DIST" = "oldstable" ]; then
 	echo "chrootless doesn't work in oldstable -- Skipping test..." >&2
 	skipped=$((skipped+1))
+elif true; then
+	# https://salsa.debian.org/pkg-debconf/debconf/-/merge_requests/8
+	echo "blocked by #983425 -- Skipping test..." >&2
+	skipped=$((skipped+1))
 elif [ "$HAVE_QEMU" = "yes" ]; then
 	./run_qemu.sh
 	runtests=$((runtests+1))
@@ -3714,8 +3633,6 @@ prefix=
 \$prefix $CMD --mode=$mode --variant=apt --architectures=arm64 $DEFAULT_DIST /tmp/debian-chroot.tar $mirror
 # we ignore differences between architectures by ignoring some files
 # and renaming others
-# in fakechroot mode, we use a fake ldconfig, so we have to
-# artificially add some files
 # in proot mode, some extra files are put there by proot
 { tar -tf /tmp/debian-chroot.tar \
 	| grep -v '^\./lib/ld-linux-aarch64\.so\.1$' \
@@ -3723,7 +3640,6 @@ prefix=
 	| grep -v '^\./usr/share/doc/[^/]\+/changelog\(\.Debian\)\?\.arm64\.gz$' \
 	| sed 's/aarch64-linux-gnu/x86_64-linux-gnu/' \
 	| sed 's/arm64/amd64/';
-	[ "$mode" = "fakechroot" ] && printf "./etc/ld.so.cache\n./var/cache/ldconfig/\n./etc/.pwd.lock\n";
 } | sort > tar2.txt
 { cat tar1.txt \
 	| grep -v '^\./usr/bin/i386$' \

examples/twb/debian-11-minimal.py

@@ -21,68 +21,80 @@ NOTE: this is the simplest config possible.
 """
 
 parser = argparse.ArgumentParser(description=__doc__)
-parser.add_argument('output_file', nargs='?', default=pathlib.Path('filesystem.img'), type=pathlib.Path)
+parser.add_argument(
+    "output_file", nargs="?", default=pathlib.Path("filesystem.img"), type=pathlib.Path
+)
 args = parser.parse_args()
 
 
-filesystem_img_size = '256M'    # big enough to include filesystem.squashfs + about 64M of bootloader, kernel, and ramdisk.
+filesystem_img_size = "256M"  # big enough to include filesystem.squashfs + about 64M of bootloader, kernel, and ramdisk.
-esp_offset = 1024 * 1024        # 1MiB
+esp_offset = 1024 * 1024  # 1MiB
-esp_label = 'UEFI-ESP'          # max 8 bytes for FAT32
+esp_label = "UEFI-ESP"  # max 8 bytes for FAT32
-live_media_path = 'debian-live'
+live_media_path = "debian-live"
 
-with tempfile.TemporaryDirectory(prefix='debian-live-bullseye-amd64-minimal.') as td:
+with tempfile.TemporaryDirectory(prefix="debian-live-bullseye-amd64-minimal.") as td:
     td = pathlib.Path(td)
     subprocess.check_call(
-        ['mmdebstrap',
+        [
-         '--mode=unshare',
+            "mmdebstrap",
-         '--variant=apt',
+            "--mode=unshare",
-         '--aptopt=Acquire::http::Proxy "http://apt-cacher-ng.cyber.com.au:3142"',
+            "--variant=apt",
-         '--aptopt=Acquire::https::Proxy "DIRECT"',
+            '--aptopt=Acquire::http::Proxy "http://apt-cacher-ng.cyber.com.au:3142"',
-         '--dpkgopt=force-unsafe-io',
+            '--aptopt=Acquire::https::Proxy "DIRECT"',
-         '--include=linux-image-amd64 init initramfs-tools live-boot netbase',
+            "--dpkgopt=force-unsafe-io",
-         '--include=dbus',          # https://bugs.debian.org/814758
+            "--include=linux-image-amd64 init initramfs-tools live-boot netbase",
-         '--include=live-config iproute2 keyboard-configuration locales sudo user-setup',
+            "--include=dbus",  # https://bugs.debian.org/814758
-         '--include=ifupdown isc-dhcp-client',  # live-config doesn't support systemd-networkd yet.
+            "--include=live-config iproute2 keyboard-configuration locales sudo user-setup",
+            "--include=ifupdown isc-dhcp-client",  # live-config doesn't support systemd-networkd yet.
+            # Do the **BARE MINIMUM** to make a USB key that can boot on X86_64 UEFI.
+            # We use mtools so we do not ever need root privileges.
+            # We can't use mkfs.vfat, as that needs kpartx or losetup (i.e. root).
+            # We can't use mkfs.udf, as that needs mount (i.e. root).
+            # We can't use "refind-install --usedefault" as that runs mount(8) (i.e. root).
+            # We don't use genisoimage because
+            # 1) ISO9660 must die;
+            # 2) incomplete UDF 1.5+ support;
+            # 3) resulting filesystem can't be tweaked after flashing (e.g. debian-live/site.dir/etc/systemd/network/up.network).
+            #
+            # We use refind because 1) I hate grub; and 2) I like refind.
+            # If you want aarch64 or ia32 you need to install their BOOTxxx.EFI files.
+            # If you want kernel+initrd on something other than FAT, you need refind/drivers_xxx/xxx_xxx.EFI.
+            #
+            # FIXME: with qemu in UEFI mode (OVMF), I get dumped into startup.nsh (UEFI REPL).
+            #        From there, I can manually type in "FS0:\EFI\BOOT\BOOTX64.EFI" to start refind, tho.
+            #        So WTF is its problem?  Does it not support fallback bootloader?
+            "--include=refind parted mtools",
+            "--essential-hook=echo refind refind/install_to_esp boolean false | chroot $1 debconf-set-selections",
+            "--customize-hook=echo refind refind/install_to_esp boolean true  | chroot $1 debconf-set-selections",
+            "--customize-hook=chroot $1 mkdir -p /boot/USB /boot/EFI/BOOT",
+            "--customize-hook=chroot $1 cp /usr/share/refind/refind/refind_x64.efi /boot/EFI/BOOT/BOOTX64.EFI",
+            f"--customize-hook=chroot $1 truncate --size={filesystem_img_size} /boot/USB/filesystem.img",
+            f"--customize-hook=chroot $1 parted --script --align=optimal /boot/USB/filesystem.img  mklabel gpt  mkpart {esp_label} {esp_offset}b 100%  set 1 esp on",
+            f"--customize-hook=chroot $1 mformat -i /boot/USB/filesystem.img@@{esp_offset} -F -v {esp_label}",
+            f"--customize-hook=chroot $1 mmd     -i /boot/USB/filesystem.img@@{esp_offset} ::{live_media_path}",
+            f"""--customize-hook=echo '"Boot with default options" "boot=live live-media-path={live_media_path}"' >$1/boot/refind_linux.conf""",
+            # NOTE: find sidesteps the "glob expands before chroot applies" problem.
+            f"""--customize-hook=chroot $1 find -O3 /boot/ -xdev -mindepth 1 -maxdepth 1 -regextype posix-egrep -iregex '.*/(EFI|refind_linux.conf|vmlinuz.*|initrd.img.*)' -exec mcopy -vsbpm -i /boot/USB/filesystem.img@@{esp_offset} {{}} :: ';'""",
+            # FIXME: copy-out doesn't handle sparseness, so is REALLY slow (about 50 seconds).
+            # Therefore instead leave it in the squashfs, and extract it later.
+            #  f'--customize-hook=copy-out /boot/USB/filesystem.img /tmp/',
+            #  f'--customize-hook=chroot $1 rm /boot/USB/filesystem.img',
+            "bullseye",
+            td / "filesystem.squashfs",
+        ]
+    )
 
-         # Do the **BARE MINIMUM** to make a USB key that can boot on X86_64 UEFI.
+    with args.output_file.open("wb") as f:
-         # We use mtools so we do not ever need root privileges.
+        subprocess.check_call(
-         # We can't use mkfs.vfat, as that needs kpartx or losetup (i.e. root).
+            ["rdsquashfs", "--cat=boot/USB/filesystem.img", td / "filesystem.squashfs"],
-         # We can't use mkfs.udf, as that needs mount (i.e. root).
+            stdout=f,
-         # We can't use "refind-install --usedefault" as that runs mount(8) (i.e. root).
+        )
-         # We don't use genisoimage because
+    subprocess.check_call(
-         # 1) ISO9660 must die;
+        [
-         # 2) incomplete UDF 1.5+ support;
+            "mcopy",
-         # 3) resulting filesystem can't be tweaked after flashing (e.g. debian-live/site.dir/etc/systemd/network/up.network).
+            "-i",
-         #
+            f"{args.output_file}@@{esp_offset}",
-         # We use refind because 1) I hate grub; and 2) I like refind.
+            td / "filesystem.squashfs",
-         # If you want aarch64 or ia32 you need to install their BOOTxxx.EFI files.
+            f"::{live_media_path}/filesystem.squashfs",
-         # If you want kernel+initrd on something other than FAT, you need refind/drivers_xxx/xxx_xxx.EFI.
+        ]
-         #
+    )
-         # FIXME: with qemu in UEFI mode (OVMF), I get dumped into startup.nsh (UEFI REPL).
-         #        From there, I can manually type in "FS0:\EFI\BOOT\BOOTX64.EFI" to start refind, tho.
-         #        So WTF is its problem?  Does it not support fallback bootloader?
-         '--include=refind parted mtools',
-         '--essential-hook=echo refind refind/install_to_esp boolean false | chroot $1 debconf-set-selections',
-         '--customize-hook=echo refind refind/install_to_esp boolean true  | chroot $1 debconf-set-selections',
-         '--customize-hook=chroot $1 mkdir -p /boot/USB /boot/EFI/BOOT',
-         '--customize-hook=chroot $1 cp /usr/share/refind/refind/refind_x64.efi /boot/EFI/BOOT/BOOTX64.EFI',
-         f'--customize-hook=chroot $1 truncate --size={filesystem_img_size} /boot/USB/filesystem.img',
-         f'--customize-hook=chroot $1 parted --script --align=optimal /boot/USB/filesystem.img  mklabel gpt  mkpart {esp_label} {esp_offset}b 100%  set 1 esp on',
-         f'--customize-hook=chroot $1 mformat -i /boot/USB/filesystem.img@@{esp_offset} -F -v {esp_label}',
-         f'--customize-hook=chroot $1 mmd     -i /boot/USB/filesystem.img@@{esp_offset} ::{live_media_path}',
-         f"""--customize-hook=echo '"Boot with default options" "boot=live live-media-path={live_media_path}"' >$1/boot/refind_linux.conf""",
-         # NOTE: find sidesteps the "glob expands before chroot applies" problem.
-         f"""--customize-hook=chroot $1 find -O3 /boot/ -xdev -mindepth 1 -maxdepth 1 -regextype posix-egrep -iregex '.*/(EFI|refind_linux.conf|vmlinuz.*|initrd.img.*)' -exec mcopy -vsbpm -i /boot/USB/filesystem.img@@{esp_offset} {{}} :: ';'""",
-         # FIXME: copy-out doesn't handle sparseness, so is REALLY slow (about 50 seconds).
-         # Therefore instead leave it in the squashfs, and extract it later.
-         #  f'--customize-hook=copy-out /boot/USB/filesystem.img /tmp/',
-         #  f'--customize-hook=chroot $1 rm /boot/USB/filesystem.img',
-
-         'bullseye',
-         td / 'filesystem.squashfs'
-        ])
-
-    with args.output_file.open('wb') as f:
-        subprocess.check_call(['rdsquashfs', '--cat=boot/USB/filesystem.img', td / 'filesystem.squashfs'], stdout=f)
-    subprocess.check_call([
-        'mcopy', '-i', f'{args.output_file}@@{esp_offset}', td / 'filesystem.squashfs', f'::{live_media_path}/filesystem.squashfs'])

examples/twb/debian-sid-zfs.py

@@ -19,197 +19,194 @@ which in turn includes a bootloader (refind), kernel, ramdisk, and filesystem.sq
 """
 
 parser = argparse.ArgumentParser(description=__doc__)
-parser.add_argument('output_file', nargs='?', default=pathlib.Path('filesystem.img'), type=pathlib.Path)
+parser.add_argument(
-parser.add_argument('--timezone', default='Australia/Melbourne', type=lambda s: s.split('/'), help='NOTE: MUST be "Area/Zone" not e.g. "UTC", for now')
+    "output_file", nargs="?", default=pathlib.Path("filesystem.img"), type=pathlib.Path
-parser.add_argument('--locale', default='en_AU.UTF-8', help='NOTE: MUST end in ".UTF-8", for now')
+)
+parser.add_argument(
+    "--timezone",
+    default="Australia/Melbourne",
+    type=lambda s: s.split("/"),
+    help='NOTE: MUST be "Area/Zone" not e.g. "UTC", for now',
+)
+parser.add_argument(
+    "--locale", default="en_AU.UTF-8", help='NOTE: MUST end in ".UTF-8", for now'
+)
 args = parser.parse_args()
 
 
-filesystem_img_size = '512M'    # big enough to include filesystem.squashfs + about 64M of bootloader, kernel, and ramdisk.
+filesystem_img_size = "512M"  # big enough to include filesystem.squashfs + about 64M of bootloader, kernel, and ramdisk.
-esp_offset = 1024 * 1024        # 1MiB
+esp_offset = 1024 * 1024  # 1MiB
-esp_label = 'UEFI-ESP'          # max 8 bytes for FAT32
+esp_label = "UEFI-ESP"  # max 8 bytes for FAT32
-live_media_path = 'debian-live'
+live_media_path = "debian-live"
 
-with tempfile.TemporaryDirectory(prefix='debian-sid-zfs.') as td:
+with tempfile.TemporaryDirectory(prefix="debian-sid-zfs.") as td:
     td = pathlib.Path(td)
     subprocess.check_call(
-        ['mmdebstrap',
+        [
-         '--mode=unshare',
+            "mmdebstrap",
-         '--variant=apt',
+            "--mode=unshare",
-         '--aptopt=Acquire::http::Proxy "http://apt-cacher-ng.cyber.com.au:3142"',
+            "--variant=apt",
-         '--aptopt=Acquire::https::Proxy "DIRECT"',
+            '--aptopt=Acquire::http::Proxy "http://apt-cacher-ng.cyber.com.au:3142"',
-         '--dpkgopt=force-unsafe-io',
+            '--aptopt=Acquire::https::Proxy "DIRECT"',
-         '--components=main contrib non-free',  # needed for CPU security patches
+            "--dpkgopt=force-unsafe-io",
-
+            "--components=main contrib non-free",  # needed for CPU security patches
-         '--include=init initramfs-tools xz-utils live-boot netbase',
+            "--include=init initramfs-tools xz-utils live-boot netbase",
-         '--include=dbus',          # https://bugs.debian.org/814758
+            "--include=dbus",  # https://bugs.debian.org/814758
-         '--include=linux-image-amd64 firmware-linux',
+            "--include=linux-image-amd64 firmware-linux",
-
+            # Have ZFS 2.0 support.
-         # Have ZFS 2.0 support.
+            "--include=zfs-dkms zfsutils-linux zfs-zed build-essential linux-headers-amd64",  # ZFS 2 support
-         '--include=zfs-dkms zfsutils-linux zfs-zed build-essential linux-headers-amd64',  # ZFS 2 support
+            # Make the initrd a little smaller (41MB -> 20MB), at the expensive of significantly slower image build time.
-
+            "--include=zstd",
-         # Make the initrd a little smaller (41MB -> 20MB), at the expensive of significantly slower image build time.
+            "--essential-hook=mkdir -p $1/etc/initramfs-tools/conf.d",
-         '--include=zstd',
+            "--essential-hook=>$1/etc/initramfs-tools/conf.d/zstd echo COMPRESS=zstd",
-         '--essential-hook=mkdir -p $1/etc/initramfs-tools/conf.d',
+            # Be the equivalent of Debian Live GNOME
-         '--essential-hook=>$1/etc/initramfs-tools/conf.d/zstd echo COMPRESS=zstd',
+            # '--include=live-task-gnome',
-
+            #'--include=live-task-xfce',
-         # Be the equivalent of Debian Live GNOME
+            # FIXME: enable this?  It makes live-task-xfce go from 1G to 16G... so no.
-         # '--include=live-task-gnome',
+            #'--aptopt=Apt::Install-Recommends "true"',
-         #'--include=live-task-xfce',
+            # ...cherry-pick instead
-         # FIXME: enable this?  It makes live-task-xfce go from 1G to 16G... so no.
+            # UPDATE: debian-installer-launcher DOES NOT WORK because we don't load crap SPECIFICALLY into /live/installer, in the ESP.
-         #'--aptopt=Apt::Install-Recommends "true"',
+            # UPDATE: network-manager-gnome DOES NOT WORK, nor is systemd-networkd auto-started... WTF?
-         # ...cherry-pick instead
+            #         end result is no networking.
-         # UPDATE: debian-installer-launcher DOES NOT WORK because we don't load crap SPECIFICALLY into /live/installer, in the ESP.
+            #'--include=live-config user-setup sudo firmware-linux haveged',
-         # UPDATE: network-manager-gnome DOES NOT WORK, nor is systemd-networkd auto-started... WTF?
+            #'--include=calamares-settings-debian udisks2',  # 300MB weirdo Qt GUI debian installer
-         #         end result is no networking.
+            #'--include=xfce4-terminal',
-         #'--include=live-config user-setup sudo firmware-linux haveged',
+            # x86_64 CPUs are undocumented proprietary RISC chips that EMULATE a documented x86_64 CISC ISA.
-         #'--include=calamares-settings-debian udisks2',  # 300MB weirdo Qt GUI debian installer
+            # The emulator is called "microcode", and is full of security vulnerabilities.
-         #'--include=xfce4-terminal',
+            # Make sure security patches for microcode for *ALL* CPUs are included.
-
+            # By default, it tries to auto-detect the running CPU, so only patches the CPU of the build server.
-         # x86_64 CPUs are undocumented proprietary RISC chips that EMULATE a documented x86_64 CISC ISA.
+            "--include=intel-microcode amd64-microcode iucode-tool",
-         # The emulator is called "microcode", and is full of security vulnerabilities.
+            "--essential-hook=>$1/etc/default/intel-microcode echo IUCODE_TOOL_INITRAMFS=yes IUCODE_TOOL_SCANCPUS=no",
-         # Make sure security patches for microcode for *ALL* CPUs are included.
+            "--essential-hook=>$1/etc/default/amd64-microcode echo AMD64UCODE_INITRAMFS=yes",
-         # By default, it tries to auto-detect the running CPU, so only patches the CPU of the build server.
+            "--dpkgopt=force-confold",  # Work around https://bugs.debian.org/981004
-         '--include=intel-microcode amd64-microcode iucode-tool',
+            # DHCP/DNS/SNTP clients...
-         '--essential-hook=>$1/etc/default/intel-microcode echo IUCODE_TOOL_INITRAMFS=yes IUCODE_TOOL_SCANCPUS=no',
+            # FIXME: use live-config ?
-         '--essential-hook=>$1/etc/default/amd64-microcode echo AMD64UCODE_INITRAMFS=yes',
+            "--include=libnss-resolve libnss-myhostname systemd-timesyncd",
-         '--dpkgopt=force-confold',  # Work around https://bugs.debian.org/981004
+            "--customize-hook=chroot $1 cp -alf /lib/systemd/resolv.conf /etc/resolv.conf",  # This probably needs to happen LAST
-
+            # FIXME: fix resolv.conf to point to resolved, not "copy from the build-time OS"
-         # DHCP/DNS/SNTP clients...
+            # FIXME: fix hostname & hosts to not exist, not "copy from the build-time OS"
-         # FIXME: use live-config ?
+            "--customize-hook=systemctl --root=$1 enable systemd-networkd systemd-timesyncd",  # is this needed?
-         '--include=libnss-resolve libnss-myhostname systemd-timesyncd',
+            # Run a DHCP client on *ALL* ifaces.
-         '--customize-hook=chroot $1 cp -alf /lib/systemd/resolv.conf /etc/resolv.conf',  # This probably needs to happen LAST
+            # Consider network "up" (start sshd and local login prompt) when *ANY* (not ALL) ifaces are up.
-         # FIXME: fix resolv.conf to point to resolved, not "copy from the build-time OS"
+            "--customize-hook=>$1/etc/systemd/network/up.network printf '%s\n'  '[Match]' Name='en*' '[Network]' DHCP=yes",  # try DHCP on all ethernet ifaces
-         # FIXME: fix hostname & hosts to not exist, not "copy from the build-time OS"
+            "--customize-hook=mkdir $1/etc/systemd/system/systemd-networkd-wait-online.service.d",
-         '--customize-hook=systemctl --root=$1 enable systemd-networkd systemd-timesyncd',   # is this needed?
+            "--customize-hook=>$1/etc/systemd/system/systemd-networkd-wait-online.service.d/any-not-all.conf printf '%s\n' '[Service]' 'ExecStart=' 'ExecStart=/lib/systemd/systemd-networkd-wait-online --any'",
-         # Run a DHCP client on *ALL* ifaces.
+            # Hope there's a central smarthost SMTP server called "mail" in the local search domain.
-         # Consider network "up" (start sshd and local login prompt) when *ANY* (not ALL) ifaces are up.
+            # FIXME: can live-config do this?
-         "--customize-hook=>$1/etc/systemd/network/up.network printf '%s\n'  '[Match]' Name='en*' '[Network]' DHCP=yes",  # try DHCP on all ethernet ifaces
+            "--include=msmtp-mta",
-         '--customize-hook=mkdir $1/etc/systemd/system/systemd-networkd-wait-online.service.d',
+            "--customize-hook=>$1/etc/msmtprc printf '%s\n' 'account default' 'syslog LOG_MAIL' 'host mail' 'auto_from on'",
-         "--customize-hook=>$1/etc/systemd/system/systemd-networkd-wait-online.service.d/any-not-all.conf printf '%s\n' '[Service]' 'ExecStart=' 'ExecStart=/lib/systemd/systemd-networkd-wait-online --any'",
+            # Hope there's a central RELP logserver called "logserv" in the local domain.
-
+            # FIXME: can live-config do this?
-         # Hope there's a central smarthost SMTP server called "mail" in the local search domain.
+            "--include=rsyslog-relp",
-         # FIXME: can live-config do this?
+            """--customize-hook=>$1/etc/rsyslog.conf printf '%s\n' 'module(load="imuxsock")' 'module(load="imklog")' 'module(load="omrelp")' 'action(type="omrelp" target="logserv" port="2514" template="RSYSLOG_SyslogProtocol23Format")'""",
-         '--include=msmtp-mta',
+            # Run self-tests on all discoverable hard disks, and (try to) email if something goes wrong.
-         "--customize-hook=>$1/etc/msmtprc printf '%s\n' 'account default' 'syslog LOG_MAIL' 'host mail' 'auto_from on'",
+            "--include=smartmontools bsd-mailx",
-
+            "--customize-hook=>$1/etc/smartd.conf echo 'DEVICESCAN -n standby,15 -a -o on -S on -s (S/../../7/00|L/../01/./01) -t -H -m root -M once'",
-         # Hope there's a central RELP logserver called "logserv" in the local domain.
+            # For rarely-updated, rarely-rebooted SOEs, apply what security updates we can into transient tmpfs COW.
-         # FIXME: can live-config do this?
+            # This CANNOT apply kernel security updates (though it will download them).
-         '--include=rsyslog-relp',
+            # This CANNOT make the upgrades persistent across reboots (they re-download each boot).
-         """--customize-hook=>$1/etc/rsyslog.conf printf '%s\n' 'module(load="imuxsock")' 'module(load="imklog")' 'module(load="omrelp")' 'action(type="omrelp" target="logserv" port="2514" template="RSYSLOG_SyslogProtocol23Format")'""",
+            # FIXME: Would it be cleaner to set Environment=NEEDRESTART_MODE=a in
-
+            #        apt-daily-upgrade.service and/or
-         # Run self-tests on all discoverable hard disks, and (try to) email if something goes wrong.
+            #        unattended-upgrades.service, so
-         '--include=smartmontools bsd-mailx',
+            #        needrestart is noninteractive only when apt is noninteractive?
-         "--customize-hook=>$1/etc/smartd.conf echo 'DEVICESCAN -n standby,15 -a -o on -S on -s (S/../../7/00|L/../01/./01) -t -H -m root -M once'",
+            "--include=unattended-upgrades needrestart",
-
+            "--customize-hook=echo 'unattended-upgrades unattended-upgrades/enable_auto_updates boolean true' | chroot $1 debconf-set-selections",
-         # For rarely-updated, rarely-rebooted SOEs, apply what security updates we can into transient tmpfs COW.
+            """--customize-hook=>$1/etc/needrestart/conf.d/unattended-needrestart.conf  echo '$nrconf{restart} = "a";'""",  # https://bugs.debian.org/894444
-         # This CANNOT apply kernel security updates (though it will download them).
+            # Do an apt update & apt upgrade at boot time (as well as @daily).
-         # This CANNOT make the upgrades persistent across reboots (they re-download each boot).
+            # The lack of /etc/machine-id causes these to be implicitly enabled.
-         # FIXME: Would it be cleaner to set Environment=NEEDRESTART_MODE=a in
+            # FIXME: use dropin in /etc.
-         #        apt-daily-upgrade.service and/or
+            "--customize-hook=>>$1/lib/systemd/system/apt-daily.service		printf '%s\n' '[Install]' 'WantedBy=multi-user.target'",
-         #        unattended-upgrades.service, so
+            "--customize-hook=>>$1/lib/systemd/system/apt-daily-upgrade.service	printf '%s\n' '[Install]' 'WantedBy=multi-user.target'",
-         #        needrestart is noninteractive only when apt is noninteractive?
+            # FIXME: add support for this stuff (for the non-live final install this happens via ansible):
-         '--include=unattended-upgrades needrestart',
+            #
-         "--customize-hook=echo 'unattended-upgrades unattended-upgrades/enable_auto_updates boolean true' | chroot $1 debconf-set-selections",
+            #            unattended-upgrades
-         """--customize-hook=>$1/etc/needrestart/conf.d/unattended-needrestart.conf  echo '$nrconf{restart} = "a";'""",  # https://bugs.debian.org/894444
+            #            smartd
-         # Do an apt update & apt upgrade at boot time (as well as @daily).
+            #            networkd (boot off ANY NIC, not EVERY NIC -- https://github.com/systemd/systemd/issues/9714)
-         # The lack of /etc/machine-id causes these to be implicitly enabled.
+            #            refind (bootloader config)
-         # FIXME: use dropin in /etc.
+            #            misc safety nets
-         "--customize-hook=>>$1/lib/systemd/system/apt-daily.service		printf '%s\n' '[Install]' 'WantedBy=multi-user.target'",
+            #            double-check that mmdebstrap's machine-id support works properly
-         "--customize-hook=>>$1/lib/systemd/system/apt-daily-upgrade.service	printf '%s\n' '[Install]' 'WantedBy=multi-user.target'",
+            # Bare minimum to let me SSH in.
-
+            # FIXME: make this configurable.
-         # FIXME: add support for this stuff (for the non-live final install this happens via ansible):
+            # FIXME: trust a CA certificate instead -- see Zero Trust SSH, Jeremy Stott, LCA 2020 <https://youtu.be/lYzklWPTbsQ>
-         #
+            # WARNING: tinysshd does not support RSA, nor MaxStartups, nor sftp (unless you also install openssh-client, which is huge).
-         #            unattended-upgrades
+            # FIXME: double-check no host keys are baked into the image (openssh-server and dropbear do this).
-         #            smartd
+            "--include=tinysshd rsync",
-         #            networkd (boot off ANY NIC, not EVERY NIC -- https://github.com/systemd/systemd/issues/9714)
+            "--essential-hook=install -dm700 $1/root/.ssh",
-         #            refind (bootloader config)
+            '--essential-hook=echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIapAZ0E0353DaY6xBnasvu/DOvdWdKQ6RQURwq4l6Wu twb@cyber.com.au (Trent W. Buck)" >$1/root/.ssh/authorized_keys',
-         #            misc safety nets
+            # Bare minimum to let me log in locally.
-         #            double-check that mmdebstrap's machine-id support works properly
+            # DO NOT use this on production builds!
-
+            "--essential-hook=chroot $1 passwd --delete root",
-         # Bare minimum to let me SSH in.
+            # Configure language (not needed to boot).
-         # FIXME: make this configurable.
+            # Racism saves a **LOT** of space -- something like 2GB for Debian Live images.
-         # FIXME: trust a CA certificate instead -- see Zero Trust SSH, Jeremy Stott, LCA 2020 <https://youtu.be/lYzklWPTbsQ>
+            # FIXME: use live-config instead?
-         # WARNING: tinysshd does not support RSA, nor MaxStartups, nor sftp (unless you also install openssh-client, which is huge).
+            "--include=locales localepurge",
-         # FIXME: double-check no host keys are baked into the image (openssh-server and dropbear do this).
+            f"--essential-hook=echo locales locales/default_environment_locale   select {args.locale}       | chroot $1 debconf-set-selections",
-         '--include=tinysshd rsync',
+            f"--essential-hook=echo locales locales/locales_to_be_generated multiselect {args.locale} UTF-8 | chroot $1 debconf-set-selections",
-         '--essential-hook=install -dm700 $1/root/.ssh',
+            # FIXME: https://bugs.debian.org/603700
-         '--essential-hook=echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIapAZ0E0353DaY6xBnasvu/DOvdWdKQ6RQURwq4l6Wu twb@cyber.com.au (Trent W. Buck)" >$1/root/.ssh/authorized_keys',
+            "--customize-hook=chroot $1 sed -i /etc/locale.nopurge -e 's/^USE_DPKG/#ARGH#&/'",
-
+            "--customize-hook=chroot $1 localepurge",
-         # Bare minimum to let me log in locally.
+            "--customize-hook=chroot $1 sed -i /etc/locale.nopurge -e 's/^#ARGH#//'",
-         # DO NOT use this on production builds!
+            # Removing documentation also saves a LOT of space.
-         '--essential-hook=chroot $1 passwd --delete root',
+            "--dpkgopt=path-exclude=/usr/share/doc/*",
-
+            "--dpkgopt=path-exclude=/usr/share/info/*",
-         # Configure language (not needed to boot).
+            "--dpkgopt=path-exclude=/usr/share/man/*",
-         # Racism saves a **LOT** of space -- something like 2GB for Debian Live images.
+            "--dpkgopt=path-exclude=/usr/share/omf/*",
-         # FIXME: use live-config instead?
+            "--dpkgopt=path-exclude=/usr/share/help/*",
-         '--include=locales localepurge',
+            "--dpkgopt=path-exclude=/usr/share/gnome/help/*",
-         f'--essential-hook=echo locales locales/default_environment_locale   select {args.locale}       | chroot $1 debconf-set-selections',
+            # Configure timezone (not needed to boot)`
-         f'--essential-hook=echo locales locales/locales_to_be_generated multiselect {args.locale} UTF-8 | chroot $1 debconf-set-selections',
+            # FIXME: use live-config instead?
-         # FIXME: https://bugs.debian.org/603700
+            "--include=tzdata",
-         "--customize-hook=chroot $1 sed -i /etc/locale.nopurge -e 's/^USE_DPKG/#ARGH#&/'",
+            f"--essential-hook=echo tzdata tzdata/Areas                    select {args.timezone[0]} | chroot $1 debconf-set-selections",
-         "--customize-hook=chroot $1 localepurge",
+            f"--essential-hook=echo tzdata tzdata/Zones/{args.timezone[0]} select {args.timezone[1]} | chroot $1 debconf-set-selections",
-         "--customize-hook=chroot $1 sed -i /etc/locale.nopurge -e 's/^#ARGH#//'",
+            # Do the **BARE MINIMUM** to make a USB key that can boot on X86_64 UEFI.
-
+            # We use mtools so we do not ever need root privileges.
-
+            # We can't use mkfs.vfat, as that needs kpartx or losetup (i.e. root).
-         # Removing documentation also saves a LOT of space.
+            # We can't use mkfs.udf, as that needs mount (i.e. root).
-         '--dpkgopt=path-exclude=/usr/share/doc/*',
+            # We can't use "refind-install --usedefault" as that runs mount(8) (i.e. root).
-         '--dpkgopt=path-exclude=/usr/share/info/*',
+            # We don't use genisoimage because
-         '--dpkgopt=path-exclude=/usr/share/man/*',
+            # 1) ISO9660 must die;
-         '--dpkgopt=path-exclude=/usr/share/omf/*',
+            # 2) incomplete UDF 1.5+ support;
-         '--dpkgopt=path-exclude=/usr/share/help/*',
+            # 3) resulting filesystem can't be tweaked after flashing (e.g. debian-live/site.dir/etc/systemd/network/up.network).
-         '--dpkgopt=path-exclude=/usr/share/gnome/help/*',
+            #
-
+            # We use refind because 1) I hate grub; and 2) I like refind.
-
+            # If you want aarch64 or ia32 you need to install their BOOTxxx.EFI files.
-         # Configure timezone (not needed to boot)`
+            # If you want kernel+initrd on something other than FAT, you need refind/drivers_xxx/xxx_xxx.EFI.
-         # FIXME: use live-config instead?
+            #
-         '--include=tzdata',
+            # FIXME: with qemu in UEFI mode (OVMF), I get dumped into startup.nsh (UEFI REPL).
-         f'--essential-hook=echo tzdata tzdata/Areas                    select {args.timezone[0]} | chroot $1 debconf-set-selections',
+            #        From there, I can manually type in "FS0:\EFI\BOOT\BOOTX64.EFI" to start refind, tho.
-         f'--essential-hook=echo tzdata tzdata/Zones/{args.timezone[0]} select {args.timezone[1]} | chroot $1 debconf-set-selections',
+            #        So WTF is its problem?  Does it not support fallback bootloader?
-
+            "--include=refind parted mtools",
-
+            "--essential-hook=echo refind refind/install_to_esp boolean false | chroot $1 debconf-set-selections",
-         # Do the **BARE MINIMUM** to make a USB key that can boot on X86_64 UEFI.
+            "--customize-hook=echo refind refind/install_to_esp boolean true  | chroot $1 debconf-set-selections",
-         # We use mtools so we do not ever need root privileges.
+            "--customize-hook=chroot $1 mkdir -p /boot/USB /boot/EFI/BOOT",
-         # We can't use mkfs.vfat, as that needs kpartx or losetup (i.e. root).
+            "--customize-hook=chroot $1 cp /usr/share/refind/refind/refind_x64.efi /boot/EFI/BOOT/BOOTX64.EFI",
-         # We can't use mkfs.udf, as that needs mount (i.e. root).
+            "--customize-hook=chroot $1 cp /usr/share/refind/refind/refind.conf-sample /boot/EFI/BOOT/refind.conf",
-         # We can't use "refind-install --usedefault" as that runs mount(8) (i.e. root).
+            f"--customize-hook=chroot $1 truncate --size={filesystem_img_size} /boot/USB/filesystem.img",
-         # We don't use genisoimage because
+            f"--customize-hook=chroot $1 parted --script --align=optimal /boot/USB/filesystem.img  mklabel gpt  mkpart {esp_label} {esp_offset}b 100%  set 1 esp on",
-         # 1) ISO9660 must die;
+            f"--customize-hook=chroot $1 mformat -i /boot/USB/filesystem.img@@{esp_offset} -F -v {esp_label}",
-         # 2) incomplete UDF 1.5+ support;
+            f"--customize-hook=chroot $1 mmd     -i /boot/USB/filesystem.img@@{esp_offset} ::{live_media_path}",
-         # 3) resulting filesystem can't be tweaked after flashing (e.g. debian-live/site.dir/etc/systemd/network/up.network).
+            f"""--customize-hook=echo '"Boot with default options" "boot=live live-media-path={live_media_path}"' >$1/boot/refind_linux.conf""",
-         #
+            f"""--customize-hook=chroot $1 find /boot/ -xdev -mindepth 1 -maxdepth 1 -not -name filesystem.img -not -name USB -exec mcopy -vsbpm -i /boot/USB/filesystem.img@@{esp_offset} {{}} :: ';'""",
-         # We use refind because 1) I hate grub; and 2) I like refind.
+            # FIXME: copy-out doesn't handle sparseness, so is REALLY slow (about 50 seconds).
-         # If you want aarch64 or ia32 you need to install their BOOTxxx.EFI files.
+            # Therefore instead leave it in the squashfs, and extract it later.
-         # If you want kernel+initrd on something other than FAT, you need refind/drivers_xxx/xxx_xxx.EFI.
+            #  f'--customize-hook=copy-out /boot/USB/filesystem.img /tmp/',
-         #
+            #  f'--customize-hook=chroot $1 rm /boot/USB/filesystem.img',
-         # FIXME: with qemu in UEFI mode (OVMF), I get dumped into startup.nsh (UEFI REPL).
+            "sid",
-         #        From there, I can manually type in "FS0:\EFI\BOOT\BOOTX64.EFI" to start refind, tho.
+            td / "filesystem.squashfs",
-         #        So WTF is its problem?  Does it not support fallback bootloader?
+        ]
-         '--include=refind parted mtools',
+    )
-         '--essential-hook=echo refind refind/install_to_esp boolean false | chroot $1 debconf-set-selections',
-         '--customize-hook=echo refind refind/install_to_esp boolean true  | chroot $1 debconf-set-selections',
-         '--customize-hook=chroot $1 mkdir -p /boot/USB /boot/EFI/BOOT',
-         '--customize-hook=chroot $1 cp /usr/share/refind/refind/refind_x64.efi /boot/EFI/BOOT/BOOTX64.EFI',
-         '--customize-hook=chroot $1 cp /usr/share/refind/refind/refind.conf-sample /boot/EFI/BOOT/refind.conf',
-         f'--customize-hook=chroot $1 truncate --size={filesystem_img_size} /boot/USB/filesystem.img',
-         f'--customize-hook=chroot $1 parted --script --align=optimal /boot/USB/filesystem.img  mklabel gpt  mkpart {esp_label} {esp_offset}b 100%  set 1 esp on',
-         f'--customize-hook=chroot $1 mformat -i /boot/USB/filesystem.img@@{esp_offset} -F -v {esp_label}',
-         f'--customize-hook=chroot $1 mmd     -i /boot/USB/filesystem.img@@{esp_offset} ::{live_media_path}',
-         f"""--customize-hook=echo '"Boot with default options" "boot=live live-media-path={live_media_path}"' >$1/boot/refind_linux.conf""",
-
-         f"""--customize-hook=chroot $1 find /boot/ -xdev -mindepth 1 -maxdepth 1 -not -name filesystem.img -not -name USB -exec mcopy -vsbpm -i /boot/USB/filesystem.img@@{esp_offset} {{}} :: ';'""",
-         # FIXME: copy-out doesn't handle sparseness, so is REALLY slow (about 50 seconds).
-         # Therefore instead leave it in the squashfs, and extract it later.
-         #  f'--customize-hook=copy-out /boot/USB/filesystem.img /tmp/',
-         #  f'--customize-hook=chroot $1 rm /boot/USB/filesystem.img',
-
-
-         'sid',
-         td / 'filesystem.squashfs'
-        ])
-
-    with args.output_file.open('wb') as f:
-        subprocess.check_call(['rdsquashfs', '--cat=boot/USB/filesystem.img', td / 'filesystem.squashfs'], stdout=f)
-    subprocess.check_call([
-        'mcopy', '-i', f'{args.output_file}@@{esp_offset}', td / 'filesystem.squashfs', f'::{live_media_path}/filesystem.squashfs'])
 
+    with args.output_file.open("wb") as f:
+        subprocess.check_call(
+            ["rdsquashfs", "--cat=boot/USB/filesystem.img", td / "filesystem.squashfs"],
+            stdout=f,
+        )
+    subprocess.check_call(
+        [
+            "mcopy",
+            "-i",
+            f"{args.output_file}@@{esp_offset}",
+            td / "filesystem.squashfs",
+            f"::{live_media_path}/filesystem.squashfs",
+        ]
+    )

gpgvnoexpkeysig

@@ -0,0 +1,50 @@
+#!/bin/sh
+#
+# This script is in the public domain
+#
+# Author: Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
+#
+# This is a wrapper around gpgv as invoked by apt. It turns EXPKEYSIG results
+# from gpgv into GOODSIG results. This is necessary for apt to access very old
+# timestamps from snapshot.debian.org for which the GPG key is already expired:
+#
+#     Get:1 http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease [242 kB]
+#     Err:1 http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease
+#       The following signatures were invalid: EXPKEYSIG 8B48AD6246925553 Debian Archive Automatic Signing Key (7.0/wheezy) <ftpmaster@debian.org>
+#     Reading package lists...
+#     W: GPG error: http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease: The following signatures were invalid: EXPKEYSIG 8B48AD6246925553 Debian Archive Automatic Signing Key (7.0/wheezy) <ftpmaster@debian.org>
+#     E: The repository 'http://snapshot.debian.org/archive/debian/20150106T000000Z unstable InRelease' is not signed.
+#
+# To use this script, call apt with
+#
+#    -o Apt::Key::gpgvcommand=/usr/libexec/mmdebstrap/gpgvnoexpkeysig
+#
+# Scripts doing similar things can be found here:
+#
+#  * debuerreotype as /usr/share/debuerreotype/scripts/.gpgv-ignore-expiration.sh
+#  * derivative census: salsa.d.o/deriv-team/census/-/blob/master/bin/fakegpgv
+
+set -eu
+
+find_gpgv_status_fd() {
+	while [ "$#" -gt 0 ]; do
+		if [ "$1" = '--status-fd' ]; then
+			echo "$2"
+			return 0
+		fi
+		shift
+	done
+	# default fd is stdout
+	echo 1
+}
+GPGSTATUSFD="$(find_gpgv_status_fd "$@")"
+
+case $GPGSTATUSFD in
+	''|*[!0-9]*)
+		echo "invalid --status-fd argument" >&2
+		exit 1
+		;;
+esac
+
+# we need eval because we cannot redirect a variable fd
+eval 'exec gpgv "$@" '"$GPGSTATUSFD"'>&1 | sed "s/^\[GNUPG:\] EXPKEYSIG /[GNUPG:] GOODSIG /" >&'"$GPGSTATUSFD"

hooks/merged-usr/setup00.sh

@@ -5,7 +5,13 @@
 #
 # Using this hook script, you can emulate what debootstrap does to set up
 # merged /usr via directory symlinks, even using the exact same shell function
-# that debootstrap uses.
+# that debootstrap uses by running mmdebstrap with:
+#
+#     --setup-hook=/usr/share/mmdebstrap/hooks/merged-usr/setup00.sh
+#
+# Alternatively, you can setup merged-/usr by installing the usrmerge package:
+#
+#     --include=usrmerge
 #
 # mmdebstrap will not include this functionality via a --merged-usr option
 # because there are many reasons against implementing merged-/usr that way:
@@ -24,6 +30,10 @@
 # The information whether a distribution uses this approach to merged-/usr or
 # not is not anymore contained in its packages but in a tool from the outside.
 #
+# Example real world problem: I'm using debbisect to bisect Debian unstable
+# between 2015 and today. For which snapshot.d.o timestamp should a merged-/usr
+# chroot be created and for which ones not?
+#
 # The problem is not the idea of merged-/usr but the problem is the way how it
 # got implemented in debootstrap via directory symlinks. That way of rolling
 # out merged-/usr is bad from the dpkg point-of-view and completely opposite of

ldconfig.fakechroot

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+#
+# This script is in the public domain
+#
+# Author: Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
+#
+# This is command substitution for ldconfig under fakechroot:
+#
+#     export FAKECHROOT_CMD_SUBST=/sbin/ldconfig=/path/to/ldconfig.fakechroot
+#
+# Statically linked binaries cannot work with fakechroot and thus have to be
+# replaced by either /bin/true or a more clever solution like this one. The
+# ldconfig command supports the -r option which allows passing a chroot
+# directory for ldconfig to work in. This can be used to run ldconfig without
+# fakechroot but still let it create /etc/ld.so.cache inside the chroot.
+#
+# Since absolute symlinks are broken without fakechroot to translate them,
+# we read /etc/ld.so.conf and turn all absolute symlink shared libraries into
+# relative ones. At program exit, the original state is restored.
+
+
+import os
+import sys
+import subprocess
+import atexit
+import glob
+from pathlib import Path
+
+symlinks = []
+
+
+def restore_symlinks():
+    for (link, target, atime, mtime) in symlinks:
+        link.unlink()
+        link.symlink_to(target)
+        os.utime(link, times=None, ns=(atime, mtime), follow_symlinks=False)
+
+
+atexit.register(restore_symlinks)
+
+
+def get_libdirs(chroot, configs):
+    res = []
+    for conf in configs:
+        for line in (Path(conf)).read_text().splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            if line.startswith("#"):
+                continue
+            if line.startswith("include "):
+                assert line.startswith("include /")
+                res.extend(
+                    get_libdirs(chroot, chroot.glob(line.removeprefix("include /")))
+                )
+                continue
+            assert line.startswith("/"), line
+            line = line.lstrip("/")
+            if not (chroot / Path(line)).is_dir():
+                continue
+            for f in (chroot / Path(line)).iterdir():
+                if not f.is_symlink():
+                    continue
+                linktarget = f.readlink()
+                # make sure that the linktarget is an absolute path inside the
+                # chroot
+                if not str(linktarget).startswith("/"):
+                    continue
+                if chroot not in linktarget.parents:
+                    continue
+                # store original link so that we can restore it later
+                symlinks.append(
+                    (f, linktarget, f.lstat().st_atime_ns, f.lstat().st_mtime_ns)
+                )
+                # replace absolute symlink by relative link
+                relative = os.path.relpath(linktarget, f.parent)
+                f.unlink()
+                f.symlink_to(relative)
+    return res
+
+
+def main():
+    if "FAKECHROOT_BASE_ORIG" not in os.environ:
+        print("FAKECHROOT_BASE_ORIG is not set", file=sys.stderr)
+        print(
+            "must be executed under fakechroot using FAKECHROOT_CMD_SUBST",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    chroot = Path(os.environ["FAKECHROOT_BASE_ORIG"])
+
+    if not (chroot / "sbin" / "ldconfig").exists():
+        sys.exit(0)
+
+    (chroot / "var" / "cache" / "ldconfig").mkdir(
+        mode=0o700, parents=True, exist_ok=True
+    )
+
+    for d in get_libdirs(chroot, [chroot / "etc" / "ld.so.conf"]):
+        make_relative(d)
+
+    # we add any additional arguments before "-r" such that any other "-r"
+    # option will be overwritten by the one we set
+    subprocess.check_call(
+        [chroot / "sbin" / "ldconfig"] + sys.argv[1:] + ["-r", chroot]
+    )
+
+
+if __name__ == "__main__":
+    main()

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '0.7.5';
+our $VERSION = '0.8.0';
 
 use English;
 use Getopt::Long;
@@ -251,6 +251,24 @@ sub get_tar_compressor {
     return;
 }
 
+# avoid dependency on String::ShellQuote by implementing the mechanism
+# from python's shlex.quote function
+sub shellescape {
+    my $string = shift;
+    if (length $string == 0) {
+        return "''";
+    }
+    # search for occurrences of characters that are not safe
+    # the 'a' regex modifier makes sure that \w only matches ASCII
+    if ($string !~ m/[^\w@\%+=:,.\/-]/a) {
+        return $string;
+    }
+    # wrap the string in single quotes and handle existing single quotes by
+    # putting them outside of the single-quoted string
+    $string =~ s/'/'"'"'/g;
+    return "'$string'";
+}
+
 sub test_unshare_userns {
     my $verbose = shift;
     if ($EFFECTIVE_USER_ID == 0) {
@@ -1977,16 +1995,15 @@ sub run_setup() {
         copy($tmpfile, \*STDERR);
     }
 
-    if (any { $_ eq $options->{mode} } ('fakechroot', 'proot')) {
+    if (none { $_ eq $options->{mode} } ('fakechroot', 'proot')) {
         # Apt dropping privileges to another user than root is not useful in
         # fakechroot and proot mode because all users are faked and thus there
-        # is no real privilege difference anyways. Thus, we also print no
+        # is no real privilege difference anyways. We could set
-        # warning message in this case.
+        # APT::Sandbox::User "root" in fakechroot and proot mode but we don't
-        open my $fh, '>>', $tmpfile
+        # because if we would, then /var/cache/apt/archives/partial/ and
-          or error "cannot open $tmpfile for appending: $!";
+        # /var/lib/apt/lists/partial/ would not be owned by the _apt user
-        print $fh "APT::Sandbox::User \"root\";\n";
+        # if mmdebstrap was run in fakechroot or proot mode.
-        close $fh;
+        #
-    } else {
         # when apt-get update is run by the root user, then apt will attempt to
         # drop privileges to the _apt user. This will fail if the _apt user
         # does not have permissions to read the root directory. In that case,
@@ -2252,6 +2269,8 @@ sub run_download() {
         # install call in run_download() instead of assembling the package list
         # here and passing it through all the way. Then this function only
         # takes care of retrieving the essential packages.
+        #
+        # https://salsa.debian.org/apt-team/apt/-/merge_requests/185
         my %ess_pkgs;
         my %ess_pkgs_target;
         my %pkgs_to_install_target = %pkgs_to_install;
@@ -2643,11 +2662,15 @@ sub run_prepare {
         # /etc/fakechroot/debootstrap.env and
         # /etc/fakechroot/chroot.env
         {
+            my $ldconfig = getcwd() . '/ldconfig.fakechroot';
+            if (!-x $ldconfig) {
+                $ldconfig = '/usr/libexec/mmdebstrap/ldconfig.fakechroot';
+            }
             my @fakechrootsubst = ();
             foreach my $d ('/usr/sbin', '/usr/bin', '/sbin', '/bin') {
                 push @fakechrootsubst, "$d/chroot=/usr/sbin/chroot.fakechroot";
                 push @fakechrootsubst, "$d/mkfifo=/bin/true";
-                push @fakechrootsubst, "$d/ldconfig=/bin/true";
+                push @fakechrootsubst, "$d/ldconfig=$ldconfig";
                 push @fakechrootsubst, "$d/ldd=/usr/bin/ldd.fakechroot";
                 push @fakechrootsubst, "$d/ischroot=/bin/true";
             }
@@ -2674,17 +2697,6 @@ sub run_prepare {
             $ENV{FAKECHROOT_AF_UNIX_PATH} = "/tmp";
         }
         {
-            my @ldsoconf = ('/etc/ld.so.conf');
-            opendir(my $dh, '/etc/ld.so.conf.d')
-              or error "Can't opendir(/etc/ld.so.conf.d): $!";
-            while (my $entry = readdir $dh) {
-                # skip the "." and ".." entries
-                next if $entry eq ".";
-                next if $entry eq "..";
-                next if $entry !~ /\.conf$/;
-                push @ldsoconf, "/etc/ld.so.conf.d/$entry";
-            }
-            closedir($dh);
             my @ldlibpath = ();
             if (defined $ENV{LD_LIBRARY_PATH}
                 && $ENV{LD_LIBRARY_PATH} ne "") {
@@ -2692,15 +2704,34 @@ sub run_prepare {
             }
             # FIXME: workaround allowing installation of systemd should
             # live in fakechroot, see #917920
-            push @ldlibpath, "/lib/systemd";
+            push @ldlibpath, "$options->{root}/lib/systemd";
-            foreach my $fname (@ldsoconf) {
+            my $parse_ld_so_conf;
-                open my $fh, "<", $fname
+            $parse_ld_so_conf = sub {
-                  or error "cannot open $fname for reading: $!";
+                foreach my $conf (@_) {
-                while (my $line = <$fh>) {
+                    next if !-r $conf;
-                    next if $line !~ /^\//;
+                    open my $fh, '<', "$conf" or error "can't read $conf: $!";
-                    push @ldlibpath, $line;
+                    while (my $line = <$fh>) {
+                        chomp $line;
+                        if ($line eq "") {
+                            next;
+                        }
+                        if ($line =~ /^#/) {
+                            next;
+                        }
+                        if ($line =~ /include (.*)/) {
+                            $parse_ld_so_conf->(glob("$options->{root}/$1"));
+                            next;
+                        }
+                        if (!-d "$options->{root}/$line") {
+                            next;
+                        }
+                        push @ldlibpath, "$options->{root}/$line";
+                    }
+                    close $fh;
                 }
-                close $fh;
+            };
+            if (-e "$options->{root}/etc/ld.so.conf") {
+                $parse_ld_so_conf->("$options->{root}/etc/ld.so.conf");
             }
             ## no critic (Variables::RequireLocalizedPunctuationVars)
             $ENV{LD_LIBRARY_PATH} = join ':', @ldlibpath;
@@ -2979,7 +3010,7 @@ sub run_install() {
             #
             # The DPkg::Install::Recursive::force=true workaround can be
             # dropped after this issue is fixed:
-            # https://salsa.debian.org/apt-team/apt/-/merge_requests/178
+            # https://salsa.debian.org/apt-team/apt/-/merge_requests/189
             #
             # We could also move the dpkg call to the outside and run dpkg with
             # --root but this would only make sense in situations where there
@@ -2998,8 +3029,12 @@ sub run_install() {
                                     'DPkg::Options::=--unset=TMPDIR',
                                     '-o',
                                     'DPkg::Options::=dpkg',
-                                    '-o',
+                                    $options->{mode} eq 'fakechroot'
-                                    'DPkg::Install::Recursive::force=true',
+                                    ? (
+                                        '-o',
+                                        'DPkg::Install::Recursive::force=true'
+                                      )
+                                    : (),
                                     '-o',
                                     "DPkg::Chroot-Directory=$options->{root}",
                                     '--yes',
@@ -3128,7 +3163,7 @@ sub run_cleanup() {
               or error "cannot unlink /etc/machine-id: $!";
             open my $fh, '>', "$options->{root}/etc/machine-id"
               or error "failed to open(): $!";
-            print $fh "uninitialized";
+            print $fh "uninitialized\n";
             close $fh;
         }
     }
@@ -4263,6 +4298,9 @@ sub approx_disk_usage {
             $installed_size += scalar @devfiles;
         } elsif ($File::Find::name =~ /^$directory\/dev\//) {
             # ignore everything below /dev
+        } elsif (-l $File::Find::name) {
+            # -f follows symlinks, so we first check if we have a symlink
+            $installed_size += 1;
         } elsif (-f $File::Find::name) {
             # add file size in 1024 byte blocks, rounded up
             $installed_size += int(((-s $File::Find::name) + 1024) / 1024);
@@ -4554,7 +4592,7 @@ sub main() {
         my $content = do { local $/; <$rfh> };
         waitpid $pid, 0;
         my $result = 0;
-        if ($? == 0 and $content =~ /^fakechroot \d\.\d+$/) {
+        if ($? == 0 and $content =~ /^fakechroot [0-9.]+$/) {
             $result = 1;
         }
         return $result;
@@ -5684,6 +5722,12 @@ sub main() {
                 }
 
                 if ($options->{mode} eq 'fakechroot') {
+                    # By default, FAKECHROOT_EXCLUDE_PATH includes /proc and
+                    # /sys which means that the resulting tarball will contain
+                    # the permission and ownership information of /proc and
+                    # /sys from the outside, which we want to avoid.
+                    ## no critic (Variables::RequireLocalizedPunctuationVars)
+                    $ENV{FAKECHROOT_EXCLUDE_PATH} = "/dev";
                     # Fakechroot requires tar to run inside the chroot or
                     # otherwise absolute symlinks will include the path to the
                     # root directory
@@ -6095,6 +6139,7 @@ the file will be appended to 99mmdebstrap verbatim.
 Example: This is necessary for allowing old timestamps from snapshot.debian.org
 
     --aptopt='Acquire::Check-Valid-Until "false"'
+    --aptopt='Apt::Key::gpgvcommand "/usr/libexec/mmdebstrap/gpgvnoexpkeysig"'
 
 Example: Settings controlling download of package description translations
 
@@ -6187,6 +6232,10 @@ package list is free of duplicates. So the following are equivalent:
     --include=pkg1/stable,pkg2=1.0,pkg3-
     --incl=pkg1/stable --incl="pkg2=1.0 pkg3-" --incl=pkg2=1.0,pkg3-
 
+Example: setting up merged-/usr via the usrmerge package
+
+    --include=usrmerge
+
 =item B<--components>=I<comp1>[,I<comp2>,...]
 
 Comma or whitespace separated list of components like main, contrib and
@@ -6233,13 +6282,19 @@ installed. At that point, the chroot directory does not contain any
 executables and thus cannot be chroot-ed into. See section B<HOOKS> for more
 information.
 
-Example: Setup merged-/usr via symlinks
+Example 1: Setup merged-/usr via symlinks, omitting the architecture specific
+symlinks
 
     --setup-hook='for d in bin sbin lib; do ln -s usr/$d "$1/$d";
         mkdir -p "$1/usr/$d"; done'
 
-Example: Setup chroot for installing a sub-essential busybox-based chroot with
+Example 2: Setup merged-/usr using debootstrap-method which takes care of the
---variant=custom
+architecture specific symlinks
+
+    --setup-hook=/usr/share/mmdebstrap/hooks/merged-usr/setup00.sh
+
+Example 3: Setup chroot for installing a sub-essential busybox-based chroot
+with --variant=custom
 --include=dpkg,busybox,libc-bin,base-files,base-passwd,debianutils
 
     --setup-hook='mkdir -p "$1/bin"'
@@ -6331,6 +6386,11 @@ Example 2: Setup chroot for installing a sub-essential busybox-based chroot
 
     --hook-dir=/usr/share/mmdebstrap/hooks/busybox
 
+Example 3: Setup merged-/usr using debootstrap-method which takes care of the
+architecture specific symlinks
+
+    --hook-dir=/usr/share/mmdebstrap/hooks/merged-usr
+
 =item B<--skip>=I<stage>[,I<stage>,...]
 
 B<mmdebstrap> tries hard to implement sensible defaults and will try to stop
@@ -6431,9 +6491,8 @@ fakeroot.env> and use C<fakeroot.env> later when entering the chroot with
 C<fakechroot fakeroot -i fakeroot.env chroot ...>. This mode will not work if
 maintainer scripts are unable to handle C<LD_PRELOAD> correctly like the
 package B<initramfs-tools> until version 0.132. This mode will also not work
-with a different libc inside the chroot than on the outside. Since ldconfig
+with a different libc inside the chroot than on the outside. See the section
-cannot run under fakechroot, the final system will not contain
+B<LIMITATIONS> in B<fakechroot(1)>.
-F</etc/ld.so.cache>. See the section B<LIMITATIONS> in B<fakechroot(1)>.
 
 =item B<proot>
 
@@ -7076,7 +7135,7 @@ extension to compressor applies:
  .lz4      lz4
  .xz       xz --threads=0
  .txz      xz --threads=0
- .zst      zstd
+ .zst      zstd --threads=0
 
 To change compression specific options, either use the respecitve environment
 variables like B<XZ_OPT> or send B<mmdebstrap> output to your compressor of

proxysolver

@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
-
+#
+# This script is in the public domain
+#
+# Author: Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
+#
 # thin layer around /usr/lib/apt/solvers/apt, so that we can capture the solver
 # result
 #

tarfilter

@@ -2,6 +2,8 @@
 #
 # This script is in the public domain
 #
+# Author: Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
+#
 # This script accepts a tarball on standard input and filters it according to
 # the same rules used by dpkg --path-exclude and --path-include, using command
 # line options of the same name. The result is then printed on standard output.

taridshift

@@ -2,6 +2,8 @@
 #
 # This script is in the public domain
 #
+# Author: Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
+#
 # This script accepts a tarball on standard input and prints a tarball on
 # standard output with the same contents but all uid and gid ownership
 # information shifted by the value given as first command line argument.