release 0.8.5

Thanks to Francesco Poli for providing ideas and testing this.

CHANGELOG.md

@@ -1,3 +1,8 @@
+0.8.5 (2022-03-07)
+------------------
+
+ - improve documentation
+
 0.8.4 (2022-02-11)
 ------------------
 

coverage.sh

@@ -304,6 +304,15 @@ for f in "/tmp/debian-$dist-debootstrap/etc/shells" "/tmp/debian-$dist-mm/etc/sh
 	sort -o "\$f" "\$f"
 done
 
+# Because of unreproducible uids (#969631) we created the _apt user ourselves
+# and because passwd is not Essential:yes we didn't use useradd. But newer
+# versions of adduser and shadow will create a different /etc/shadow
+for f in shadow shadow-; do
+	if grep -q '^_apt:!:' /tmp/debian-$dist-debootstrap/etc/\$f; then
+		sed -i 's/^_apt:\*:\([^:]\+\):0:99999:7:::$/_apt:!:\1::::::/' /tmp/debian-$dist-mm/etc/\$f
+	fi
+done
+
 # workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
 if ! cmp /tmp/debian-$dist-debootstrap/etc/shadow /tmp/debian-$dist-mm/etc/shadow; then
 	echo patching /etc/shadow on $dist $variant >&2
@@ -335,7 +344,7 @@ if [ $variant = - ]; then
 fi
 
 # check if the file content differs
-diff --unified --no-dereference --recursive /tmp/debian-$dist-debootstrap /tmp/debian-$dist-mm
+diff --unified --no-dereference --recursive /tmp/debian-$dist-debootstrap /tmp/debian-$dist-mm >&2
 
 # check permissions, ownership, symlink targets, modification times using tar
 # directory mtimes will differ, thus we equalize them first

make_mirror.sh

@@ -638,7 +638,7 @@ END
 		copy-in "$tmpdir/mini-httpd" /etc/default : \
 		copy-in "$tmpdir/hosts" /etc/ : \
 		touch /mmdebstrap-testenv : \
-		upload /usr/lib/SYSLINUX/mbr.bin /mbr.bin : \
+		upload /usr/lib/EXTLINUX/mbr.bin /mbr.bin : \
 		copy-file-to-device /mbr.bin /dev/sda size:440 : \
 		rm /mbr.bin : \
 		extlinux / : \

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '0.8.4';
+our $VERSION = '0.8.5';
 
 use English;
 use Getopt::Long;
@@ -4094,7 +4094,10 @@ sub approx_disk_usage {
             $installed_size += 1;
         }
     };
-    find($scan_installed_size, $directory);
+    # We use no_chdir because otherwise the unshared user has to have read
+    # permissions for the current working directory when producing an ext2
+    # image. See https://bugs.debian.org/1005857
+    find({ wanted => $scan_installed_size, no_chdir => 1 }, $directory);
 
     # because the above is only a heuristic we add 10% extra for good measure
     return int($installed_size * 1.1);
@@ -6159,7 +6162,7 @@ information.
 
 Example: Install busybox symlinks
 
-    --extract-hook='chroot "$1" busybox --install -s'
+    --extract-hook='chroot "$1" /bin/busybox --install -s'
 
 =item B<--essential-hook>=I<command>
 
@@ -6589,7 +6592,8 @@ chroot as I<fileoutside>. In contrast to B<copy-out>, this command only
 handles files and not directories. To copy a directory recursively out of the
 chroot, use B<copy-out> or B<tar-out>. Its advantage is, that by being able to
 specify the full path on the outside, including the filename, the file on the
-outside can have a different name from the file on the inside.
+outside can have a different name from the file on the inside. In contrast to
+B<copy-out> and B<tar-out>, this command follows symlinks.
 
 =item B<upload> I<fileoutside> I<fileinside>
 
@@ -6829,6 +6833,20 @@ Create a bootable USB Stick that boots into a full Debian desktop:
     $ qemu-system-x86_64 -m 1G -enable-kvm debian-unstable.img
     $ sudo dd if=debian-unstable.img of=/dev/sdXXX status=progress
 
+On architectures without extlinux you can also boot using grub2:
+
+    $ mmdebstrap --include=linux-image-amd64,grub2,systemd-sysv unstable fs.tar
+    $ guestfish -N debian-unstable.img=disk:2G -- \
+        part-disk /dev/sda mbr : \
+        part-set-bootable /dev/sda 1 true : \
+        set-label /dev/sda1 rootfs : \
+        mkfs ext4 /dev/sda1 : mount /dev/sda1 / : \
+        tar-in fs.tar / xattrs:true : \
+        command "grub-install /dev/sda" : \
+        command update-grub : \
+        sync : umount / : shutdown
+    $ qemu-system-x86_64 -m 1G -enable-kvm debian-unstable.img
+
 Build libdvdcss2.deb without installing installing anything or changing apt
 sources on the current system:
 
@@ -6854,7 +6872,7 @@ Use as replacement for autopkgtest-build-qemu and vmdb2:
     >
     > label linux
     > kernel /vmlinuz
-    > append initrd=/initrd.img root=/dev/vda1 console=ttyS0
+    > append initrd=/initrd.img root=/dev/vda1 rw console=ttyS0
     END
     $ guestfish -N debian-unstable.img=disk:8G -- \
         part-disk /dev/sda mbr : \

mmdebstrap-autopkgtest-build-qemu

@@ -0,0 +1,255 @@
+#!/bin/sh
+#
+# © 2022 Johannes Schauer Marin Rodrigues <josch@mister-muffin.de>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# The software is provided "as is", without warranty of any kind, express or
+# implied, including but not limited to the warranties of merchantability,
+# fitness for a particular purpose and noninfringement. In no event shall the
+# authors or copyright holders be liable for any claim, damages or other
+# liability, whether in an action of contract, tort or otherwise, arising
+# from, out of or in connection with the software or the use or other dealings
+# in the software.
+
+set -eu
+
+# This script creates debian-$RELEASE.qcow2 in the current directory which can
+# then be used by the autopkgtest qemu backend.
+#
+# Thanks to Francesco Poli for providing ideas and testing this.
+#
+# Thanks to Lars Wirzenius of vmdb2 where the grub and efi magic comes from.
+#
+# Only the native architecture is supported because guestfish doesn't support
+# foreign architectures.
+
+usage() {
+	echo "Usage: $0 [--size=SIZE] [--boot=BOOT] RELEASE IMAGE" >&2
+	echo >&2
+	echo "RELEASE is a Debian release like unstable" >&2
+	echo "IMAGE will be stored in qcow2 format" >&2
+	echo "SIZE is 25G by default" >&2
+	echo "BOOT is either auto (the default), bios, efi or ieee1275" >&2
+}
+
+nativearch="$(dpkg --print-architecture)"
+
+SIZE="25G" # default from autopkgtest-build-qemu
+BOOT="auto"
+if [ "$#" -lt 2 ]; then
+	echo "Error: Insufficient number of arguments" >&2
+	usage
+	exit 1
+elif [ "$#" -eq 2 ]; then
+	RELEASE=$1
+	IMAGE=$2
+else
+	# parse options
+	OPTS=$(getopt -n "$0" -o h --long size:,boot:,architecture:,help -- "$@")
+	if [ "$?" -ne 0 ]; then
+		echo "Error: Cannot parse arguments" >&2
+		usage
+		exit 1
+	fi
+	eval set -- "$OPTS"
+	while true; do
+		case "$1" in
+			--size) SIZE="$2"; shift 2; continue;;
+			--boot) BOOT="$2"; shift 2; continue;;
+			--help) usage; exit 1;;
+			--architecture)
+				echo "Error: cannot (yet) create foreign architecture images" >&2
+				exit 1
+				;;
+			--) shift; break;;
+			*)
+				echo "Error: unknown option $1" >&2
+				usage
+				exit 1
+				;;
+		esac
+	done
+	RELEASE=$1
+	IMAGE=$2
+fi
+
+# By default with --boot=auto (the default), bios boot is chosen for
+# amd64 and i386. Compare /usr/share/autopkgtest/lib/autopkgtest_qemu.py
+# But in practice, amd64 and i386 also support efi boot. But then
+# autopkgtest-virt-qemu has to be run with --boot=efi
+case "$BOOT" in
+	auto)
+		case "$nativearch" in
+			amd64|i386) BOOT=bios;;
+			armhf|arm64) BOOT=efi;;
+			ppc64el) BOOT=ieee1275;;
+		esac
+		;;
+	bios)
+		case "$nativearch" in amd64|i386);;
+			*)
+				echo "bios booting only possible on amd64 and i386" >&2
+				exit 1
+				;;
+		esac
+		;;
+	efi)
+		case "$nativearch" in amd64|i386|armhf|arm64);;
+			*)
+				echo "efi booting only possible on amd64, i386, armhf and arm64" >&2
+				exit 1
+				;;
+		esac
+		;;
+	ieee1275)
+		if [ "$nativearch" != "ppc64el" ]; then
+			echo "ieee1275 booting only possible on ppc64el" >&2
+			exit 1
+		fi
+		;;
+	*)
+		echo "invalid value for --boot" >&2;;
+esac
+
+case "$nativearch" in
+	amd64)
+		[ $BOOT = bios ] || [ $BOOT = efi ]
+		if [ $BOOT = bios ]; then
+			include="linux-image-amd64 grub-pc"
+			grub_target="i386-pc"
+		elif [ $BOOT = efi ]; then
+			include="linux-image-amd64 grub-efi"
+			grub_target="x86_64-efi"
+		fi
+		;;
+	arm64)
+		[ $BOOT = efi ]
+		include="linux-image-arm64 grub-efi"
+		grub_target="arm64-efi"
+		;;
+	armhf)
+		[ $BOOT = efi ]
+		include="linux-image-armmp-lpae grub-efi"
+		grub_target="arm-efi"
+		;;
+	i386)
+		[ $BOOT = bios ] || [ $BOOT = efi ]
+		if [ $BOOT = bios ]; then
+			include="linux-image-686-pae grub-pc"
+			grub_target="i386-efi"
+		elif [ $BOOT = efi ]; then
+			include="linux-image-686-pae grub-efi"
+			grub_target="i386-efi"
+		fi
+		;;
+	ppc64el)
+		[ $BOOT = ieee1275 ]
+		include="linux-image-powerpc64le grub-ieee1275"
+		grub_target="powerpc-ieee1275"
+		;;
+	*)
+		echo "architecture $nativearch not yet supported" >&2
+		exit 1
+		;;
+esac
+
+case "$nativearch" in
+	arm64|armhf) serial="loglevel=3 console=tty0 console=ttyAMA0,115200n8" ;;
+	ppc64el) serial="loglevel=3 console=tty0 console=hvc0,115200n8" ;;
+	*) serial="loglevel=3 console=tty0 console=ttyS0,115200n8" ;;
+esac
+
+if ! command -v guestfish >/dev/null; then
+	echo "Error: requires guestfish being installed" >&2
+	exit 1
+fi
+
+if [ ! -e /usr/share/autopkgtest/setup-commands/setup-testbed ]; then
+	echo "Error: requires autopkgtest being installed" >&2
+	exit 1
+fi
+
+run_mmdebstrap() {
+	mmdebstrap --variant=important --include="$include" \
+	--customize-hook='chroot "$1" passwd --delete root' \
+	--customize-hook='chroot "$1" useradd --home-dir /home/user --create-home user' \
+	--customize-hook='chroot "$1" passwd --delete user' \
+	--customize-hook='echo host > "$1/etc/hostname"' \
+	--customize-hook='echo "127.0.0.1 localhost host" > "$1/etc/hosts"' \
+	--customize-hook='env AUTOPKGTEST_BUILD_QEMU=1 /usr/share/autopkgtest/setup-commands/setup-testbed "$1"' \
+	"$RELEASE" -
+}
+
+guestfish_bios() {
+	guestfish -- \
+	disk-create "$IMAGE" qcow2 "$SIZE" : \
+	add-drive "$IMAGE" format:qcow2 : \
+	launch : \
+	part-disk /dev/sda mbr : \
+	part-set-bootable /dev/sda 1 true : \
+	mkfs ext4 /dev/sda1 : mount /dev/sda1 / : \
+	tar-in - / xattrs:true : \
+	command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda1) / ext4 errors=remount-ro 0 1 > /etc/fstab'" : \
+	command "update-initramfs -u" : \
+	command "grub-mkconfig -o /boot/grub/grub.cfg" : \
+	command "grub-install /dev/sda --target=$grub_target --no-nvram --force-extra-removable --no-floppy --modules=part_gpt --grub-mkdevicemap=/boot/grub/device.map" : \
+	sync : umount / : shutdown
+}
+
+guestfish_efi() {
+	guestfish -- \
+	disk-create "$IMAGE" qcow2 "$SIZE" : \
+	add-drive "$IMAGE" format:qcow2 : \
+	launch : \
+	part-init /dev/sda gpt : \
+	part-add /dev/sda primary 8192 262144 : \
+	part-add /dev/sda primary 262145 -34 : \
+	part-set-gpt-type /dev/sda 1 C12A7328-F81F-11D2-BA4B-00A0C93EC93B  : \
+	mkfs ext4 /dev/sda2 : mount /dev/sda2 / : \
+	tar-in - / xattrs:true : \
+	mkdir-p /boot/efi : \
+	mkfs vfat /dev/sda1 : mount /dev/sda1 /boot/efi : \
+	command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda2) / ext4 errors=remount-ro 0 1 > /etc/fstab'" : \
+	command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda1) /boot/efi vfat errors=remount-ro 0 2 >> /etc/fstab'" : \
+	command "sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=/GRUB_CMDLINE_LINUX_DEFAULT=\"biosdevname=0 net.ifnames=0 consoleblank=0 rw $serial\"/' /etc/default/grub" : \
+	command "update-initramfs -u" : \
+	command "grub-mkconfig -o /boot/grub/grub.cfg" : \
+	command "grub-install /dev/sda --target=$grub_target --no-nvram --force-extra-removable --no-floppy --modules=part_gpt --grub-mkdevicemap=/boot/grub/device.map" : \
+	sync : umount /boot/efi : umount / : shutdown
+}
+
+guestfish_ieee1275() {
+	guestfish -- \
+	disk-create "$IMAGE" qcow2 "$SIZE" : \
+	add-drive "$IMAGE" format:qcow2 : \
+	launch : \
+	part-init /dev/sda gpt : \
+	part-add /dev/sda primary 8192 20480 : \
+	part-add /dev/sda primary 20481 -34 : \
+	part-set-gpt-type /dev/sda 1 9E1A2D38-C612-4316-AA26-8B49521E5A8B : \
+	mkfs ext4 /dev/sda2 : mount /dev/sda2 / : \
+	tar-in - / xattrs:true : \
+	command "sh -c 'echo UUID=\$(blkid -c /dev/null -o value -s UUID /dev/sda2) / ext4 errors=remount-ro 0 1 > /etc/fstab'" : \
+	command "sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=/GRUB_CMDLINE_LINUX_DEFAULT=\"biosdevname=0 net.ifnames=0 consoleblank=0 rw $serial\"/' /etc/default/grub" : \
+	command "update-initramfs -u" : \
+	command "grub-mkconfig -o /boot/grub/grub.cfg" : \
+	command "grub-install /dev/sda --target=$grub_target --no-nvram --force-extra-removable --no-floppy --modules=part_gpt --grub-mkdevicemap=/boot/grub/device.map" : \
+	sync : umount / : shutdown
+}
+
+case "$BOOT" in
+	bios) run_mmdebstrap | guestfish_bios;;
+	efi) run_mmdebstrap | guestfish_efi;;
+	ieee1275) run_mmdebstrap | guestfish_ieee1275;;
+esac
+
+echo "Success! The image is stored as $IMAGE" >&2