mmdebstrap/mmdebstrap-autopkgtest-build-qemu

461 lines
14 KiB
Text
Raw Permalink Normal View History

2023-09-17 05:48:14 +00:00
#!/bin/sh
# Copyright 2023 Johannes Schauer Marin Rodrigues <josch@debian.org>
# Copyright 2023 Helmut Grohne <helmut@subdivi.de>
# SPDX-License-Identifier: MIT
# We generally use single quotes to avoid variable expansion:
# shellcheck disable=SC2016
# Replacement for autopkgtest-build-qemu and vmdb2 for all architectures
# supporting EFI booting (amd64, arm64, armhf, i386, riscv64).
# For use as replacement for autopkgtest-build-qemu and vmdb2 on ppc64el which
# neither supports extlinux nor efi booting there is an unmaintained script
# which uses grub instead to boot:
#
# https://gitlab.mister-muffin.de/josch/mmdebstrap/src/commit/
# e523741610a4ed8579642bfc755956f64c847ef3/mmdebstrap-autopkgtest-build-qemu
: <<'POD2MAN'
=head1 NAME
mmdebstrap-autopkgtest-build-qemu - autopkgtest-build-qemu without vmdb2 but mmdebstrap and EFI boot
=head1 SYNOPSIS
B<mmdebstrap-autopkgtest-build-qemu> [I<OPTIONS>] B<--boot>=B<efi> I<RELEASE> I<IMAGE>
=head1 DESCRIPTION
B<mmdebstrap-autopkgtest-build-qemu> is a mostly compatible drop-in replacement
2023-10-28 13:24:51 +00:00
for L<autopkgtest-build-qemu(1)> with two main differences: Firstly, it uses
L<mmdebstrap(1)> instead of L<vmdb2(1)> and thus is able to create QEMU disk
images without requiring superuser privileges and with bit-by-bit reproducible
output. Secondly, it uses L<systemd-boot(7)> and thus only supports booting via
EFI. For architectures for which L<autopkgtest-virt-qemu(1)> does not default
to EFI booting you must pass B<--boot=efi> when invoking the autopkgtest virt
backend.
=head1 POSITIONAL PARAMETERS
=over 8
=item I<RELEASE>
The release to download from the I<MIRROR>. This parameter is required.
=item I<IMAGE>
The file to write, in raw format. This parameter is required.
=back
=head1 OPTIONS
=over 8
=item B<--mirror>=I<MIRROR>
Specify which distribution to install. It defaults to
http://deb.debian.org/debian (i.e. Debian), but you can pass a mirror of any
Debian derivative.
=item B<--architecture>=I<ARCHITECTURE>
2023-10-28 13:24:51 +00:00
Set the architecture for the virtual machine image, specified as a L<dpkg(1)>
architecture. If omitted, the host architecture is assumed.
B<--arch>=I<ARCH> is an alias for this option.
=item B<--script>=I<SCRIPT>
Specifies a user script that will be called with the root filesystem of the
image as its first parameter. This script can them make any necesssary
modifications to the root filesystem.
The script must be a POSIX shell script, and should not depend on bash-specific
2023-10-28 13:24:51 +00:00
features. This script will be executed inside a L<chroot(1)> call in the
virtual machine root filesystem.
=item B<--size>=I<SIZE>
Specifies the image size for the virtual machine, defaulting to 25G.
=item B<--apt-proxy>=I<PROXY>
Specify an apt proxy to use in the virtual machine. By default, if you have
an apt proxy configured on the host, the virtual machine will automatically use
this, otherwise there is no default.
=item B<--boot>=B<efi>, B<--efi>
Select the way the generated image will expect to be booted. Unless you
explicitly select --boot=efi, operation will fail.
=item B<--keyring>=I<KEYRING>
Passes an additional B<--keyring> parameter to B<mmdebstrap>.
=back
=head1 EXAMPLES
Make sure, that F</path/to/debian-unstable.img> is a path that the unshared
user has access to. This can be done by ensuring world-execute permissions on
all path components or by creating the image in a world-readable directory like
/tmp before copying it into its final location.
$ mmdebstrap-autopkgtest-build-qemu --boot=efi --arch=amd64 unstable /path/to/debian-unstable.img
[...]
$ autopkgtest mypackage -- qemu --boot=efi --dpkg-architecture=amd64 /path/to/debian-unstable.img
Make sure to add B<--boot=efi> to both the B<mmdebstrap-autopkgtest-build-qemu>
as well as the B<autopkgtest-virt-qemu> invocation.
Create bit-by-bit reproducible images from a given snapshot.d.o timestamp.
SOURCE_DATE_EPOCH=1612543740 mmdebstrap-autopkgtest-build-qemu --boot=efi \
--mirror=http://snapshot.debian.org/archive/debian/20210205T164900Z/ \
unstable /path/to/debian-unstable.img
=head1 SEE ALSO
2023-10-28 13:24:51 +00:00
L<autopkgtest-build-qemu(1)>, L<autopkgtest-virt-qemu(1)>, L<mmdebstrap(1)>, L<autopkgtest(1)>
=cut
POD2MAN
2023-09-17 05:48:14 +00:00
set -eu
die() {
echo "$*" 1>&2
exit 1
2023-09-17 05:48:14 +00:00
}
usage() {
die "usage: $0 [--architecture=|--apt-proxy=|--keyring=|--mirror=|--script=|--size=] --boot=efi <RELEASE> <IMAGE>"
2023-09-17 05:48:14 +00:00
}
usage_error() {
echo "error: $*" 1>&2
usage
2023-09-17 05:48:14 +00:00
}
BOOT=auto
ARCHITECTURE=$(dpkg --print-architecture)
IMAGE=
MIRROR=
KEYRING=
RELEASE=
SIZE=25G
SCRIPT=
# consumed by setup-testbed
export AUTOPKGTEST_BUILD_QEMU=1
opt_boot() {
BOOT="$1"
2023-09-17 05:48:14 +00:00
}
opt_architecture() {
ARCHITECTURE="$1"
2023-09-17 05:48:14 +00:00
}
opt_arch() {
ARCHITECTURE="$1"
}
2023-09-17 05:48:14 +00:00
opt_apt_proxy() {
# consumed by setup-testbed
export AUTOPKGTEST_APT_PROXY="$1"
# consumed by mmdebstrap
if test "$1" = DIRECT; then
unset http_proxy
else
export http_proxy="$1"
fi
2023-09-17 05:48:14 +00:00
}
opt_keyring() {
KEYRING="$1"
2023-09-17 05:48:14 +00:00
}
opt_mirror() {
# consumed by setup-testbed
export MIRROR="$1"
2023-09-17 05:48:14 +00:00
}
opt_script() {
test -f "$1" || die "passed script '$1' does not refer to a file"
SCRIPT="$1"
2023-09-17 05:48:14 +00:00
}
opt_size() {
SIZE="$1"
2023-09-17 05:48:14 +00:00
}
positional=1
positional_1() {
# consumed by setup-testbed
export RELEASE="$1"
2023-09-17 05:48:14 +00:00
}
positional_2() {
IMAGE="$1"
2023-09-17 05:48:14 +00:00
}
positional_3() { opt_mirror "$@"; }
positional_4() { opt_architecture "$@"; }
positional_5() { opt_script "$@"; }
positional_6() { opt_size "$@"; }
positional_7() {
die "too many positional options"
2023-09-17 05:48:14 +00:00
}
while test "$#" -gt 0; do
case "$1" in
--architecture=* | --arch=* | --boot=* | --keyring=* | --mirror=* | --script=* | --size=*)
optname="${1%%=*}"
"opt_${optname#--}" "${1#*=}"
;;
--apt-proxy=*)
opt_apt_proxy "${1#*=}"
;;
--architecture | --arch | --boot | --keyring | --mirror | --script | --size)
test "$#" -ge 2 || usage_error "missing argument for $1"
"opt_${1#--}" "$2"
shift
;;
--apt-proxy)
test "$#" -ge 2 || usage_error "missing argument for $1"
opt_apt_proxy "$2"
shift
;;
--efi)
opt_boot efi
;;
--*)
usage_error "unrecognized argument $1"
;;
*)
"positional_$positional" "$1"
positional=$((positional + 1))
;;
esac
shift
2023-09-17 05:48:14 +00:00
done
test -z "$RELEASE" -o -z "$IMAGE" && usage_error "missing positional arguments"
test "$BOOT" = efi \
|| die "this tool does not support boot modes other than efi"
2023-09-17 05:48:14 +00:00
case "$ARCHITECTURE" in
amd64)
EFIIMG=bootx64.efi
QEMUARCH=x86_64
VMFPKG=ovmf
LINUXIMAGE=linux-image-amd64
;;
arm64)
EFIIMG=bootaa64.efi
QEMUARCH=aarch64
VMFPKG=qemu-efi-aarch64
LINUXIMAGE=linux-image-arm64
;;
armhf)
EFIIMG=bootarm.efi
QEMUARCH=arm
VMFPKG=qemu-efi-arm
LINUXIMAGE=linux-image-armmp
;;
i386)
EFIIMG=bootia32.efi
QEMUARCH=i386
VMFPKG=ovmf-ia32
LINUXIMAGE=linux-image-686-pae
;;
riscv64)
EFIIMG=bootriscv64.efi
QEMUARCH=riscv64
VMFPKG=
LINUXIMAGE=linux-image-riscv64
;;
*)
die "unsupported architecture: $ARCHITECTURE"
;;
2023-09-17 05:48:14 +00:00
esac
if test "$(dpkg-query -f '${db:Status-Status}' -W binutils-multiarch)" = installed; then
GNU_PREFIX=
BINUTILS=
else
GNU_ARCHITECTURE="$(dpkg-architecture "-a$ARCHITECTURE" -qDEB_HOST_GNU_TYPE)"
GNU_PREFIX="$GNU_ARCHITECTURE-"
GNU_SUFFIX="-$(echo "$GNU_ARCHITECTURE" | tr _ -)"
BINUTILS=", binutils$GNU_SUFFIX | binutils-multiarch"
fi
arches=" $(dpkg --print-architecture) $(dpkg --print-foreign-architectures | tr '\n' ' ') "
case $arches in
*" $ARCHITECTURE "*) : ;; # nothing to do
*) die "enable $ARCHITECTURE by running: sudo dpkg --add-architecture $ARCHITECTURE && sudo apt update" ;;
esac
test "$(dpkg-query -f '${db:Status-Status}' -W "dpkg-dev")" = installed \
|| die "please install dpkg-dev"
dpkg-checkbuilddeps -d "autopkgtest, dosfstools, e2fsprogs, fdisk, mount, mtools, passwd, uidmap, libarchive13, systemd-boot-efi:$ARCHITECTURE $BINUTILS" /dev/null \
|| die "please install the required packages listed above"
2023-09-17 05:48:14 +00:00
BOOTSTUB="/usr/lib/systemd/boot/efi/linux${EFIIMG#boot}.stub"
WORKDIR=
cleanup() {
test -n "$WORKDIR" && rm -Rf "$WORKDIR"
2023-09-17 05:48:14 +00:00
}
trap cleanup EXIT INT TERM QUIT
WORKDIR=$(mktemp -d)
FAT_OFFSET_SECTORS=$((1024 * 2))
FAT_SIZE_SECTORS=$((1024 * 254))
2023-09-17 05:48:14 +00:00
# The image is raw and not in qcow2 format because:
# - faster run-time as the "qemu-image convert" step is not needed
# - image can be used independent of qemu tooling
# - modifying the image just with "mount" instead of requiring qemu-nbd
# - sparse images make the file just as small as with qcow2
# - trim support is more difficult on qcow2
# - snapshots and overlays work just as well with raw images
# - users who prefer qcow2 get to choose to run it themselves with their own
# custom options like compression
2023-09-17 05:48:14 +00:00
set -- \
--mode=unshare \
--format=tar \
--variant=important \
--architecture="$ARCHITECTURE"
case $MIRROR in http://snapshot.debian.org/archive/* | https://snapshot.debian.org/archive/*)
set -- "$@" --aptopt='Acquire::Check-Valid-Until "false"'
;;
esac
EXT_FEATURES=
if test "$RELEASE" = jessie; then
set -- "$@" --keyring=/usr/share/keyrings/debian-archive-removed-keys.gpg
set -- "$@" --aptopt='Apt::Key::gpgvcommand "/usr/libexec/mmdebstrap/gpgvnoexpkeysig"'
set -- "$@" --hook-dir=/usr/share/mmdebstrap/hooks/jessie-or-older
EXT_FEATURES="^metadata_csum,^metadata_csum_seed,^orphan_file"
fi
2023-09-17 05:48:14 +00:00
set -- "$@" \
"--include=init,$LINUXIMAGE,python3" \
'--customize-hook=echo host >"$1/etc/hostname"' \
'--customize-hook=echo 127.0.0.1 localhost host >"$1/etc/hosts"' \
'--customize-hook=passwd --root "$1" --delete root' \
'--customize-hook=useradd --root "$1" --home-dir /home/user --create-home user' \
'--customize-hook=passwd --root "$1" --delete user' \
'--customize-hook=/usr/share/autopkgtest/setup-commands/setup-testbed'
2023-09-17 05:48:14 +00:00
if test -n "$SCRIPT"; then
set -- "$@" \
"--customize-hook=upload '$SCRIPT' /userscript" \
"--chrooted-customize-hook=sh /userscript" \
'--customize-hook=rm -f "$1/userscript"'
fi
2023-09-17 05:48:14 +00:00
set -- "$@" \
"--customize-hook=download vmlinuz '$WORKDIR/kernel'" \
"--customize-hook=download initrd.img '$WORKDIR/initrd'" \
"$RELEASE" \
-
2023-09-17 05:48:14 +00:00
test -n "$MIRROR" && set -- "$@" "$MIRROR"
test -n "$KEYRING" && set -- "$@" "--keyring=$KEYRING"
echo "+ mmdebstrap $*" >&2
# https://github.com/koalaman/shellcheck/issues/2555
# shellcheck disable=SC3040
set -o pipefail
mmdebstrap "$@" | {
set -- -t ext4 -L autopkgtestvm -d -
if test -n "$EXT_FEATURES"; then
set -- "$@" -O "$EXT_FEATURES"
fi
EXTOPTS="offset=$(((FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS) * 512))"
if test -n "${SOURCE_DATE_EPOCH-}"; then
uuid="$(uuidgen --sha1 --namespace="$(uuidgen --sha1 --namespace='@dns' --name mister-muffin.de)" --name "$SOURCE_DATE_EPOCH")"
set -- "$@" -U "$uuid"
EXTOPTS="$EXTOPTS,hash_seed=$uuid"
fi
set -- "$@" -E "$EXTOPTS" "$IMAGE" "$SIZE"
echo "+ mke2fs $*" >&2
/sbin/mke2fs "$@"
}
2023-09-17 05:48:14 +00:00
echo "root=LABEL=autopkgtestvm rw console=ttyS0" >"$WORKDIR/cmdline"
2023-09-17 05:48:14 +00:00
align_size() {
echo "$((($1) + ($2) - 1 - (($1) + ($2) - 1) % ($2)))"
2023-09-17 05:48:14 +00:00
}
alignment=$("${GNU_PREFIX}objdump" -p "$BOOTSTUB" | sed 's/^SectionAlignment\s\+\([0-9]\)/0x/;t;d')
2023-09-17 05:48:14 +00:00
test -z "$alignment" && die "failed to discover the alignment of the efi stub"
echo "determined efi vma alignment as $alignment"
test "$RELEASE" = jessie -a "$((alignment))" -lt "$((1024 * 1024))" && {
echo "increasing efi vma alignment for jessie"
alignment=$((1024 * 1024))
2023-09-17 05:48:14 +00:00
}
lastoffset=0
# shellcheck disable=SC2034 # unused variables serve documentation
lastoffset="$("${GNU_PREFIX}objdump" -h "$BOOTSTUB" \
| while read -r idx name size vma lma fileoff algn behind; do
test -z "$behind" -a "${algn#"2**"}" != "$algn" || continue
offset=$((0x$vma + 0x$size))
test "$offset" -gt "$lastoffset" || continue
lastoffset="$offset"
echo "$lastoffset"
done | tail -n1)"
2023-09-17 05:48:14 +00:00
lastoffset=$(align_size "$lastoffset" "$alignment")
echo "determined minimum efi vma offset as $lastoffset"
cmdline_size="$(stat -Lc%s "$WORKDIR/cmdline")"
cmdline_size="$(align_size "$cmdline_size" "$alignment")"
linux_size="$(stat -Lc%s "$WORKDIR/kernel")"
linux_size="$(align_size "$linux_size" "$alignment")"
cmdline_offset="$lastoffset"
linux_offset=$((cmdline_offset + cmdline_size))
initrd_offset=$((linux_offset + linux_size))
SOURCE_DATE_EPOCH=0 \
"${GNU_PREFIX}objcopy" \
--enable-deterministic-archives \
--add-section .cmdline="$WORKDIR/cmdline" \
--change-section-vma .cmdline="$(printf 0x%x "$cmdline_offset")" \
--add-section .linux="$WORKDIR/kernel" \
--change-section-vma .linux="$(printf 0x%x "$linux_offset")" \
--add-section .initrd="$WORKDIR/initrd" \
--change-section-vma .initrd="$(printf 0x%x "$initrd_offset")" \
"$BOOTSTUB" "$WORKDIR/efiimg"
2023-09-17 05:48:14 +00:00
rm -f "$WORKDIR/kernel" "$WORKDIR/initrd"
truncate -s "$((FAT_SIZE_SECTORS * 512))" "$WORKDIR/fat"
/sbin/mkfs.fat -F 32 --invariant "$WORKDIR/fat"
mmd -i "$WORKDIR/fat" EFI EFI/BOOT
mcopy -i "$WORKDIR/fat" "$WORKDIR/efiimg" "::EFI/BOOT/$EFIIMG"
rm -f "$WORKDIR/efiimg"
truncate --size="+$((34 * 512))" "$IMAGE"
2023-09-17 05:48:14 +00:00
/sbin/sfdisk "$IMAGE" <<EOF
label: gpt
unit: sectors
start=$FAT_OFFSET_SECTORS, size=$FAT_SIZE_SECTORS, type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B
start=$((FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS)), type=0FC63DAF-8483-4772-8E79-3D69D8477DE4
EOF
dd if="$WORKDIR/fat" of="$IMAGE" conv=notrunc,sparse bs=512 "seek=$FAT_OFFSET_SECTORS" status=none
if test "$(dpkg --print-architecture)" != "$ARCHITECTURE" && test "$(dpkg-query -f '${db:Status-Status}' -W "qemu-system-$QEMUARCH")" != installed; then
echo "I: you might need to install a package providing qemu-system-$QEMUARCH to use this image with autopkgtest-virt-qemu" >&2
fi
if test -n "$VMFPKG" && test "$(dpkg-query -f '${db:Status-Status}' -W "$VMFPKG")" != installed; then
echo "I: you might need to install $VMFPKG to use this image with autopkgtest-virt-qemu" >&2
fi
echo "I: SUCCESS! Your new image can be found here: $IMAGE" >&2
echo "I: Don't forget to pass --boot=efi when running autopkgtest-virt-qemu with this image" >&2