Compare commits

..

No commits in common. 'f2020cf3ed5a9c634fd75a2b29407c21cd022558' and '4d76d04cfe5c4d2ff5c4a9658aa4c1988789603f' have entirely different histories.

@ -1,21 +1,3 @@
1.4.0 (2023-10-24)
------------------
- add mmdebstrap-autopkgtest-build-qemu
- export container=mmdebstrap-unshare env variable in unshare-mode hooks
- add new skip options: output/dev, output/mknod, tar-in/mknod,
copy-in/mknod, sync-in/mknod
- stop copying qemu-$arch-static binary into the chroot
- tarfilter: add --type-exclude option
- set MMDEBSTRAP_FORMAT in hooks
- do not install priority:required in buildd variant following debootstrap
1.3.8 (2023-08-20)
------------------
- hooks/merged-usr: implement post-merging as debootstrap does
- exclude ./lost+found from tarball
1.3.7 (2023-06-21)
------------------

@ -432,6 +432,11 @@ def main():
print(f"skipped because of {reason}:", file=sys.stderr)
for t in l:
print(f" {t}", file=sys.stderr)
if failed:
print("failed %d:" % len(failed), file=sys.stderr)
for f in failed:
print(f, file=sys.stderr)
exit(1)
if len(time_per_test) > 1:
print_time_per_test(time_per_test)
if len(acc_time_per_test) > 1:
@ -442,15 +447,6 @@ def main():
},
"accumulated test",
)
if failed:
print("failed %d:" % len(failed), file=sys.stderr)
for f in failed:
print(f, file=sys.stderr)
currenttime = time.time()
walltime = timedelta(seconds=int(currenttime - starttime))
print(f"total runtime: {walltime}", file=sys.stderr)
if failed:
exit(1)
if __name__ == "__main__":

@ -95,11 +95,4 @@ END
echo
fi
# check if the wiki has to be updated with pod2markdown output
if [ "${DEBEMAIL:-}" = "josch@debian.org" ]; then
bash -exc "diff -u <(curl --silent https://gitlab.mister-muffin.de/josch/mmdebstrap/wiki/raw/Home | dos2unix) <(pod2markdown < mmdebstrap)" || :
fi
rm -f shared/test.sh shared/tar1.txt shared/tar2.txt shared/pkglist.txt shared/doc-debian.tar.list shared/mmdebstrap shared/tarfilter shared/proxysolver
echo "$0 finished successfully" >&2

@ -277,14 +277,12 @@ Variants: - standard
Skip-If:
variant == "-" and hostarch not in ["armel", "armhf", "mipsel"] # #1031276
variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
Test: debug
Variants: - standard
Skip-If:
variant == "-" and hostarch not in ["armel", "armhf", "mipsel"] # #1031276
variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
Test: quiet
Needs-Root: true
@ -418,18 +416,7 @@ Test: empty-sources.list
Test: merged-fakechroot-inside-unmerged-chroot
Needs-Root: true
Needs-APT-Config: true
Skip-If:
hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
dist in ["testing", "unstable"] # #1053671
Skip-If: hostarch in ["i386", "armel", "armhf", "mipsel"] # #1023286
Test: auto-mode-as-normal-user
Modes: auto
Test: skip-output-dev
Modes: root unshare
Test: skip-output-mknod
Modes: root unshare
Test: skip-tar-in-mknod
Modes: unshare

@ -29,13 +29,10 @@ for f in "$SOURCELIST" \
"$PREFERENCES" \
"$PREFERENCESPARTS"/*; do
[ -e "$f" ] || continue
mkdir --parents "$(dirname "$rootdir/$f")"
if [ -e "$rootdir/$f" ]; then
if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 2 ]; then
echo "I: $f already exists in chroot, appending..." >&2
fi
# Add extra newline between old content and new content.
# This is required in case of deb822 files.
echo >> "$rootdir/$f"
fi
cat "$f" >> "$rootdir/$f"

@ -26,7 +26,7 @@ esac
# 2. using ./hooks
# 3. using /usr/share/mmdebstrap/hooks/
for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/extract00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
"$p/merged-usr/essential00.sh" "$1"
exit 0
fi

@ -1,27 +0,0 @@
#!/bin/sh
set -eu
env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-get update --error-on=any
# if the usr-is-merged package cannot be installed with apt, do nothing
if ! env --chdir="$1" APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-cache show --no-all-versions usr-is-merged > /dev/null 2>&1; then
echo "no package called usr-is-merged found -- not running merged-usr extract hook" >&2
exit 0
else
echo "package usr-is-merged found -- running merged-usr extract hook" >&2
fi
# resolve the script path using several methods in order:
# 1. using dirname -- "$0"
# 2. using ./hooks
# 3. using /usr/share/mmdebstrap/hooks/
for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/extract00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
"$p/merged-usr/extract00.sh" "$1"
exit 0
fi
done
echo "cannot find merged-usr hook anywhere" >&2
exit 1

@ -17,7 +17,7 @@ fi
# 2. using ./hooks
# 3. using /usr/share/mmdebstrap/hooks/
for p in "$(dirname -- "$0")/.." ./hooks /usr/share/mmdebstrap/hooks; do
if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/extract00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
if [ -x "$p/merged-usr/setup00.sh" ] && [ -x "$p/merged-usr/essential00.sh" ]; then
"$p/merged-usr/setup00.sh" "$1"
exit 0
fi

@ -10,7 +10,6 @@ TARGET="$1"
if [ "${MMDEBSTRAP_MODE:-}" = "chrootless" ]; then
APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install \
-oDPkg::Chroot-Directory= \
-oDPkg::Options::=--force-not-root \
-oDPkg::Options::=--force-script-chrootless \
-oDPkg::Options::=--root="$TARGET" \
@ -21,7 +20,7 @@ if [ "${MMDEBSTRAP_MODE:-}" = "chrootless" ]; then
dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
dpkg --compare-versions "1" "lt" "$(dpkg-query --showformat '${Version}\n' --show usr-is-merged)"
else
APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install usr-is-merged
APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$TARGET" usr-is-merged
chroot "$TARGET" dpkg-query --showformat '${db:Status-Status}\n' --show usr-is-merged | grep -q '^installed$'
chroot "$TARGET" dpkg-query --showformat '${Source}\n' --show usr-is-merged | grep -q '^usrmerge$'
dpkg --compare-versions "1" "lt" "$(chroot "$TARGET" dpkg-query --showformat '${Version}\n' --show usr-is-merged)"

@ -1,85 +0,0 @@
#!/bin/sh
set -eu
if [ "${MMDEBSTRAP_VERBOSITY:-1}" -ge 3 ]; then
set -x
fi
TARGET="$1"
# can_usrmerge_symlink() and can_usrmerge_symlink() are
# Copyright 2023 Helmut Grohne <helmut@subdivi.de>
# and part of the debootstrap source in /usr/share/debootstrap/functions
# https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/96
# https://bugs.debian.org/104989
can_usrmerge_symlink() {
# Absolute symlinks can be relocated without problems.
test "${2#/}" = "$2" || return 0
while :; do
if test "${2#/}" != "$2"; then
# Handle double-slashes.
set -- "$1" "${2#/}"
elif test "${2#./}" != "$2"; then
# Handle ./ inside a link target.
set -- "$1" "${2#./}"
elif test "$2" = ..; then
# A parent directory symlink is ok if it does not
# cross the top level directory.
test "${1%/*/*}" != "$1" -a -n "${1%/*/*}"
return $?
elif test "${2#../}" != "$2"; then
# Symbolic link crossing / cannot be moved safely.
# This is prohibited by Debian Policy 10.5.
test "${1%/*/*}" = "$1" -o -z "${1%/*/*}" && return 1
set -- "${1%/*}" "${2#../}"
else
# Consider the symlink ok if its target does not
# contain a parent directory. When we fail here,
# the link target is non-minimal and doesn't happen
# in the archive.
test "${2#*/../}" = "$2"
return $?
fi
done
}
merge_usr_entry() {
# shellcheck disable=SC3043
local entry canon
canon="$TARGET/usr/${1#"$TARGET/"}"
test -h "$canon" &&
error 1 USRMERGEFAIL "cannot move %s as its destination exists as a symlink" "${1#"$TARGET"}"
if ! test -e "$canon"; then
mv "$1" "$canon"
return 0
fi
test -d "$1" ||
error 1 USRMERGEFAIL "cannot move non-directory %s as its destination exists" "${1#"$TARGET"}"
test -d "$canon" ||
error 1 USRMERGEFAIL "cannot move directory %s as its destination is not a directory" "${1#"$TARGET"}"
for entry in "$1/"* "$1/."*; do
# Some shells return . and .. on dot globs.
test "${entry%/.}" != "${entry%/..}" && continue
if test -h "$entry" && ! can_usrmerge_symlink "${entry#"$TARGET"}" "$(readlink "$entry")"; then
error 1 USRMERGEFAIL "cannot move relative symlink crossing top-level directory" "${entry#"$TARGET"}"
fi
# Ignore glob match failures
if test "${entry%'/*'}" != "${entry%'/.*'}" && ! test -e "$entry"; then
continue
fi
merge_usr_entry "$entry"
done
rmdir "$1"
}
# This is list includes all possible multilib directories. It must be
# updated when new multilib directories are being added. Hopefully,
# all new architectures use multiarch instead, so we never get to
# update this.
for dir in bin lib lib32 lib64 libo32 libx32 sbin; do
test -h "$TARGET/$dir" && continue
test -e "$TARGET/$dir" || continue
merge_usr_entry "$TARGET/$dir"
ln -s "usr/$dir" "$TARGET/$dir"
done

@ -47,6 +47,40 @@ fi
TARGET="$1"
ARCH=$(dpkg --print-architecture)
eval "$(APT_CONFIG="$MMDEBSTRAP_APT_CONFIG" apt-config shell ARCH Apt::Architecture)"
if [ -e /usr/share/debootstrap/functions ]; then
# shellcheck disable=SC1091
. /usr/share/debootstrap/functions
doing_variant () { [ "$1" != "buildd" ]; }
# shellcheck disable=SC2034
MERGED_USR="yes"
setup_merged_usr
else
link_dir=""
case $ARCH in
hurd-*) exit 0;;
amd64) link_dir="lib32 lib64 libx32" ;;
i386) link_dir="lib64 libx32" ;;
mips|mipsel) link_dir="lib32 lib64" ;;
mips64*|mipsn32*) link_dir="lib32 lib64 libo32" ;;
powerpc) link_dir="lib64" ;;
ppc64) link_dir="lib32 lib64" ;;
ppc64el) link_dir="lib64" ;;
s390x) link_dir="lib32" ;;
sparc) link_dir="lib64" ;;
sparc64) link_dir="lib32 lib64" ;;
x32) link_dir="lib32 lib64 libx32" ;;
esac
link_dir="bin sbin lib $link_dir"
for dir in $link_dir; do
ln -s usr/"$dir" "$TARGET/$dir"
mkdir -p "$TARGET/usr/$dir"
done
fi
# now install an empty "usr-is-merged" package to avoid installing the
# usrmerge package on this system even after init-system-helpers starts
# depending on "usrmerge | usr-is-merged".

@ -18,7 +18,6 @@ fi
TARGET="$1"
echo "Warning: starting with Debian 12 (Bookworm), systems without merged-/usr are not supported anymore" >&2
echo "Warning: starting with Debian 13 (Trixie), merged-/usr symlinks are shipped by packages in the essential-set making this hook ineffective" >&2
echo "this system will not be supported in the future" > "$TARGET/etc/unsupported-skip-usrmerge-conversion"

@ -250,17 +250,6 @@ END
trap "-" EXIT INT TERM
)
check_proxy_running() {
if timeout 1 bash -c 'exec 3<>/dev/tcp/127.0.0.1/8080 && printf "GET http://deb.debian.org/debian/dists/'"$DEFAULT_DIST"'/InRelease HTTP/1.1\nHost: deb.debian.org\n\n" >&3 && grep "Suite: '"$DEFAULT_DIST"'" <&3 >/dev/null' 2>/dev/null; then
return 0
elif timeout 1 env http_proxy="http://127.0.0.1:8080/" wget --quiet -O - "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" | grep "Suite: $DEFAULT_DIST" >/dev/null; then
return 0
elif timeout 1 curl --proxy "http://127.0.0.1:8080/" --silent "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" | grep "Suite: $DEFAULT_DIST" >/dev/null; then
return 0
fi
return 1
}
if [ -e "./shared/cache.A" ] && [ -e "./shared/cache.B" ]; then
echo "both ./shared/cache.A and ./shared/cache.B exist" >&2
echo "was a former run of the script aborted?" >&2
@ -312,9 +301,8 @@ components=main
# by default, use the mmdebstrap executable in the current directory
: "${CMD:=./mmdebstrap}"
: "${USE_HOST_APT_CONFIG:=no}"
: "${FORCE_UPDATE:=no}"
if [ "$FORCE_UPDATE" != "yes" ] && [ -e "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
if [ -e "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/InRelease")
case "$http_code" in
200) ;; # need update
@ -328,7 +316,7 @@ PROXYPID=$!
trap 'kill "$PROXYPID" || :' EXIT INT TERM
for i in $(seq 10); do
check_proxy_running && break
curl --proxy "http://127.0.0.1:8080/" --silent -o /dev/null "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" && break
sleep 1
done
if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
@ -437,7 +425,7 @@ if [ "$HAVE_QEMU" = "yes" ]; then
PROXYPID=$!
for i in $(seq 10); do
check_proxy_running && break
curl --proxy "http://127.0.0.1:8080/" --silent -o /dev/null "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" && break
sleep 1
done
if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then
@ -449,7 +437,7 @@ if [ "$HAVE_QEMU" = "yes" ]; then
tmpdir="$(mktemp -d)"
trap 'kill "$PROXYPID" || :;cleanuptmpdir; cleanup_newcachedir' EXIT INT TERM
pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,squashfs-tools-ng,genext2fs,linux-image-generic
pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,disorderfs,squashfs-tools-ng,genext2fs,linux-image-generic
if [ ! -e ./mmdebstrap ]; then
pkgs="$pkgs,mmdebstrap"
fi
@ -530,7 +518,8 @@ END
fi
# set PATH to pick up the correct mmdebstrap variant
env PATH="$(dirname "$(realpath --canonicalize-existing "$CMD")"):$PATH" \
debvm-create --skip=usrmerge --size="$DISK_SIZE" --release="$DEFAULT_DIST" \
debvm-create --skip=usrmerge --size="$DISK_SIZE" \
--release="$DEFAULT_DIST" --skip=usrmerge \
--output="$newcachedir/debian-$DEFAULT_DIST.ext4" -- \
--architectures="$arches" --include="$pkgs" \
--setup-hook='echo "Acquire::http::Proxy \"http://127.0.0.1:8080/\";" > "$1/etc/apt/apt.conf.d/00proxy"' \
@ -561,5 +550,3 @@ mv --no-target-directory ./shared/cache.tmp ./shared/cache
deletecache "$oldcachedir"
trap - EXIT INT TERM
echo "$0 finished successfully" >&2

File diff suppressed because it is too large Load Diff

@ -1,389 +0,0 @@
#!/bin/sh
# Copyright 2023 Johannes Schauer Marin Rodrigues <josch@debian.org>
# Copyright 2023 Helmut Grohne <helmut@subdivi.de>
# SPDX-License-Identifier: MIT
# We generally use single quotes to avoid variable expansion:
# shellcheck disable=SC2016
# Replacement for autopkgtest-build-qemu and vmdb2 for all architectures
# supporting EFI booting (amd64, arm64, armhf, i386, riscv64).
# For use as replacement for autopkgtest-build-qemu and vmdb2 on ppc64el which
# neither supports extlinux nor efi booting there is an unmaintained script
# which uses grub instead to boot:
#
# https://gitlab.mister-muffin.de/josch/mmdebstrap/src/commit/
# e523741610a4ed8579642bfc755956f64c847ef3/mmdebstrap-autopkgtest-build-qemu
: <<'POD2MAN'
=head1 NAME
mmdebstrap-autopkgtest-build-qemu - autopkgtest-build-qemu without vmdb2 but mmdebstrap and EFI boot
=head1 SYNOPSIS
B<mmdebstrap-autopkgtest-build-qemu> [I<OPTIONS>] B<--boot>=B<efi> I<RELEASE> I<IMAGE>
=head1 DESCRIPTION
B<mmdebstrap-autopkgtest-build-qemu> is a mostly compatible drop-in replacement
for L<autopkgtest-build-qemu(1)> with two main differences: Firstly, it uses
L<mmdebstrap(1)> instead of L<vmdb2(1)> and thus is able to create QEMU disk
images without requiring superuser privileges. Secondly, it uses
L<systemd-boot(7)> and thus only supports booting via EFI.
=head1 POSITIONAL PARAMETERS
=over 8
=item I<RELEASE>
The release to download from the I<MIRROR>. This parameter is required.
=item I<IMAGE>
The file to write, in raw format. This parameter is required.
=back
=head1 OPTIONS
=over 8
=item B<--mirror>=I<MIRROR>
Specify which distribution to install. It defaults to
http://deb.debian.org/debian (i.e. Debian), but you can pass a mirror of any
Debian derivative.
=item B<--architecture>=I<ARCHITECTURE>
Set the architecture for the virtual machine image, specified as a L<dpkg(1)>
architecture. If omitted, the host architecture is assumed.
B<--arch>=I<ARCH> is an alias for this option.
=item B<--script>=I<SCRIPT>
Specifies a user script that will be called with the root filesystem of the
image as its first parameter. This script can them make any necesssary
modifications to the root filesystem.
The script must be a POSIX shell script, and should not depend on bash-specific
features. This script will be executed inside a L<chroot(1)> call in the
virtual machine root filesystem.
=item B<--size>=I<SIZE>
Specifies the image size for the virtual machine, defaulting to 25G.
=item B<--apt-proxy>=I<PROXY>
Specify an apt proxy to use in the virtual machine. By default, if you have
an apt proxy configured on the host, the virtual machine will automatically use
this, otherwise there is no default.
=item B<--boot>=B<efi>, B<--efi>
Select the way the generated image will expect to be booted. Unless you
explicitly select --boot=efi, operation will fail.
=item B<--keyring>=I<KEYRING>
Passes an additional B<--keyring> parameter to B<mmdebstrap>.
=back
=head1 EXAMPLES
$ mmdebstrap-autopkgtest-build-qemu --boot=efi stable /path/to/debian-stable-i386.img i386
$ mmdebstrap-autopkgtest-build-qemu --boot=efi unstable /path/to/debian-unstable.img
=head1 SEE ALSO
L<autopkgtest-build-qemu(1)>, L<autopkgtest-virt-qemu(1)>, L<mmdebstrap(1)>, L<autopkgtest(1)>
=cut
POD2MAN
set -eu
die() {
echo "$*" 1>&2
exit 1
}
usage() {
die "usage: $0 [--architecture=|--apt-proxy=|--keyring=|--mirror=|--script=|--size=] --boot=efi <RELEASE> <IMAGE>"
}
usage_error() {
echo "error: $*" 1>&2
usage
}
BOOT=auto
ARCHITECTURE=$(dpkg --print-architecture)
IMAGE=
MIRROR=
KEYRING=
RELEASE=
SIZE=25G
SCRIPT=
# consumed by setup-testbed
export AUTOPKGTEST_BUILD_QEMU=1
opt_boot() {
BOOT="$1"
}
opt_architecture() {
ARCHITECTURE="$1"
}
opt_arch() {
ARCHITECTURE="$1"
}
opt_apt_proxy() {
# consumed by setup-testbed
export AUTOPKGTEST_APT_PROXY="$1"
# consumed by mmdebstrap
if test "$1" = DIRECT; then
unset http_proxy
else
export http_proxy="$1"
fi
}
opt_keyring() {
KEYRING="$1"
}
opt_mirror() {
# consumed by setup-testbed
export MIRROR="$1"
}
opt_script() {
test -f "$1" || die "passed script '$1' does not refer to a file"
SCRIPT="$1"
}
opt_size() {
SIZE="$1"
}
positional=1
positional_1() {
# consumed by setup-testbed
export RELEASE="$1"
}
positional_2() {
IMAGE="$1"
}
positional_3() { opt_mirror "$@"; }
positional_4() { opt_architecture "$@"; }
positional_5() { opt_script "$@"; }
positional_6() { opt_size "$@"; }
positional_7() {
die "too many positional options"
}
while test "$#" -gt 0; do
case "$1" in
--architecture=*|--arch=*|--boot=*|--keyring=*|--mirror=*|--script=*|--size=*)
optname="${1%%=*}"
"opt_${optname#--}" "${1#*=}"
;;
--apt-proxy=*)
opt_apt_proxy "${1#*=}"
;;
--architecture|--arch|--boot|--keyring|--mirror|--script|--size)
test "$#" -ge 2 || usage_error "missing argument for $1"
"opt_${1#--}" "$2"
shift
;;
--apt-proxy)
test "$#" -ge 2 || usage_error "missing argument for $1"
opt_apt_proxy "$2"
shift
;;
--efi)
opt_boot efi
;;
--*)
usage_error "unrecognized argument $1"
;;
*)
"positional_$positional" "$1"
positional=$((positional + 1))
;;
esac
shift
done
test -z "$RELEASE" -o -z "$IMAGE" && usage_error "missing positional arguments"
test "$BOOT" = efi ||
die "this tool does not support boot modes other than efi"
case "$ARCHITECTURE" in
amd64)
EFIIMG=bootx64.efi
;;
arm64)
EFIIMG=bootaa64.efi
;;
armhf)
EFIIMG=bootarm.efi
;;
i386)
EFIIMG=bootia32.efi
;;
riscv64)
EFIIMG=bootriscv64.efi
;;
*)
die "unsupported architecture"
;;
esac
if test "$(dpkg-query -f '${db:Status-Status}' -W binutils-multiarch)" = installed; then
GNU_PREFIX=
else
GNU_ARCHITECTURE="$(dpkg-architecture "-a$ARCHITECTURE" -qDEB_HOST_GNU_TYPE)"
GNU_PREFIX="$GNU_ARCHITECTURE-"
GNU_SUFFIX="-$(echo "$GNU_ARCHITECTURE" | tr _ -)"
test "$(dpkg-query -f '${db:Status-Status}' -W "binutils$GNU_SUFFIX")" = installed ||
die "please install binutils$GNU_SUFFIX or binutils-multiarch"
fi
for pkg in autopkgtest dosfstools e2fsprogs fdisk mount mtools passwd "systemd-boot-efi:$ARCHITECTURE" uidmap; do
test "$(dpkg-query -f '${db:Status-Status}' -W "$pkg")" = installed ||
die "please install $pkg"
done
BOOTSTUB="/usr/lib/systemd/boot/efi/linux${EFIIMG#boot}.stub"
WORKDIR=
cleanup() {
test -n "$WORKDIR" && rm -Rf "$WORKDIR"
}
trap cleanup EXIT INT TERM QUIT
WORKDIR=$(mktemp -d)
FAT_OFFSET_SECTORS=$((1024*2))
FAT_SIZE_SECTORS=$((1024*254))
# Make the image writeable to the first subgid. mmdebstrap will map this gid to
# the root group. unshare instead will map the current gid to 0 and the first
# subgid to 1. Therefore mmdebstrap will be able to write to the image.
rm -f "$IMAGE"
: >"$IMAGE"
unshare -U -r --map-groups=auto chown 0:1 "$IMAGE"
chmod 0660 "$IMAGE"
set -- \
--mode=unshare \
--variant=important \
--architecture="$ARCHITECTURE"
test "$RELEASE" = jessie &&
set -- "$@" --hook-dir=/usr/share/mmdebstrap/hooks/jessie-or-older
set -- "$@" \
"--include=init,linux-image-$ARCHITECTURE,python3" \
'--customize-hook=echo autopkgtestvm >"$1/etc/hostname"' \
'--customize-hook=echo 127.0.0.1 localhost autopkgtestvm >"$1/etc/hosts"' \
'--customize-hook=passwd --root "$1" --delete root' \
'--customize-hook=useradd --root "$1" --home-dir /home/user --create-home user' \
'--customize-hook=passwd --root "$1" --delete user' \
'--customize-hook=/usr/share/autopkgtest/setup-commands/setup-testbed'
if test -n "$SCRIPT"; then
set -- "$@" \
"--customize-hook=upload '$SCRIPT' /userscript" \
"--chrooted-customize-hook=sh /userscript" \
'--customize-hook=rm -f "$1/userscript"'
fi
EXT4_OFFSET_BYTES=$(( (FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS) * 512))
EXT4_OPTIONS="offset=$EXT4_OFFSET_BYTES,assume_storage_prezeroed=1"
set -- "$@" \
"--customize-hook=download vmlinuz '$WORKDIR/kernel'" \
"--customize-hook=download initrd.img '$WORKDIR/initrd'" \
'--customize-hook=mount --bind "$1" "$1/mnt"' \
'--customize-hook=mount --bind "$1/mnt/mnt" "$1/mnt/dev"' \
'--customize-hook=/sbin/mkfs.ext4 -d "$1/mnt" -L autopkgtestvm -E '"'$EXT4_OPTIONS' '$IMAGE' '$SIZE'" \
'--customize-hook=umount --lazy "$1/mnt"' \
"$RELEASE" \
/dev/null
test -n "$MIRROR" && set -- "$@" "$MIRROR"
test -n "$KEYRING" && set -- "$@" "--keyring=$KEYRING"
echo "mmdebstrap $*"
mmdebstrap "$@" || die "mmdebstrap failed"
unshare -U -r --map-groups=auto chown 0:0 "$IMAGE"
chmod "$(printf %o "$(( 0666 - 0$(umask) ))")" "$IMAGE"
echo "root=LABEL=autopkgtestvm rw console=ttyS0" > "$WORKDIR/cmdline"
align_size() {
echo "$(( ($1) + ($2) - 1 - (($1) + ($2) - 1) % ($2) ))"
}
alignment=$("${GNU_PREFIX}objdump" -p "$BOOTSTUB" | sed 's/^SectionAlignment\s\+\([0-9]\)/0x/;t;d')
test -z "$alignment" && die "failed to discover the alignment of the efi stub"
echo "determined efi vma alignment as $alignment"
test "$RELEASE" = jessie -a "$((alignment))" -lt "$((1024*1024))" && {
echo "increasing efi vma alignment for jessie"
alignment=$((1024*1024))
}
lastoffset=0
# shellcheck disable=SC2034 # unused variables serve documentation
lastoffset="$("${GNU_PREFIX}objdump" -h "$BOOTSTUB" |
while read -r idx name size vma lma fileoff algn behind; do
test -z "$behind" -a "${algn#"2**"}" != "$algn" || continue
offset=$(( 0x$vma + 0x$size ))
test "$offset" -gt "$lastoffset" || continue
lastoffset="$offset"
echo "$lastoffset"
done | tail -n1)"
lastoffset=$(align_size "$lastoffset" "$alignment")
echo "determined minimum efi vma offset as $lastoffset"
cmdline_size="$(stat -Lc%s "$WORKDIR/cmdline")"
cmdline_size="$(align_size "$cmdline_size" "$alignment")"
linux_size="$(stat -Lc%s "$WORKDIR/kernel")"
linux_size="$(align_size "$linux_size" "$alignment")"
cmdline_offset="$lastoffset"
linux_offset=$((cmdline_offset + cmdline_size))
initrd_offset=$((linux_offset + linux_size))
SOURCE_DATE_EPOCH=0 \
"${GNU_PREFIX}objcopy" \
--enable-deterministic-archives \
--add-section .cmdline="$WORKDIR/cmdline" \
--change-section-vma .cmdline="$(printf 0x%x "$cmdline_offset")" \
--add-section .linux="$WORKDIR/kernel" \
--change-section-vma .linux="$(printf 0x%x "$linux_offset")" \
--add-section .initrd="$WORKDIR/initrd" \
--change-section-vma .initrd="$(printf 0x%x "$initrd_offset")" \
"$BOOTSTUB" "$WORKDIR/efiimg"
rm -f "$WORKDIR/kernel" "$WORKDIR/initrd"
truncate -s "$((FAT_SIZE_SECTORS * 512))" "$WORKDIR/fat"
/sbin/mkfs.fat -F 32 --invariant "$WORKDIR/fat"
mmd -i "$WORKDIR/fat" EFI EFI/BOOT
mcopy -i "$WORKDIR/fat" "$WORKDIR/efiimg" "::EFI/BOOT/$EFIIMG"
rm -f "$WORKDIR/efiimg"
truncate --size="+$((34*512))" "$IMAGE"
/sbin/sfdisk "$IMAGE" <<EOF
label: gpt
unit: sectors
start=$FAT_OFFSET_SECTORS, size=$FAT_SIZE_SECTORS, type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B
start=$((FAT_OFFSET_SECTORS + FAT_SIZE_SECTORS)), type=0FC63DAF-8483-4772-8E79-3D69D8477DE4
EOF
dd if="$WORKDIR/fat" of="$IMAGE" conv=notrunc,sparse bs=512 "seek=$FAT_OFFSET_SECTORS" status=none

@ -43,29 +43,6 @@ class PaxFilterAction(argparse.Action):
setattr(namespace, "paxfilter", items)
class TypeFilterAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, "typefilter", [])
match values:
case "REGTYPE" | "0":
items.append(tarfile.REGTYPE)
case "LNKTYPE" | "1":
items.append(tarfile.LNKTYPE)
case "SYMTYPE" | "2":
items.append(tarfile.SYMTYPE)
case "CHRTYPE" | "3":
items.append(tarfile.CHRTYPE)
case "BLKTYPE" | "4":
items.append(tarfile.BLKTYPE)
case "DIRTYPE" | "5":
items.append(tarfile.DIRTYPE)
case "FIFOTYPE" | "6":
items.append(tarfile.FIFOTYPE)
case _:
raise ValueError("invalid type: %s" % values)
setattr(namespace, "typefilter", items)
class TransformAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, "trans", [])
@ -112,11 +89,10 @@ dpkg(1) for information on how these two options work in detail. To reuse the
exact same semantics as used by dpkg, paths must be given as /path and not as
./path even though they might be stored as such in the tarball.
Secondly, filter out unwanted pax extended headers using --pax-exclude and
--pax-include. This is useful in cases where a tool only accepts certain xattr
prefixes. For example tar2sqfs only supports SCHILY.xattr.user.*,
SCHILY.xattr.trusted.* and SCHILY.xattr.security.* but not
SCHILY.xattr.system.posix_acl_default.*.
Secondly, filter out unwanted pax extended headers. This is useful in cases
where a tool only accepts certain xattr prefixes. For example tar2sqfs only
supports SCHILY.xattr.user.*, SCHILY.xattr.trusted.* and
SCHILY.xattr.security.* but not SCHILY.xattr.system.posix_acl_default.*.
Both types of options use Unix shell-style wildcards:
@ -125,16 +101,10 @@ Both types of options use Unix shell-style wildcards:
[seq] matches any character in seq
[!seq] matches any character not in seq
Thirdly, filter out files matching a specific tar archive member type using
--type-exclude. Valid type names are REGTYPE (regular file), LNKTYPE
(hardlink), SYMTYPE (symlink), CHRTYPE (character special), BLKTYPE (block
special), DIRTYPE (directory), FIFOTYPE (fifo) or their tar format flag value
(0-6, respectively).
Fourthly, transform the path of tar members using a sed expression just as with
Thirdly, transform the path of tar members using a sed expression just as with
GNU tar --transform.
Fifthly, strip leading directory components off of tar members. Just as with
Fourthly, strip leading directory components off of tar members. Just as with
GNU tar --strip-components, tar members that have less or equal components in
their path are not passed through.
@ -170,15 +140,6 @@ Lastly, shift user id and group id of each entry by the value given by the
help="Re-include a pax header after a previous exclusion. "
"This option can be specified multiple times.",
)
parser.add_argument(
"--type-exclude",
metavar="type",
action=TypeFilterAction,
help="Exclude certain member types by their type. Choose types either "
"by their name (REGTYPE, LNKTYPE, SYMTYPE, CHRTYPE, BLKTYPE, DIRTYPE, "
"FIFOTYPE) or by their tar format flag values (0-6, respectively). "
"This option can be specified multiple times.",
)
parser.add_argument(
"--transform",
"--xform",
@ -203,7 +164,6 @@ Lastly, shift user id and group id of each entry by the value given by the
if (
not hasattr(args, "pathfilter")
and not hasattr(args, "paxfilter")
and not hasattr(args, "typefilter")
and not hasattr(args, "strip_components")
):
from shutil import copyfileobj
@ -247,24 +207,14 @@ Lastly, shift user id and group id of each entry by the value given by the
skip = True
return skip
def type_filter_should_skip(member):
if not hasattr(args, "typefilter"):
return False
for t in args.typefilter:
if member.type == t:
return True
return False
# starting with Python 3.8, the default format became PAX_FORMAT but we
# are still explicit here in case of future changes.
# starting with Python 3.8, the default format became PAX_FORMAT, so this
# is only for compatibility with older versions of Python 3
with tarfile.open(fileobj=sys.stdin.buffer, mode="r|*") as in_tar, tarfile.open(
fileobj=sys.stdout.buffer, mode="w|", format=tarfile.PAX_FORMAT
) as out_tar:
for member in in_tar:
if path_filter_should_skip(member):
continue
if type_filter_should_skip(member):
continue
if args.strip_components:
comps = member.name.split("/")
# just as with GNU tar, archive members with less or equal

@ -35,8 +35,15 @@ if [ -n "$AUTOPROXY" ] && [ -x "$AUTOPROXY" ] && [ -e /tmp/.auto-apt-proxy-0 ];
chmod 644 "$TMP_APT_CONFIG"
fi
# debootstrap runs mount -t proc proc /proc which doesn't work in an unshared
# namespace on privileged docker (like salsaci), so mount /proc manually
# https://bugs.debian.org/1031222
# https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/91
$prefix {{ CMD }} --variant=custom --mode={{ MODE }} \
--setup-hook='env '"${AUTOPROXY:+APT_CONFIG='$TMP_APT_CONFIG'}"' debootstrap --variant={{ VARIANT }} unstable "$1" {{ MIRROR }}' \
--setup-hook='env '"${AUTOPROXY:+APT_CONFIG='$TMP_APT_CONFIG'}"' container=lxc debootstrap --variant={{ VARIANT }} unstable "$1" {{ MIRROR }}' \
--setup-hook='mount -o rbind /proc "$1/proc"' \
--setup-hook='chroot "$1" dpkg-reconfigure systemd || true' \
--setup-hook='umount --lazy "$1/proc"' \
- /tmp/debian-mm.tar {{ MIRROR }}
if [ -n "$AUTOPROXY" ] && [ -x "$AUTOPROXY" ] && [ -e /tmp/.auto-apt-proxy-0 ]; then
rm "$TMP_APT_CONFIG"

@ -13,7 +13,9 @@ rmdir /etc/apt/trusted.gpg.d
mkdir /etc/apt/trusted.gpg.d
for f in /usr/share/keyrings/*.gpg; do
name=$(basename "$f" .gpg)
gpg --no-default-keyring --keyring="/usr/share/keyrings/$name.gpg" --armor --output="/etc/apt/trusted.gpg.d/$name.asc" --export
gpg --enarmor < "/usr/share/keyrings/$name.gpg" \
| sed 's/ PGP ARMORED FILE/ PGP PUBLIC KEY BLOCK/;/^Comment: /d' \
> "/etc/apt/trusted.gpg.d/$name.asc"
rm "/usr/share/keyrings/$name.gpg"
done
{{ CMD }} --mode=root --variant=apt {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}

@ -174,13 +174,6 @@ if [ "{{ VARIANT }}" = "-" ]; then
done
fi
# since debootstrap 1.0.133 there is no tzdata in the buildd variant and thus
# debootstrap creates its own /etc/localtime
if [ "{{ VARIANT }}" = "buildd" ]; then
[ "$(readlink /tmp/debian-{{ DIST }}-debootstrap/etc/localtime)" = /usr/share/zoneinfo/UTC ]
rm /tmp/debian-{{ DIST }}-debootstrap/etc/localtime
fi
# check if the file content differs
diff --unified --no-dereference --recursive /tmp/debian-{{ DIST }}-debootstrap /tmp/debian-{{ DIST }}-mm >&2

@ -4,6 +4,6 @@ export LC_ALL=C.UTF-8
trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
{{ CMD }} --mode={{ MODE }} --variant=essential --include=apt \
--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get update' \
--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install apt' \
--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="$1" apt' \
{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -

@ -1,35 +0,0 @@
#!/bin/sh
set -eu
export LC_ALL=C.UTF-8
prefix=
if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
if [ ! -e /mmdebstrap-testenv ]; then
echo "this test modifies the system and should only be run inside a container" >&2
exit 1
fi
useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
fi
prefix="runuser -u ${SUDO_USER:-user} --"
fi
# test this for both unshare and root mode because the code paths creating
# entries in /dev are different depending on whether mknod is available or not
$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --skip=output/dev {{ DIST }} - {{ MIRROR }} | {
tar -t;
echo ./dev/console;
echo ./dev/fd;
echo ./dev/full;
echo ./dev/null;
echo ./dev/ptmx;
echo ./dev/pts/;
echo ./dev/random;
echo ./dev/shm/;
echo ./dev/stderr;
echo ./dev/stdin;
echo ./dev/stdout;
echo ./dev/tty;
echo ./dev/urandom;
echo ./dev/zero;
} | sort | diff -u tar1.txt -

@ -1,30 +0,0 @@
#!/bin/sh
set -eu
export LC_ALL=C.UTF-8
prefix=
if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
if [ ! -e /mmdebstrap-testenv ]; then
echo "this test modifies the system and should only be run inside a container" >&2
exit 1
fi
useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
fi
prefix="runuser -u ${SUDO_USER:-user} --"
fi
# test this for both unshare and root mode because the code paths creating
# entries in /dev are different depending on whether mknod is available or not
$prefix {{ CMD }} --mode={{ MODE }} --variant=apt --skip=output/mknod \
{{ DIST }} - {{ MIRROR }} | {
tar -t;
echo ./dev/console;
echo ./dev/full;
echo ./dev/null;
echo ./dev/ptmx;
echo ./dev/random;
echo ./dev/tty;
echo ./dev/urandom;
echo ./dev/zero;
} | sort | diff -u tar1.txt -

@ -1,28 +0,0 @@
#!/bin/sh
set -eu
export LC_ALL=C.UTF-8
export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
[ {{ MODE }} = "unshare" ]
trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
prefix=
if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
if [ ! -e /mmdebstrap-testenv ]; then
echo "this test modifies the system and should only be run inside a container" >&2
exit 1
fi
useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
fi
prefix="runuser -u ${SUDO_USER:-user} --"
fi
$prefix {{ CMD }} --mode={{ MODE }} --variant=custom \
--skip=update,setup,cleanup,tar-in/mknod \
--setup-hook='tar-in ./cache/mmdebstrap-{{ DIST }}-apt.tar /' \
'' /tmp/debian-chroot.tar
cmp ./cache/mmdebstrap-{{ DIST }}-apt.tar /tmp/debian-chroot.tar \
|| diffoscope ./cache/mmdebstrap-{{ DIST }}-apt.tar /tmp/debian-chroot.tar

@ -8,7 +8,7 @@ mkdir /tmp/root/real
run_testA() {
echo content > /tmp/foo
# shellcheck disable=SC2094
{ { { {{ CMD }} --hook-helper /tmp/root root setup '' 1 upload /tmp/foo "$1" < /tmp/myfifo 3>&-; echo $? >&3; printf "\\000\\000adios";
{ { { {{ CMD }} --hook-helper /tmp/root root setup env 1 upload /tmp/foo "$1" < /tmp/myfifo 3>&-; echo $? >&3; printf "\\000\\000adios";
} | {{ CMD }} --hook-listener 1 3>&- >/tmp/myfifo; echo $?; } 3>&1;
} | { read -r xs1; [ "$xs1" -eq 0 ]; read -r xs2; [ "$xs2" -eq 0 ]; }
echo content | diff -u - /tmp/root/real/foo

@ -8,12 +8,12 @@ echo "MMDEBSTRAP_APT_CONFIG $MMDEBSTRAP_APT_CONFIG"
echo "$MMDEBSTRAP_HOOK" >> /tmp/hooks
[ "$MMDEBSTRAP_MODE" = "root" ]
echo test-content $MMDEBSTRAP_HOOK > test
{{ CMD }} --hook-helper "$1" "$MMDEBSTRAP_MODE" "$MMDEBSTRAP_HOOK" '' 1 upload test /test <&$MMDEBSTRAP_HOOKSOCK >&$MMDEBSTRAP_HOOKSOCK
{{ CMD }} --hook-helper "$1" "$MMDEBSTRAP_MODE" "$MMDEBSTRAP_HOOK" env 1 upload test /test <&$MMDEBSTRAP_HOOKSOCK >&$MMDEBSTRAP_HOOKSOCK
rm test
echo "content inside chroot:"
cat "$1/test"
[ "test-content $MMDEBSTRAP_HOOK" = "$(cat "$1/test")" ]
{{ CMD }} --hook-helper "$1" "$MMDEBSTRAP_MODE" "$MMDEBSTRAP_HOOK" '' 1 download /test test <&$MMDEBSTRAP_HOOKSOCK >&$MMDEBSTRAP_HOOKSOCK
{{ CMD }} --hook-helper "$1" "$MMDEBSTRAP_MODE" "$MMDEBSTRAP_HOOK" env 1 download /test test <&$MMDEBSTRAP_HOOKSOCK >&$MMDEBSTRAP_HOOKSOCK
echo "content outside chroot:"
cat test
[ "test-content $MMDEBSTRAP_HOOK" = "$(cat test)" ]

Loading…
Cancel
Save