#!/bin/sh set -eu # This script fills either cache.A or cache.B with new content and then # atomically switches the cache symlink from one to the other at the end. # This way, at no point will the cache be in an non-working state, even # when this script got canceled at any point. # Working with two directories also automatically prunes old packages in # the local repository. deletecache() { dir="$1" echo "running deletecache $dir">&2 if [ ! -e "$dir" ]; then return fi if [ ! -e "$dir/mmdebstrapcache" ]; then echo "$dir cannot be the mmdebstrap cache" >&2 return 1 fi # be very careful with removing the old directory # experimental is pulled in with USE_HOST_APT_CONFIG=yes on debci # when testing a package from experimental for dist in oldstable stable testing unstable experimental; do # deleting artifacts from test "debootstrap" for variant in minbase buildd -; do if [ -e "$dir/debian-$dist-$variant.tar" ]; then rm "$dir/debian-$dist-$variant.tar" else echo "does not exist: $dir/debian-$dist-$variant.tar" >&2 fi done # deleting artifacts from test "mmdebstrap" for variant in essential apt minbase buildd - standard; do for format in tar ext2 squashfs; do if [ -e "$dir/mmdebstrap-$dist-$variant.$format" ]; then # attempt to delete for all dists because DEFAULT_DIST might've been different the last time rm "$dir/mmdebstrap-$dist-$variant.$format" elif [ "$dist" = "$DEFAULT_DIST" ]; then # only warn about non-existance when it's expected to exist echo "does not exist: $dir/mmdebstrap-$dist-$variant.$format" >&2 fi done done if [ -e "$dir/debian/dists/$dist" ]; then rm --one-file-system --recursive "$dir/debian/dists/$dist" else echo "does not exist: $dir/debian/dists/$dist" >&2 fi case "$dist" in oldstable|stable) if [ -e "$dir/debian/dists/$dist-updates" ]; then rm --one-file-system --recursive "$dir/debian/dists/$dist-updates" else echo "does not exist: $dir/debian/dists/$dist-updates" >&2 fi ;; esac case "$dist" in oldstable|stable) if [ -e "$dir/debian-security/dists/$dist-security" ]; then rm --one-file-system --recursive "$dir/debian-security/dists/$dist-security" else echo "does not exist: $dir/debian-security/dists/$dist-security" >&2 fi ;; esac done for f in "$dir/debian-"*.ext4; do if [ -e "$f" ]; then rm --one-file-system "$f" fi done # on i386 and amd64, the intel-microcode and amd64-microcode packages # from non-free-firwame get pulled in because they are # priority:standard with USE_HOST_APT_CONFIG=yes for c in main non-free-firmware; do if [ -e "$dir/debian/pool/$c" ]; then rm --one-file-system --recursive "$dir/debian/pool/$c" else echo "does not exist: $dir/debian/pool/$c" >&2 fi done if [ -e "$dir/debian-security/pool/updates/main" ]; then rm --one-file-system --recursive "$dir/debian-security/pool/updates/main" else echo "does not exist: $dir/debian-security/pool/updates/main" >&2 fi for i in $(seq 1 6); do if [ ! -e "$dir/debian$i" ]; then continue fi rm "$dir/debian$i" done rm "$dir/mmdebstrapcache" # remove all symlinks find "$dir" -type l -delete # now the rest should only be empty directories if [ -e "$dir" ]; then find "$dir" -depth -print0 | xargs -0 --no-run-if-empty rmdir else echo "does not exist: $dir" >&2 fi } cleanup_newcachedir() { echo "running cleanup_newcachedir" deletecache "$newcachedir" } cleanupapt() { echo "running cleanupapt" >&2 if [ ! -e "$rootdir" ]; then return fi for f in \ "$rootdir/var/cache/apt/archives/"*.deb \ "$rootdir/var/cache/apt/archives/partial/"*.deb \ "$rootdir/var/cache/apt/"*.bin \ "$rootdir/var/lib/apt/lists/"* \ "$rootdir/var/lib/dpkg/status" \ "$rootdir/var/lib/dpkg/lock-frontend" \ "$rootdir/var/lib/dpkg/lock" \ "$rootdir/var/lib/apt/lists/lock" \ "$rootdir/etc/apt/apt.conf" \ "$rootdir/etc/apt/sources.list.d/"* \ "$rootdir/etc/apt/preferences.d/"* \ "$rootdir/etc/apt/sources.list" \ "$rootdir/var/cache/apt/archives/lock"; do if [ ! -e "$f" ]; then echo "does not exist: $f" >&2 continue fi if [ -d "$f" ]; then rmdir "$f" else rm "$f" fi done find "$rootdir" -depth -print0 | xargs -0 --no-run-if-empty rmdir } # note: this function uses brackets instead of curly braces, so that it's run # in its own process and we can handle traps independent from the outside update_cache() ( dist="$1" nativearch="$2" # use a subdirectory of $newcachedir so that we can use # hardlinks rootdir="$newcachedir/apt" mkdir -p "$rootdir" # we only set this trap here and overwrite the previous trap, because # the update_cache function is run as part of a pipe and thus in its # own process which will EXIT after it finished trap 'kill "$PROXYPID" || :;cleanupapt' EXIT INT TERM for p in /etc/apt/apt.conf.d /etc/apt/sources.list.d /etc/apt/preferences.d /var/cache/apt/archives /var/lib/apt/lists/partial /var/lib/dpkg; do mkdir -p "$rootdir/$p" done # read sources.list content from stdin cat > "$rootdir/etc/apt/sources.list" cat << END > "$rootdir/etc/apt/apt.conf" Apt::Architecture "$nativearch"; Apt::Architectures "$nativearch"; Dir::Etc "$rootdir/etc/apt"; Dir::State "$rootdir/var/lib/apt"; Dir::Cache "$rootdir/var/cache/apt"; Apt::Install-Recommends false; Apt::Get::Download-Only true; Acquire::Languages "none"; Dir::Etc::Trusted "/etc/apt/trusted.gpg"; Dir::Etc::TrustedParts "/etc/apt/trusted.gpg.d"; Acquire::http::Proxy "http://127.0.0.1:8080/"; END : > "$rootdir/var/lib/dpkg/status" if [ "$dist" = "$DEFAULT_DIST" ] && [ "$nativearch" = "$HOSTARCH" ] && [ "$USE_HOST_APT_CONFIG" = "yes" ]; then # we append sources and settings instead of overwriting after # an empty line for f in /etc/apt/sources.list /etc/apt/sources.list.d/*; do [ -e "$f" ] || continue [ -e "$rootdir/$f" ] && echo >> "$rootdir/$f" # Filter out file:// repositories as they are added # to each mmdebstrap call verbatim by # debian/tests/copy_host_apt_config # Also filter out all mirrors that are not of suite # $DEFAULT_DIST, except experimental if the suite # is unstable. This prevents packages from # unstable entering a testing mirror. if [ "$dist" = unstable ]; then grep -v ' file://' "$f" \ | grep -E " (unstable|experimental) " \ >> "$rootdir/$f" || : else grep -v ' file://' "$f" \ | grep " $DEFAULT_DIST " \ >> "$rootdir/$f" || : fi done for f in /etc/apt/preferences.d/*; do [ -e "$f" ] || continue [ -e "$rootdir/$f" ] && echo >> "$rootdir/$f" cat "$f" >> "$rootdir/$f" done fi echo "creating mirror for $dist" >&2 for f in /etc/apt/sources.list /etc/apt/sources.list.d/* /etc/apt/preferences.d/*; do [ -e "$rootdir/$f" ] || continue echo "contents of $f:" >&2 cat "$rootdir/$f" >&2 done APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update --error-on=any pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \ --format '$(FILENAME)' 'Created-By: Packages' "Architecture: $nativearch" \ | xargs --delimiter='\n' /usr/lib/apt/apt-helper cat-file \ | grep-dctrl --no-field-names --show-field=Package --exact-match \ \( --field=Essential yes --or --field=Priority required \ --or --field=Priority important --or --field=Priority standard \ \)) pkgs="$pkgs build-essential busybox gpg eatmydata fakechroot fakeroot" # we need usr-is-merged to simulate debootstrap behaviour for all dists # starting from Debian 12 (Bullseye) case "$dist" in oldstable) : ;; *) pkgs="$pkgs usr-is-merged usrmerge" ;; esac # shellcheck disable=SC2086 APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs rm "$rootdir/var/cache/apt/archives/lock" rmdir "$rootdir/var/cache/apt/archives/partial" APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --option Dir::Etc::SourceList=/dev/null update APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get clean cleanupapt # this function is run in its own process, so we unset all traps before # returning trap "-" EXIT INT TERM ) if [ -e "./shared/cache.A" ] && [ -e "./shared/cache.B" ]; then echo "both ./shared/cache.A and ./shared/cache.B exist" >&2 echo "was a former run of the script aborted?" >&2 if [ -e ./shared/cache ]; then echo "cache symlink points to $(readlink ./shared/cache)" >&2 case "$(readlink ./shared/cache)" in cache.A) echo "removing ./shared/cache.B" >&2 rm -r ./shared/cache.B ;; cache.B) echo "removing ./shared/cache.A" >&2 rm -r ./shared/cache.A ;; *) echo "unexpected" >&2 exit 1 ;; esac else echo "./shared/cache doesn't exist" >&2 exit 1 fi fi if [ -e "./shared/cache.A" ]; then oldcache=cache.A newcache=cache.B else oldcache=cache.B newcache=cache.A fi oldcachedir="./shared/$oldcache" newcachedir="./shared/$newcache" oldmirrordir="$oldcachedir/debian" newmirrordir="$newcachedir/debian" mirror="http://deb.debian.org/debian" security_mirror="http://security.debian.org/debian-security" components=main : "${DEFAULT_DIST:=unstable}" : "${ONLY_DEFAULT_DIST:=no}" : "${ONLY_HOSTARCH:=no}" : "${HAVE_QEMU:=yes}" : "${RUN_MA_SAME_TESTS:=yes}" # by default, use the mmdebstrap executable in the current directory : "${CMD:=./mmdebstrap}" : "${USE_HOST_APT_CONFIG:=no}" if [ -e "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/InRelease" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/InRelease") case "$http_code" in 200) ;; # need update 304) echo up-to-date; exit 0;; *) echo "unexpected status: $http_code"; exit 1;; esac fi ./caching_proxy.py "$oldcachedir" "$newcachedir" & PROXYPID=$! trap 'kill "$PROXYPID" || :' EXIT INT TERM for i in $(seq 10); do curl --proxy "http://127.0.0.1:8080/" --silent -o /dev/null "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" && break sleep 1 done if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then echo "failed to start proxy" >&2 kill $PROXYPID exit 1 fi trap 'kill "$PROXYPID" || :;cleanup_newcachedir' EXIT INT TERM mkdir -p "$newcachedir" touch "$newcachedir/mmdebstrapcache" HOSTARCH=$(dpkg --print-architecture) arches="$HOSTARCH" if [ "$HOSTARCH" = amd64 ]; then arches="$arches arm64 i386" elif [ "$HOSTARCH" = arm64 ]; then arches="$arches amd64 armhf" fi # we need the split_inline_sig() function # shellcheck disable=SC1091 . /usr/share/debootstrap/functions for dist in oldstable stable testing unstable; do for nativearch in $arches; do # non-host architectures are only downloaded for $DEFAULT_DIST if [ "$nativearch" != "$HOSTARCH" ] && [ "$DEFAULT_DIST" != "$dist" ]; then continue fi # if ONLY_DEFAULT_DIST is set, only download DEFAULT_DIST if [ "$ONLY_DEFAULT_DIST" = "yes" ] && [ "$DEFAULT_DIST" != "$dist" ]; then continue fi if [ "$ONLY_HOSTARCH" = "yes" ] && [ "$nativearch" != "$HOSTARCH" ]; then continue fi # we need a first pass without updates and security patches # because otherwise, old package versions needed by # debootstrap will not get included echo "deb [arch=$nativearch] $mirror $dist $components" | update_cache "$dist" "$nativearch" # we need to include the base mirror again or otherwise # packages like build-essential will be missing case "$dist" in oldstable|stable) cat << END | update_cache "$dist" "$nativearch" deb [arch=$nativearch] $mirror $dist $components deb [arch=$nativearch] $mirror $dist-updates main deb [arch=$nativearch] $security_mirror $dist-security main END ;; esac done codename=$(awk '/^Codename: / { print $2; }' < "$newmirrordir/dists/$dist/InRelease") ln -s "$dist" "$newmirrordir/dists/$codename" # split the InRelease file into Release and Release.gpg not because apt # or debootstrap need it that way but because grep-dctrl does split_inline_sig \ "$newmirrordir/dists/$dist/InRelease" \ "$newmirrordir/dists/$dist/Release" \ "$newmirrordir/dists/$dist/Release.gpg" touch --reference="$newmirrordir/dists/$dist/InRelease" "$newmirrordir/dists/$dist/Release" "$newmirrordir/dists/$dist/Release.gpg" done kill $PROXYPID # Create some symlinks so that we can trick apt into accepting multiple apt # lines that point to the same repository but look different. This is to # avoid the warning: # W: Target Packages (main/binary-all/Packages) is configured multiple times... for i in $(seq 1 6); do ln -s debian "$newcachedir/debian$i" done tmpdir="" cleanuptmpdir() { if [ -z "$tmpdir" ]; then return fi if [ ! -e "$tmpdir" ]; then return fi for f in "$tmpdir/worker.sh" "$tmpdir/mmdebstrap.service"; do if [ ! -e "$f" ]; then echo "does not exist: $f" >&2 continue fi rm "$f" done rmdir "$tmpdir" } SOURCE_DATE_EPOCH="$(date --date="$(grep-dctrl -s Date -n '' "$newmirrordir/dists/$DEFAULT_DIST/Release")" +%s)" export SOURCE_DATE_EPOCH if [ "$HAVE_QEMU" = "yes" ]; then # we use the caching proxy again when building the qemu image # - we can re-use the packages that were already downloaded earlier # - we make sure that the qemu image uses the same Release file even # if a mirror push happened between now and earlier # - we avoid polluting the mirror with the additional packages by # using --readonly ./caching_proxy.py --readonly "$oldcachedir" "$newcachedir" & PROXYPID=$! for i in $(seq 10); do curl --proxy "http://127.0.0.1:8080/" --silent -o /dev/null "http://deb.debian.org/debian/dists/$DEFAULT_DIST/InRelease" && break sleep 1 done if [ ! -s "$newmirrordir/dists/$DEFAULT_DIST/InRelease" ]; then echo "failed to start proxy" >&2 kill $PROXYPID exit 1 fi tmpdir="$(mktemp -d)" trap 'kill "$PROXYPID" || :;cleanuptmpdir; cleanup_newcachedir' EXIT INT TERM pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,disorderfs,squashfs-tools-ng,genext2fs,linux-image-generic if [ ! -e ./mmdebstrap ]; then pkgs="$pkgs,mmdebstrap" fi arches=$HOSTARCH if [ "$RUN_MA_SAME_TESTS" = "yes" ]; then case "$HOSTARCH" in amd64) arches=amd64,arm64 pkgs="$pkgs,libfakechroot:arm64,libfakeroot:arm64" ;; arm64) arches=arm64,amd64 pkgs="$pkgs,libfakechroot:amd64,libfakeroot:amd64" ;; esac fi cat << END > "$tmpdir/mmdebstrap.service" [Unit] Description=mmdebstrap worker script [Service] Type=oneshot ExecStart=/worker.sh [Install] WantedBy=multi-user.target END # here is something crazy: # as we run mmdebstrap, the process ends up being run by different users with # different privileges (real or fake). But for being able to collect # Devel::Cover data, they must all share a single directory. The only way that # I found to make this work is to mount the database directory with a # filesystem that doesn't support ownership information at all and a umask that # gives read/write access to everybody. # https://github.com/pjcj/Devel--Cover/issues/223 cat << 'END' > "$tmpdir/worker.sh" #!/bin/sh echo 'root:root' | chpasswd mount -t 9p -o trans=virtio,access=any,msize=128k mmdebstrap /mnt # need to restart mini-httpd because we mounted different content into www-root systemctl restart mini-httpd handler () { while IFS= read -r line || [ -n "$line" ]; do printf "%s %s: %s\n" "$(date -u -d "0 $(date +%s.%3N) seconds - $2 seconds" +"%T.%3N")" "$1" "$line" done } ( cd /mnt; if [ -e cover_db.img ]; then mkdir -p cover_db mount -o loop,umask=000 cover_db.img cover_db fi now=$(date +%s.%3N) ret=0 { { { { { sh -x ./test.sh 2>&1 1>&4 3>&- 4>&-; echo $? >&2; } | handler E "$now" >&3; } 4>&1 | handler O "$now" >&3; } 2>&1; } | { read xs; exit $xs; }; } 3>&1 || ret=$? echo $ret > /mnt/exitstatus.txt if [ -e cover_db.img ]; then df -h cover_db umount cover_db fi ) > /mnt/output.txt 2>&1 umount /mnt systemctl poweroff END chmod +x "$tmpdir/worker.sh" if [ -z ${DISK_SIZE+x} ]; then DISK_SIZE=10G fi # set PATH to pick up the correct mmdebstrap variant env PATH="$(dirname "$(realpath --canonicalize-existing "$CMD")"):$PATH" \ debvm-create --skip=usrmerge --size="$DISK_SIZE" --release="$DEFAULT_DIST" \ --output="$newcachedir/debian-$DEFAULT_DIST.ext4" -- \ --architectures="$arches" --include="$pkgs" \ --setup-hook='echo "Acquire::http::Proxy \"http://127.0.0.1:8080/\";" > "$1/etc/apt/apt.conf.d/00proxy"' \ --hook-dir=/usr/share/mmdebstrap/hooks/maybe-merged-usr \ --customize-hook='rm "$1/etc/apt/apt.conf.d/00proxy"' \ --customize-hook='mkdir -p "$1/etc/systemd/system/multi-user.target.wants"' \ --customize-hook='ln -s ../mmdebstrap.service "$1/etc/systemd/system/multi-user.target.wants/mmdebstrap.service"' \ --customize-hook='touch "$1/mmdebstrap-testenv"' \ --customize-hook='copy-in "'"$tmpdir"'/mmdebstrap.service" /etc/systemd/system/' \ --customize-hook='copy-in "'"$tmpdir"'/worker.sh" /' \ --customize-hook='printf 127.0.0.1 localhost > "$1/etc/hosts"' \ --customize-hook='printf "START=1\nDAEMON_OPTS=\"-h 127.0.0.1 -p 80 -u nobody -dd /mnt/cache -i /var/run/mini-httpd.pid -T UTF-8\"\n" > "$1/etc/default/mini-httpd"' \ "$mirror" kill $PROXYPID cleanuptmpdir trap "cleanup_newcachedir" EXIT INT TERM fi # delete possibly leftover symlink if [ -e ./shared/cache.tmp ]; then rm ./shared/cache.tmp fi # now atomically switch the symlink to point to the other directory ln -s $newcache ./shared/cache.tmp mv --no-target-directory ./shared/cache.tmp ./shared/cache deletecache "$oldcachedir" trap - EXIT INT TERM