Johannes Schauer Marin Rodrigues
5885291213
- only download Release files once and not by apt as well as with curl and thus avoid a mirror push happening between both downloads - no heuristic needed to place the file in their correct mirror location - no manual checksum checking - only throttle download speed when actually downloading and not when retrieving files from the cache - no translation of filenames between how the epoch colon is stored in files in /var/cache/apt/archives versus how it is stored in files on the mirrors - no special handling of stable update and security mirrors - implemented in Python instead of shell and thus an order of magnitude faster
141 lines
6.8 KiB
Bash
141 lines
6.8 KiB
Bash
#!/bin/sh
|
|
set -eu
|
|
export LC_ALL=C.UTF-8
|
|
export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
|
|
|
|
prefix=
|
|
if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
|
|
if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
|
|
if [ ! -e /mmdebstrap-testenv ]; then
|
|
echo "this test modifies the system and should only be run inside a container" >&2
|
|
exit 1
|
|
fi
|
|
useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
|
|
fi
|
|
prefix="runuser -u ${SUDO_USER:-user} --"
|
|
fi
|
|
|
|
# debootstrap uses apt-config to figure out whether the system running it has
|
|
# any proxies configured and then runs the binary to set the http_proxy
|
|
# environment variable. This will fail if debootstrap is run in a linux user
|
|
# namespace because auto-apt-proxy will see /tmp/.auto-apt-proxy-0 as being
|
|
# owned by the user "nobody" and group "nogroup" and fail with:
|
|
# insecure cache dir /tmp/.auto-apt-proxy-0. Must be owned by UID 0 and have permissions 700
|
|
# We cannot overwrite a configuration item using the APT_CONFIG environment
|
|
# variable, so instead we use it to set the Dir configuration option
|
|
# to /dev/null to force all apt settings to their defaults.
|
|
# There is currently no better way to disable this behavior. See also:
|
|
# https://bugs.debian.org/1031105
|
|
# https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/90
|
|
AUTOPROXY=
|
|
eval "$(apt-config shell AUTOPROXY Acquire::http::Proxy-Auto-Detect)"
|
|
if [ -n "$AUTOPROXY" ] && [ -x "$AUTOPROXY" ] && [ -e /tmp/.auto-apt-proxy-0 ]; then
|
|
TMP_APT_CONFIG=$(mktemp)
|
|
echo "Dir \"/dev/null\";" > "$TMP_APT_CONFIG"
|
|
chmod 644 "$TMP_APT_CONFIG"
|
|
fi
|
|
|
|
# debootstrap runs mount -t proc proc /proc which doesn't work in an unshared
|
|
# namespace on privileged docker (like salsaci), so mount /proc manually
|
|
# https://bugs.debian.org/1031222
|
|
# https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/91
|
|
$prefix {{ CMD }} --variant=custom --mode={{ MODE }} \
|
|
--setup-hook='env '"${AUTOPROXY:+APT_CONFIG='$TMP_APT_CONFIG'}"' container=lxc debootstrap --variant={{ VARIANT }} unstable "$1" {{ MIRROR }}' \
|
|
--setup-hook='mount -o rbind /proc "$1/proc"' \
|
|
--setup-hook='chroot "$1" dpkg-reconfigure systemd || true' \
|
|
--setup-hook='umount --lazy "$1/proc"' \
|
|
- /tmp/debian-mm.tar {{ MIRROR }}
|
|
if [ -n "$AUTOPROXY" ] && [ -x "$AUTOPROXY" ] && [ -e /tmp/.auto-apt-proxy-0 ]; then
|
|
rm "$TMP_APT_CONFIG"
|
|
fi
|
|
|
|
mkdir /tmp/debian-mm
|
|
tar --xattrs --xattrs-include='*' -C /tmp/debian-mm -xf /tmp/debian-mm.tar
|
|
|
|
mkdir /tmp/debian-debootstrap
|
|
tar --xattrs --xattrs-include='*' -C /tmp/debian-debootstrap -xf "cache/debian-unstable-{{ VARIANT }}.tar"
|
|
|
|
# diff cannot compare device nodes, so we use tar to do that for us and then
|
|
# delete the directory
|
|
tar -C /tmp/debian-debootstrap -cf dev1.tar ./dev
|
|
tar -C /tmp/debian-mm -cf dev2.tar ./dev
|
|
cmp dev1.tar dev2.tar >&2
|
|
rm dev1.tar dev2.tar
|
|
rm -r /tmp/debian-debootstrap/dev /tmp/debian-mm/dev
|
|
|
|
# remove downloaded deb packages
|
|
rm /tmp/debian-debootstrap/var/cache/apt/archives/*.deb
|
|
# remove aux-cache
|
|
rm /tmp/debian-debootstrap/var/cache/ldconfig/aux-cache
|
|
# remove logs
|
|
rm /tmp/debian-debootstrap/var/log/dpkg.log \
|
|
/tmp/debian-debootstrap/var/log/bootstrap.log \
|
|
/tmp/debian-debootstrap/var/log/alternatives.log \
|
|
/tmp/debian-mm/var/log/bootstrap.log
|
|
|
|
# clear out /run except for /run/lock
|
|
find /tmp/debian-debootstrap/run/ -mindepth 1 -maxdepth 1 ! -name lock -print0 | xargs --no-run-if-empty -0 rm -r
|
|
|
|
# debootstrap doesn't clean apt
|
|
rm /tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_main_binary-{{ HOSTARCH }}_Packages \
|
|
/tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_InRelease \
|
|
/tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_Release \
|
|
/tmp/debian-debootstrap/var/lib/apt/lists/127.0.0.1_debian_dists_unstable_Release.gpg
|
|
|
|
if [ -e /tmp/debian-debootstrap/etc/machine-id ]; then
|
|
rm /tmp/debian-debootstrap/etc/machine-id /tmp/debian-mm/etc/machine-id
|
|
fi
|
|
rm /tmp/debian-mm/var/cache/apt/archives/lock
|
|
rm /tmp/debian-mm/var/lib/apt/lists/lock
|
|
rm /tmp/debian-mm/var/lib/dpkg/arch
|
|
|
|
# workaround for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=917773
|
|
# also needed for users that are created by systemd-sysusers before systemd 252
|
|
# https://github.com/systemd/systemd/pull/24534
|
|
for f in shadow shadow-; do
|
|
if [ ! -e /tmp/debian-debootstrap/etc/$f ]; then
|
|
continue
|
|
fi
|
|
if ! cmp /tmp/debian-debootstrap/etc/$f /tmp/debian-mm/etc/$f >&2; then
|
|
echo patching /etc/$f >&2
|
|
awk -v FS=: -v OFS=: -v SDE={{ SOURCE_DATE_EPOCH }} '{ print $1,$2,int(SDE/60/60/24),$4,$5,$6,$7,$8,$9 }' < /tmp/debian-mm/etc/$f > /tmp/debian-mm/etc/$f.bak
|
|
cat /tmp/debian-mm/etc/$f.bak > /tmp/debian-mm/etc/$f
|
|
rm /tmp/debian-mm/etc/$f.bak
|
|
else
|
|
echo no difference for /etc/$f >&2
|
|
fi
|
|
done
|
|
|
|
# isc-dhcp-client postinst doesn't create this file in debootstrap run with
|
|
# unshared wrapper. The responsible postinst snippet was automatically added
|
|
# by dh_apparmor since isc-dhcp-client 4.4.3-P1-1.1
|
|
if [ -e /tmp/debian-debootstrap/etc/apparmor.d/local/sbin.dhclient ] && [ ! -s /tmp/debian-debootstrap/etc/apparmor.d/local/sbin.dhclient ]; then
|
|
echo /sbin/setcap > /tmp/debian-debootstrap/etc/apparmor.d/local/sbin.dhclient
|
|
fi
|
|
|
|
# check if the file content differs
|
|
diff --unified --no-dereference --recursive /tmp/debian-debootstrap /tmp/debian-mm >&2
|
|
|
|
# check permissions, ownership, symlink targets, modification times using tar
|
|
# mtimes of directories created by mmdebstrap will differ, thus we equalize them first
|
|
for d in etc/apt/preferences.d/ etc/apt/sources.list.d/ etc/dpkg/dpkg.cfg.d/ var/log/apt/; do
|
|
touch --date="@{{ SOURCE_DATE_EPOCH }}" /tmp/debian-debootstrap/$d /tmp/debian-mm/$d
|
|
done
|
|
# debootstrap never ran apt -- fixing permissions
|
|
for d in ./var/lib/apt/lists/partial ./var/cache/apt/archives/partial; do
|
|
chroot /tmp/debian-debootstrap chmod 0700 $d
|
|
chroot /tmp/debian-debootstrap chown _apt:root $d
|
|
done
|
|
tar -C /tmp/debian-debootstrap --numeric-owner --xattrs --xattrs-include='*' --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/root1.tar .
|
|
tar -C /tmp/debian-mm --numeric-owner --xattrs --xattrs-include='*' --sort=name --clamp-mtime --mtime="$(date --utc --date=@{{ SOURCE_DATE_EPOCH }} --iso-8601=seconds)" -cf /tmp/root2.tar .
|
|
tar --full-time --verbose -tf /tmp/root1.tar > /tmp/root1.tar.list
|
|
tar --full-time --verbose -tf /tmp/root2.tar > /tmp/root2.tar.list
|
|
# despite SOURCE_DATE_EPOCH and --clamp-mtime, the timestamps in the tarball
|
|
# will slightly differ from each other in the sub-second precision (last
|
|
# decimals) so the tarballs will not be identical, so we use diff to compare
|
|
# content and tar to compare attributes
|
|
diff -u /tmp/root1.tar.list /tmp/root2.tar.list >&2
|
|
rm /tmp/root1.tar /tmp/root2.tar /tmp/root1.tar.list /tmp/root2.tar.list
|
|
|
|
rm /tmp/debian-mm.tar
|
|
rm -r /tmp/debian-debootstrap /tmp/debian-mm
|