forked from josch/mmdebstrap
make_mirror.sh: never leave the cache in an inconsistent state by atomically switching symlinks
This commit is contained in:
parent
8d92412380
commit
b113771a81
3 changed files with 91 additions and 34 deletions
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
mirrordir="./shared/debian"
|
mirrordir="./shared/cache/debian"
|
||||||
|
|
||||||
./make_mirror.sh
|
./make_mirror.sh
|
||||||
|
|
||||||
|
|
121
make_mirror.sh
121
make_mirror.sh
|
@ -2,8 +2,32 @@
|
||||||
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
mirrordir="./shared/debian"
|
# This script fills either cache.A or cache.B with new content and then
|
||||||
cachedir="./shared/cache"
|
# atomically switches the cache symlink from one to the other at the end.
|
||||||
|
# This way, at no point will the cache be in an non-working state, even
|
||||||
|
# when this script got canceled at any point.
|
||||||
|
# Working with two directories also automatically prunes old packages in
|
||||||
|
# the local repository.
|
||||||
|
|
||||||
|
if [ -e "./shared/cache.A" ] && [ -e "./shared/cache.B" ]; then
|
||||||
|
echo "both ./shared/cache.A and ./shared/cache.B exist" >&2
|
||||||
|
echo "was a former run of the script aborted?" >&2
|
||||||
|
echo "cache symlink points to $(readlink ./shared/cache)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -e "./shared/cache.A" ]; then
|
||||||
|
oldcache=cache.A
|
||||||
|
newcache=cache.B
|
||||||
|
else
|
||||||
|
oldcache=cache.B
|
||||||
|
newcache=cache.A
|
||||||
|
fi
|
||||||
|
|
||||||
|
oldcachedir="./shared/$oldcache"
|
||||||
|
newcachedir="./shared/$newcache"
|
||||||
|
oldmirrordir="$oldcachedir/debian"
|
||||||
|
newmirrordir="$newcachedir/debian"
|
||||||
|
|
||||||
mirror="http://deb.debian.org/debian"
|
mirror="http://deb.debian.org/debian"
|
||||||
arch1=$(dpkg --print-architecture)
|
arch1=$(dpkg --print-architecture)
|
||||||
|
@ -15,8 +39,8 @@ components=main
|
||||||
|
|
||||||
: "${HAVE_QEMU:=yes}"
|
: "${HAVE_QEMU:=yes}"
|
||||||
|
|
||||||
if [ -e "$mirrordir/dists/unstable/Release" ]; then
|
if [ -e "$oldmirrordir/dists/unstable/Release" ]; then
|
||||||
http_code=$(curl --output /dev/null --silent --location --head --time-cond "$mirrordir/dists/unstable/Release" --write-out '%{http_code}' "$mirror/dists/unstable/Release")
|
http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/unstable/Release" --write-out '%{http_code}' "$mirror/dists/unstable/Release")
|
||||||
case "$http_code" in
|
case "$http_code" in
|
||||||
200) ;; # need update
|
200) ;; # need update
|
||||||
304) echo up-to-date; exit 0;;
|
304) echo up-to-date; exit 0;;
|
||||||
|
@ -24,14 +48,6 @@ if [ -e "$mirrordir/dists/unstable/Release" ]; then
|
||||||
esac
|
esac
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# use -f because the files might not exist
|
|
||||||
for dist in stable testing unstable; do
|
|
||||||
for variant in minbase buildd -; do
|
|
||||||
rm -f "$cachedir/debian-$dist-$variant.tar"
|
|
||||||
done
|
|
||||||
done
|
|
||||||
rm -f "$cachedir/debian-unstable.qcow"
|
|
||||||
|
|
||||||
for nativearch in $arch1 $arch2; do
|
for nativearch in $arch1 $arch2; do
|
||||||
for dist in stable testing unstable; do
|
for dist in stable testing unstable; do
|
||||||
rootdir=$(mktemp --directory)
|
rootdir=$(mktemp --directory)
|
||||||
|
@ -67,12 +83,12 @@ END
|
||||||
# file, copy all old *.deb packages from the mirror to
|
# file, copy all old *.deb packages from the mirror to
|
||||||
# /var/cache/apt/archives so that apt will not re-download *.deb
|
# /var/cache/apt/archives so that apt will not re-download *.deb
|
||||||
# packages that we already have
|
# packages that we already have
|
||||||
if [ -e "$mirrordir/dists/$dist/main/binary-$nativearch/Packages.gz" ]; then
|
if [ -e "$oldmirrordir/dists/$dist/main/binary-$nativearch/Packages.gz" ]; then
|
||||||
gzip -dc "$mirrordir/dists/$dist/main/binary-$nativearch/Packages.gz" \
|
gzip -dc "$oldmirrordir/dists/$dist/main/binary-$nativearch/Packages.gz" \
|
||||||
| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename '' \
|
| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename '' \
|
||||||
| paste -sd " \n" \
|
| paste -sd " \n" \
|
||||||
| while read name ver arch fname; do
|
| while read name ver arch fname; do
|
||||||
if [ ! -e "$mirrordir/$fname" ]; then
|
if [ ! -e "$oldmirrordir/$fname" ]; then
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
# apt stores deb files with the colon encoded as %3a while
|
# apt stores deb files with the colon encoded as %3a while
|
||||||
|
@ -85,7 +101,7 @@ END
|
||||||
# doesn't recognize symlinks
|
# doesn't recognize symlinks
|
||||||
# we cannot do a hardlink because the two
|
# we cannot do a hardlink because the two
|
||||||
# directories might be on different devices
|
# directories might be on different devices
|
||||||
cp -a "$mirrordir/$fname" "$aptname"
|
cp -a "$oldmirrordir/$fname" "$aptname"
|
||||||
echo "$aptname" >> "$rootdir/oldaptnames"
|
echo "$aptname" >> "$rootdir/oldaptnames"
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
@ -103,10 +119,10 @@ END
|
||||||
APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs
|
APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs
|
||||||
|
|
||||||
# to be able to also test gpg verification, we need to create a mirror
|
# to be able to also test gpg verification, we need to create a mirror
|
||||||
mkdir -p "$mirrordir/dists/$dist/" "$mirrordir/dists/$dist/main/binary-$nativearch/"
|
mkdir -p "$newmirrordir/dists/$dist/" "$newmirrordir/dists/$dist/main/binary-$nativearch/"
|
||||||
curl --location "$mirror/dists/$dist/Release" > "$mirrordir/dists/$dist/Release"
|
curl --location "$mirror/dists/$dist/Release" > "$newmirrordir/dists/$dist/Release"
|
||||||
curl --location "$mirror/dists/$dist/Release.gpg" > "$mirrordir/dists/$dist/Release.gpg"
|
curl --location "$mirror/dists/$dist/Release.gpg" > "$newmirrordir/dists/$dist/Release.gpg"
|
||||||
curl --location "$mirror/dists/$dist/main/binary-$nativearch/Packages.gz" > "$mirrordir/dists/$dist/main/binary-$nativearch/Packages.gz"
|
curl --location "$mirror/dists/$dist/main/binary-$nativearch/Packages.gz" > "$newmirrordir/dists/$dist/main/binary-$nativearch/Packages.gz"
|
||||||
|
|
||||||
# the deb files downloaded by apt must be moved to their right locations in the
|
# the deb files downloaded by apt must be moved to their right locations in the
|
||||||
# pool directory
|
# pool directory
|
||||||
|
@ -117,7 +133,7 @@ END
|
||||||
# stripping the epoch from the filename and will break once mirrors change.
|
# stripping the epoch from the filename and will break once mirrors change.
|
||||||
# This way, it doesn't matter where the mirror ends up storing the package.
|
# This way, it doesn't matter where the mirror ends up storing the package.
|
||||||
> "$rootdir/newaptnames"
|
> "$rootdir/newaptnames"
|
||||||
gzip -dc "$mirrordir/dists/$dist/main/binary-$nativearch/Packages.gz" \
|
gzip -dc "$newmirrordir/dists/$dist/main/binary-$nativearch/Packages.gz" \
|
||||||
| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename,MD5sum '' \
|
| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename,MD5sum '' \
|
||||||
| paste -sd " \n" \
|
| paste -sd " \n" \
|
||||||
| while read name ver arch fname md5; do
|
| while read name ver arch fname md5; do
|
||||||
|
@ -129,8 +145,8 @@ END
|
||||||
if [ -e "$aptname" ]; then
|
if [ -e "$aptname" ]; then
|
||||||
# make sure that we found the right file by checking its hash
|
# make sure that we found the right file by checking its hash
|
||||||
echo "$md5 $aptname" | md5sum --check
|
echo "$md5 $aptname" | md5sum --check
|
||||||
mkdir -p "$mirrordir/$dir"
|
mkdir -p "$newmirrordir/$dir"
|
||||||
mv "$aptname" "$mirrordir/$fname"
|
mv "$aptname" "$newmirrordir/$fname"
|
||||||
echo "$aptname" >> "$rootdir/newaptnames"
|
echo "$aptname" >> "$rootdir/newaptnames"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
@ -154,9 +170,6 @@ END
|
||||||
done
|
done
|
||||||
done
|
done
|
||||||
|
|
||||||
# now fill the cache with new content
|
|
||||||
mkdir -p "$cachedir"
|
|
||||||
|
|
||||||
if [ "$HAVE_QEMU" = "yes" ]; then
|
if [ "$HAVE_QEMU" = "yes" ]; then
|
||||||
# We must not use any --dpkgopt here because any dpkg options still
|
# We must not use any --dpkgopt here because any dpkg options still
|
||||||
# leak into the chroot with chrootless mode.
|
# leak into the chroot with chrootless mode.
|
||||||
|
@ -200,6 +213,8 @@ END
|
||||||
cat << 'END' > "$tmpdir/worker.sh"
|
cat << 'END' > "$tmpdir/worker.sh"
|
||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
mount -t 9p -o trans=virtio,access=any mmdebstrap /mnt
|
mount -t 9p -o trans=virtio,access=any mmdebstrap /mnt
|
||||||
|
# need to restart mini-httpd because we mounted different content into www-root
|
||||||
|
systemctl restart mini-httpd
|
||||||
(
|
(
|
||||||
cd /mnt;
|
cd /mnt;
|
||||||
if [ -e cover_db.img ]; then
|
if [ -e cover_db.img ]; then
|
||||||
|
@ -218,9 +233,11 @@ umount /mnt
|
||||||
systemctl poweroff
|
systemctl poweroff
|
||||||
END
|
END
|
||||||
chmod +x "$tmpdir/worker.sh"
|
chmod +x "$tmpdir/worker.sh"
|
||||||
cat << 'END' > "$tmpdir/mini-httpd"
|
# initially we serve from the new cache so that debootstrap can grab
|
||||||
|
# the new package repository and not the old
|
||||||
|
cat << END > "$tmpdir/mini-httpd"
|
||||||
START=1
|
START=1
|
||||||
DAEMON_OPTS="-h 127.0.0.1 -p 80 -u nobody -dd /mnt -i /var/run/mini-httpd.pid -T UTF-8"
|
DAEMON_OPTS="-h 127.0.0.1 -p 80 -u nobody -dd /mnt/$newcache -i /var/run/mini-httpd.pid -T UTF-8"
|
||||||
END
|
END
|
||||||
cat << 'END' > "$tmpdir/hosts"
|
cat << 'END' > "$tmpdir/hosts"
|
||||||
127.0.0.1 localhost
|
127.0.0.1 localhost
|
||||||
|
@ -242,13 +259,13 @@ END
|
||||||
copy-in "$tmpdir/mini-httpd" /etc/default : \
|
copy-in "$tmpdir/mini-httpd" /etc/default : \
|
||||||
copy-in "$tmpdir/hosts" /etc/ :
|
copy-in "$tmpdir/hosts" /etc/ :
|
||||||
rm "$tmpdir/extlinux.conf" "$tmpdir/worker.sh" "$tmpdir/mini-httpd" "$tmpdir/hosts" "$tmpdir/debian-unstable.tar" "$tmpdir/mmdebstrap.service"
|
rm "$tmpdir/extlinux.conf" "$tmpdir/worker.sh" "$tmpdir/mini-httpd" "$tmpdir/hosts" "$tmpdir/debian-unstable.tar" "$tmpdir/mmdebstrap.service"
|
||||||
qemu-img convert -O qcow2 "$tmpdir/debian-unstable.img" "$cachedir/debian-unstable.qcow"
|
qemu-img convert -O qcow2 "$tmpdir/debian-unstable.img" "$newcachedir/debian-unstable.qcow"
|
||||||
rm "$tmpdir/debian-unstable.img"
|
rm "$tmpdir/debian-unstable.img"
|
||||||
rmdir "$tmpdir"
|
rmdir "$tmpdir"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
mirror="http://127.0.0.1/debian"
|
mirror="http://127.0.0.1/debian"
|
||||||
SOURCE_DATE_EPOCH=$(date --date="$(grep-dctrl -s Date -n '' "$mirrordir/dists/unstable/Release")" +%s)
|
SOURCE_DATE_EPOCH=$(date --date="$(grep-dctrl -s Date -n '' "$newmirrordir/dists/unstable/Release")" +%s)
|
||||||
for dist in stable testing unstable; do
|
for dist in stable testing unstable; do
|
||||||
for variant in minbase buildd -; do
|
for variant in minbase buildd -; do
|
||||||
# skip because of different userids for apt/systemd
|
# skip because of different userids for apt/systemd
|
||||||
|
@ -266,9 +283,49 @@ set -eu
|
||||||
export LC_ALL=C.UTF-8
|
export LC_ALL=C.UTF-8
|
||||||
export SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
|
export SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
|
||||||
debootstrap --no-merged-usr --variant=$variant $dist /tmp/debian-$dist-debootstrap $mirror
|
debootstrap --no-merged-usr --variant=$variant $dist /tmp/debian-$dist-debootstrap $mirror
|
||||||
tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system -C /tmp/debian-$dist-debootstrap -c . > "cache/debian-$dist-$variant.tar"
|
tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system -C /tmp/debian-$dist-debootstrap -c . > "$newcache/debian-$dist-$variant.tar"
|
||||||
rm -r /tmp/debian-$dist-debootstrap
|
rm -r /tmp/debian-$dist-debootstrap
|
||||||
END
|
END
|
||||||
./run_qemu.sh SUDO
|
if [ "$HAVE_QEMU" = "yes" ]; then
|
||||||
|
cachedir=$newcachedir ./run_qemu.sh
|
||||||
|
else
|
||||||
|
./run_null.sh SUDO
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# now replace the minihttpd config with one that serves the new repository
|
||||||
|
# create a temporary directory because "copy-in" cannot rename the file
|
||||||
|
tmpdir="$(mktemp -d)"
|
||||||
|
cat << END > "$tmpdir/mini-httpd"
|
||||||
|
START=1
|
||||||
|
DAEMON_OPTS="-h 127.0.0.1 -p 80 -u nobody -dd /mnt/cache -i /var/run/mini-httpd.pid -T UTF-8"
|
||||||
|
END
|
||||||
|
guestfish -a "$newcachedir/debian-unstable.qcow" -i copy-in "$tmpdir/mini-httpd" /etc/default
|
||||||
|
rm "$tmpdir/mini-httpd"
|
||||||
|
rmdir "$tmpdir"
|
||||||
|
|
||||||
|
# delete possibly leftover symlink
|
||||||
|
if [ -e ./shared/cache.tmp ]; then
|
||||||
|
rm ./shared/cache.tmp
|
||||||
|
fi
|
||||||
|
# now atomically switch the symlink to point to the other directory
|
||||||
|
ln -s $newcache ./shared/cache.tmp
|
||||||
|
mv --no-target-directory ./shared/cache.tmp ./shared/cache
|
||||||
|
# be very careful with removing the old directory
|
||||||
|
for dist in stable testing unstable; do
|
||||||
|
for variant in minbase buildd -; do
|
||||||
|
if [ -e "$oldcachedir/debian-$dist-$variant.tar" ]; then
|
||||||
|
rm "$oldcachedir/debian-$dist-$variant.tar"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ -e "$oldcachedir/debian/dists/$dist" ]; then
|
||||||
|
rm --one-file-system --recursive "$oldcachedir/debian/dists/$dist"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ -e $oldcachedir/debian-unstable.qcow ]; then
|
||||||
|
rm --one-file-system $oldcachedir/debian-unstable.qcow
|
||||||
|
fi
|
||||||
|
rm --one-file-system --recursive $oldcachedir/debian/pool/main
|
||||||
|
# now the rest should only be empty directories
|
||||||
|
find $oldcachedir -depth -print0 | xargs -0 rmdir
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
cachedir="./shared/cache"
|
: "${cachedir:=./shared/cache}"
|
||||||
tmpdir="$(mktemp -d)"
|
tmpdir="$(mktemp -d)"
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
|
|
Loading…
Reference in a new issue