make_mirror.sh: set traps to automatically cleanup on failure
This commit is contained in:
parent
aaac3e677f
commit
ca9cb3145e
GPG key ID:
F2CBA5C78FBD83E1
1 changed files with 199 additions and 97 deletions
288
make_mirror.sh
288
make_mirror.sh
|
|
@ -9,57 +9,71 @@ set -eu
|
||||||
# Working with two directories also automatically prunes old packages in
|
# Working with two directories also automatically prunes old packages in
|
||||||
# the local repository.
|
# the local repository.
|
||||||
|
|
||||||
if [ -e "./shared/cache.A" ] && [ -e "./shared/cache.B" ]; then
|
deletecache() {
|
||||||
echo "both ./shared/cache.A and ./shared/cache.B exist" >&2
|
dir="$1"
|
||||||
echo "was a former run of the script aborted?" >&2
|
echo "running deletecache $dir">&2
|
||||||
if [ -e ./shared/cache ]; then
|
if [ ! -e "$dir" ]; then
|
||||||
echo "cache symlink points to $(readlink ./shared/cache)" >&2
|
return
|
||||||
case "$(readlink ./shared/cache)" in
|
|
||||||
cache.A)
|
|
||||||
echo "maybe rm -r ./shared/cache.B" >&2
|
|
||||||
;;
|
|
||||||
cache.B)
|
|
||||||
echo "maybe rm -r ./shared/cache.A" >&2
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "unexpected" >&2
|
|
||||||
esac
|
|
||||||
fi
|
fi
|
||||||
exit 1
|
if [ ! -e "$dir/mmdebstrapcache" ]; then
|
||||||
|
echo "$dir cannot be the mmdebstrap cache" >&2
|
||||||
|
return 1
|
||||||
fi
|
fi
|
||||||
|
# be very careful with removing the old directory
|
||||||
if [ -e "./shared/cache.A" ]; then
|
for dist in stable testing unstable; do
|
||||||
oldcache=cache.A
|
for variant in minbase buildd -; do
|
||||||
newcache=cache.B
|
if [ -e "$dir/debian-$dist-$variant.tar" ]; then
|
||||||
|
rm "$dir/debian-$dist-$variant.tar"
|
||||||
else
|
else
|
||||||
oldcache=cache.B
|
echo "does not exist: $dir/debian-$dist-$variant.tar" >&2
|
||||||
newcache=cache.A
|
|
||||||
fi
|
fi
|
||||||
|
done
|
||||||
oldcachedir="./shared/$oldcache"
|
if [ -e "$dir/debian/dists/$dist" ]; then
|
||||||
newcachedir="./shared/$newcache"
|
rm --one-file-system --recursive "$dir/debian/dists/$dist"
|
||||||
oldmirrordir="$oldcachedir/debian"
|
else
|
||||||
newmirrordir="$newcachedir/debian"
|
echo "does not exist: $dir/debian/dists/$dist" >&2
|
||||||
|
|
||||||
mirror="http://deb.debian.org/debian"
|
|
||||||
security_mirror="http://security.debian.org/debian-security"
|
|
||||||
if [ "$(dpkg --print-architecture)" != amd64 ]; then
|
|
||||||
echo "script only supports being run on amd64" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
fi
|
||||||
components=main
|
if [ "$dist" = "stable" ]; then
|
||||||
|
if [ -e "$dir/debian/dists/stable-updates" ]; then
|
||||||
: "${DEFAULT_DIST:=unstable}"
|
rm --one-file-system --recursive "$dir/debian/dists/stable-updates"
|
||||||
: "${HAVE_QEMU:=yes}"
|
else
|
||||||
|
echo "does not exist: $dir/debian/dists/stable-updates" >&2
|
||||||
if [ -e "$oldmirrordir/dists/$DEFAULT_DIST/Release" ]; then
|
|
||||||
http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/Release" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/Release")
|
|
||||||
case "$http_code" in
|
|
||||||
200) ;; # need update
|
|
||||||
304) echo up-to-date; exit 0;;
|
|
||||||
*) echo "unexpected status: $http_code"; exit 1;;
|
|
||||||
esac
|
|
||||||
fi
|
fi
|
||||||
|
if [ -e "$dir/debian-security/dists/stable/updates" ]; then
|
||||||
|
rm --one-file-system --recursive "$dir/debian-security/dists/stable/updates"
|
||||||
|
else
|
||||||
|
echo "does not exist: $dir/debian-security/dists/stable/updates" >&2
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ -e $dir/debian-*.qcow ]; then
|
||||||
|
rm --one-file-system "$dir"/debian-*.qcow
|
||||||
|
else
|
||||||
|
echo "does not exist: $dir/debian-*.qcow" >&2
|
||||||
|
fi
|
||||||
|
if [ -e "$dir/debian/pool/main" ]; then
|
||||||
|
rm --one-file-system --recursive "$dir/debian/pool/main"
|
||||||
|
else
|
||||||
|
echo "does not exist: $dir/debian/pool/main" >&2
|
||||||
|
fi
|
||||||
|
if [ -e "$dir/debian-security/pool/updates/main" ]; then
|
||||||
|
rm --one-file-system --recursive "$dir/debian-security/pool/updates/main"
|
||||||
|
else
|
||||||
|
echo "does not exist: $dir/debian-security/pool/updates/main" >&2
|
||||||
|
fi
|
||||||
|
rm "$dir/mmdebstrapcache"
|
||||||
|
# now the rest should only be empty directories
|
||||||
|
if [ -e "$dir" ]; then
|
||||||
|
find "$dir" -depth -print0 | xargs -0 --no-run-if-empty rmdir
|
||||||
|
else
|
||||||
|
echo "does not exist: $dir" >&2
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup_newcachedir() {
|
||||||
|
echo "running cleanup_newcachedir"
|
||||||
|
deletecache "$newcachedir"
|
||||||
|
}
|
||||||
|
|
||||||
get_oldaptnames() {
|
get_oldaptnames() {
|
||||||
if [ ! -e "$1/$2" ]; then
|
if [ ! -e "$1/$2" ]; then
|
||||||
|
|
@ -127,7 +141,40 @@ get_newaptnames() {
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
update_cache() {
|
cleanupapt() {
|
||||||
|
echo "running cleanupapt" >&2
|
||||||
|
if [ ! -e "$rootdir" ]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
for f in \
|
||||||
|
"$rootdir/var/cache/apt/archives/"*.deb \
|
||||||
|
"$rootdir/var/cache/apt/archives/partial/"*.deb \
|
||||||
|
"$rootdir/var/cache/apt/"*.bin \
|
||||||
|
"$rootdir/var/lib/apt/lists/"* \
|
||||||
|
"$rootdir/var/lib/dpkg/status" \
|
||||||
|
"$rootdir/var/lib/dpkg/lock-frontend" \
|
||||||
|
"$rootdir/var/lib/dpkg/lock" \
|
||||||
|
"$rootdir/etc/apt/apt.conf" \
|
||||||
|
"$rootdir/etc/apt/sources.list" \
|
||||||
|
"$rootdir/oldaptnames" \
|
||||||
|
"$rootdir/newaptnames" \
|
||||||
|
"$rootdir/var/cache/apt/archives/lock"; do
|
||||||
|
if [ ! -e "$f" ]; then
|
||||||
|
echo "does not exist: $f" >&2
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [ -d "$f" ]; then
|
||||||
|
rmdir "$f"
|
||||||
|
else
|
||||||
|
rm "$f"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
find "$rootdir" -depth -print0 | xargs -0 --no-run-if-empty rmdir
|
||||||
|
}
|
||||||
|
|
||||||
|
# note: this function uses brackets instead of curly braces, so that it's run
|
||||||
|
# in its own process and we can handle traps independent from the outside
|
||||||
|
update_cache() (
|
||||||
dist="$1"
|
dist="$1"
|
||||||
nativearch="$2"
|
nativearch="$2"
|
||||||
|
|
||||||
|
|
@ -136,6 +183,11 @@ update_cache() {
|
||||||
rootdir="$newcachedir/apt"
|
rootdir="$newcachedir/apt"
|
||||||
mkdir -p "$rootdir"
|
mkdir -p "$rootdir"
|
||||||
|
|
||||||
|
# we only set this trap here and overwrite the previous trap, because
|
||||||
|
# the update_cache function is run as part of a pipe and thus in its
|
||||||
|
# own process which will EXIT after it finished
|
||||||
|
trap "cleanupapt" EXIT INT TERM
|
||||||
|
|
||||||
for p in /etc/apt/apt.conf.d /etc/apt/sources.list.d /etc/apt/preferences.d /var/cache/apt/archives /var/lib/apt/lists/partial /var/lib/dpkg; do
|
for p in /etc/apt/apt.conf.d /etc/apt/sources.list.d /etc/apt/preferences.d /var/cache/apt/archives /var/lib/apt/lists/partial /var/lib/dpkg; do
|
||||||
mkdir -p "$rootdir/$p"
|
mkdir -p "$rootdir/$p"
|
||||||
done
|
done
|
||||||
|
|
@ -229,20 +281,73 @@ END
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# cleanup
|
|
||||||
APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --option Dir::Etc::SourceList=/dev/null update
|
APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --option Dir::Etc::SourceList=/dev/null update
|
||||||
APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get clean
|
APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get clean
|
||||||
rm "$rootdir/var/cache/apt/archives/lock"
|
|
||||||
rm "$rootdir/var/lib/apt/lists/lock"
|
cleanupapt
|
||||||
rm "$rootdir/var/lib/dpkg/status"
|
|
||||||
rm "$rootdir/var/lib/dpkg/lock-frontend"
|
# this function is run in its own process, so we unset all traps before
|
||||||
rm "$rootdir/var/lib/dpkg/lock"
|
# returning
|
||||||
rm "$rootdir/etc/apt/apt.conf"
|
trap "-" EXIT INT TERM
|
||||||
rm "$rootdir/etc/apt/sources.list"
|
)
|
||||||
rm "$rootdir/oldaptnames"
|
|
||||||
rm "$rootdir/newaptnames"
|
if [ -e "./shared/cache.A" ] && [ -e "./shared/cache.B" ]; then
|
||||||
find "$rootdir" -depth -print0 | xargs -0 rmdir
|
echo "both ./shared/cache.A and ./shared/cache.B exist" >&2
|
||||||
}
|
echo "was a former run of the script aborted?" >&2
|
||||||
|
if [ -e ./shared/cache ]; then
|
||||||
|
echo "cache symlink points to $(readlink ./shared/cache)" >&2
|
||||||
|
case "$(readlink ./shared/cache)" in
|
||||||
|
cache.A)
|
||||||
|
echo "maybe rm -r ./shared/cache.B" >&2
|
||||||
|
;;
|
||||||
|
cache.B)
|
||||||
|
echo "maybe rm -r ./shared/cache.A" >&2
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "unexpected" >&2
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -e "./shared/cache.A" ]; then
|
||||||
|
oldcache=cache.A
|
||||||
|
newcache=cache.B
|
||||||
|
else
|
||||||
|
oldcache=cache.B
|
||||||
|
newcache=cache.A
|
||||||
|
fi
|
||||||
|
|
||||||
|
oldcachedir="./shared/$oldcache"
|
||||||
|
newcachedir="./shared/$newcache"
|
||||||
|
|
||||||
|
oldmirrordir="$oldcachedir/debian"
|
||||||
|
newmirrordir="$newcachedir/debian"
|
||||||
|
|
||||||
|
mirror="http://deb.debian.org/debian"
|
||||||
|
security_mirror="http://security.debian.org/debian-security"
|
||||||
|
if [ "$(dpkg --print-architecture)" != amd64 ]; then
|
||||||
|
echo "script only supports being run on amd64" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
components=main
|
||||||
|
|
||||||
|
: "${DEFAULT_DIST:=unstable}"
|
||||||
|
: "${HAVE_QEMU:=yes}"
|
||||||
|
|
||||||
|
if [ -e "$oldmirrordir/dists/$DEFAULT_DIST/Release" ]; then
|
||||||
|
http_code=$(curl --output /dev/null --silent --location --head --time-cond "$oldmirrordir/dists/$DEFAULT_DIST/Release" --write-out '%{http_code}' "$mirror/dists/$DEFAULT_DIST/Release")
|
||||||
|
case "$http_code" in
|
||||||
|
200) ;; # need update
|
||||||
|
304) echo up-to-date; exit 0;;
|
||||||
|
*) echo "unexpected status: $http_code"; exit 1;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
|
trap "cleanup_newcachedir" EXIT INT TERM
|
||||||
|
|
||||||
|
mkdir -p "$newcachedir"
|
||||||
|
touch "$newcachedir/mmdebstrapcache"
|
||||||
|
|
||||||
for nativearch in amd64 armhf i386; do
|
for nativearch in amd64 armhf i386; do
|
||||||
for dist in stable testing unstable; do
|
for dist in stable testing unstable; do
|
||||||
|
|
@ -264,6 +369,30 @@ END
|
||||||
done
|
done
|
||||||
done
|
done
|
||||||
|
|
||||||
|
tmpdir=""
|
||||||
|
|
||||||
|
cleanuptmpdir() {
|
||||||
|
if [ -z "$tmpdir" ]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
if [ ! -e "$tmpdir" ]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
for f in "$tmpdir/extlinux.conf" \
|
||||||
|
"$tmpdir/worker.sh" \
|
||||||
|
"$tmpdir/mini-httpd" "$tmpdir/hosts" \
|
||||||
|
"$tmpdir/debian-chroot.tar" \
|
||||||
|
"$tmpdir/mmdebstrap.service" \
|
||||||
|
"$tmpdir/debian-$DEFAULT_DIST.img"; do
|
||||||
|
if [ ! -e "$f" ]; then
|
||||||
|
echo "does not exist: $f" >&2
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
rm "$f"
|
||||||
|
done
|
||||||
|
rmdir "$tmpdir"
|
||||||
|
}
|
||||||
|
|
||||||
if [ "$HAVE_QEMU" = "yes" ]; then
|
if [ "$HAVE_QEMU" = "yes" ]; then
|
||||||
# We must not use any --dpkgopt here because any dpkg options still
|
# We must not use any --dpkgopt here because any dpkg options still
|
||||||
# leak into the chroot with chrootless mode.
|
# leak into the chroot with chrootless mode.
|
||||||
|
|
@ -272,6 +401,8 @@ if [ "$HAVE_QEMU" = "yes" ]; then
|
||||||
# - it doesn't matter if the base system is from a different mirror timestamp
|
# - it doesn't matter if the base system is from a different mirror timestamp
|
||||||
# procps is needed for /sbin/sysctl
|
# procps is needed for /sbin/sysctl
|
||||||
tmpdir="$(mktemp -d)"
|
tmpdir="$(mktemp -d)"
|
||||||
|
trap "cleanuptmpdir; cleanup_newcachedir" EXIT INT TERM
|
||||||
|
|
||||||
./mmdebstrap --variant=apt --architectures=amd64,armhf \
|
./mmdebstrap --variant=apt --architectures=amd64,armhf \
|
||||||
--include=perl-doc,linux-image-amd64,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,proot,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,debootstrap,libfakechroot:armhf,libfakeroot:armhf,procps,apt-cudf,aspcud \
|
--include=perl-doc,linux-image-amd64,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,proot,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,debootstrap,libfakechroot:armhf,libfakeroot:armhf,procps,apt-cudf,aspcud \
|
||||||
$DEFAULT_DIST - "$mirror" > "$tmpdir/debian-chroot.tar"
|
$DEFAULT_DIST - "$mirror" > "$tmpdir/debian-chroot.tar"
|
||||||
|
|
@ -369,10 +500,9 @@ END
|
||||||
copy-in "$tmpdir/mini-httpd" /etc/default : \
|
copy-in "$tmpdir/mini-httpd" /etc/default : \
|
||||||
copy-in "$tmpdir/hosts" /etc/ : \
|
copy-in "$tmpdir/hosts" /etc/ : \
|
||||||
touch /mmdebstrap-testenv :
|
touch /mmdebstrap-testenv :
|
||||||
rm "$tmpdir/extlinux.conf" "$tmpdir/worker.sh" "$tmpdir/mini-httpd" "$tmpdir/hosts" "$tmpdir/debian-chroot.tar" "$tmpdir/mmdebstrap.service"
|
|
||||||
qemu-img convert -O qcow2 "$tmpdir/debian-$DEFAULT_DIST.img" "$newcachedir/debian-$DEFAULT_DIST.qcow"
|
qemu-img convert -O qcow2 "$tmpdir/debian-$DEFAULT_DIST.img" "$newcachedir/debian-$DEFAULT_DIST.qcow"
|
||||||
rm "$tmpdir/debian-$DEFAULT_DIST.img"
|
cleanuptmpdir
|
||||||
rmdir "$tmpdir"
|
trap "cleanup_newcachedir" EXIT INT TERM
|
||||||
fi
|
fi
|
||||||
|
|
||||||
mirror="http://127.0.0.1/debian"
|
mirror="http://127.0.0.1/debian"
|
||||||
|
|
@ -414,35 +544,7 @@ fi
|
||||||
# now atomically switch the symlink to point to the other directory
|
# now atomically switch the symlink to point to the other directory
|
||||||
ln -s $newcache ./shared/cache.tmp
|
ln -s $newcache ./shared/cache.tmp
|
||||||
mv --no-target-directory ./shared/cache.tmp ./shared/cache
|
mv --no-target-directory ./shared/cache.tmp ./shared/cache
|
||||||
# be very careful with removing the old directory
|
|
||||||
for dist in stable testing unstable; do
|
deletecache "$oldcachedir"
|
||||||
for variant in minbase buildd -; do
|
|
||||||
if [ -e "$oldcachedir/debian-$dist-$variant.tar" ]; then
|
trap - EXIT INT TERM
|
||||||
rm "$oldcachedir/debian-$dist-$variant.tar"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
if [ -e "$oldcachedir/debian/dists/$dist" ]; then
|
|
||||||
rm --one-file-system --recursive "$oldcachedir/debian/dists/$dist"
|
|
||||||
fi
|
|
||||||
if [ "$dist" = "stable" ]; then
|
|
||||||
if [ -e "$oldcachedir/debian/dists/stable-updates" ]; then
|
|
||||||
rm --one-file-system --recursive "$oldcachedir/debian/dists/stable-updates"
|
|
||||||
fi
|
|
||||||
if [ -e "$oldcachedir/debian-security/dists/stable/updates" ]; then
|
|
||||||
rm --one-file-system --recursive "$oldcachedir/debian-security/dists/stable/updates"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
if [ -e $oldcachedir/debian-*.qcow ]; then
|
|
||||||
rm --one-file-system "$oldcachedir"/debian-*.qcow
|
|
||||||
fi
|
|
||||||
if [ -e "$oldcachedir/debian/pool/main" ]; then
|
|
||||||
rm --one-file-system --recursive "$oldcachedir/debian/pool/main"
|
|
||||||
fi
|
|
||||||
if [ -e "$oldcachedir/debian-security/pool/updates/main" ]; then
|
|
||||||
rm --one-file-system --recursive "$oldcachedir/debian-security/pool/updates/main"
|
|
||||||
fi
|
|
||||||
# now the rest should only be empty directories
|
|
||||||
if [ -e "$oldcachedir" ]; then
|
|
||||||
find "$oldcachedir" -depth -print0 | xargs -0 --no-run-if-empty rmdir
|
|
||||||
fi
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue