josch/mmdebstrap
2
3
Fork 6
Code Issues 8 Pull requests 1 Projects Releases Wiki Activity

make_mirror.sh: also mirror security.debian.org and stable-updates

Browse source
This commit is contained in:
Johannes 'josch' Schauer 2019-01-24 12:40:09 +01:00
parent bde4c795ef
commit 21270ef56d
Signed by: josch
GPG key ID: F2CBA5C78FBD83E1
1 changed files with 176 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

167
make_mirror.sh
View file

@ -30,6 +30,7 @@ oldmirrordir="$oldcachedir/debian"
newmirrordir="$newcachedir/debian" newmirrordir="$newcachedir/debian"
mirror="http://deb.debian.org/debian" mirror="http://deb.debian.org/debian"
security_mirror="http://security.debian.org/debian-security"
arch1=$(dpkg --print-architecture) arch1=$(dpkg --print-architecture)
arch2=armhf arch2=armhf
if [ "$arch1" = "$arch2" ]; then if [ "$arch1" = "$arch2" ]; then
@ -48,8 +49,66 @@ if [ -e "$oldmirrordir/dists/unstable/Release" ]; then
esac esac
fi fi
for nativearch in $arch1 $arch2; do get_oldaptnames() {
for dist in stable testing unstable; do if [ ! -e "$1/$2" ]; then
return
fi
gzip -dc "$1/$2" \
| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename '' \
| paste -sd " \n" \
| while read name ver arch fname; do
if [ ! -e "$1/$fname" ]; then
continue
fi
# apt stores deb files with the colon encoded as %3a while
# mirrors do not contain the epoch at all #645895
case "$ver" in *:*) ver="${ver%%:*}%3a${ver#*:}";; esac
aptname="$rootdir/var/cache/apt/archives/${name}_${ver}_${arch}.deb"
# we have to cp and not mv because other
# distributions might still need this file
# we have to cp and not symlink because apt
# doesn't recognize symlinks
cp --link "$1/$fname" "$aptname"
echo "$aptname"
done
}
get_newaptnames() {
if [ ! -e "$1/$2" ]; then
return
fi
gzip -dc "$1/$2" \
| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename,MD5sum '' \
| paste -sd " \n" \
| while read name ver arch fname md5; do
dir="${fname%/*}"
# apt stores deb files with the colon encoded as %3a while
# mirrors do not contain the epoch at all #645895
case "$ver" in *:*) ver="${ver%%:*}%3a${ver#*:}";; esac
aptname="$rootdir/var/cache/apt/archives/${name}_${ver}_${arch}.deb"
if [ -e "$aptname" ]; then
# make sure that we found the right file by checking its hash
echo "$md5 $aptname" | md5sum --check >&2
mkdir -p "$1/$dir"
# since we move hardlinks around, the same hardlink might've been
# moved already into the same place by another distribution.
# mv(1) refuses to copy A to B if both are hardlinks of each other.
if [ "$aptname" -ef "$1/$fname" ]; then
# both files are already the same so we just need to
# delete the source
rm "$aptname"
else
mv "$aptname" "$1/$fname"
fi
echo "$aptname"
fi
done
}
update_cache() {
dist="$1"
nativearch="$2"
# use a subdirectory of $newcachedir so that we can use # use a subdirectory of $newcachedir so that we can use
# hardlinks # hardlinks
rootdir="$newcachedir/apt" rootdir="$newcachedir/apt"
@ -59,6 +118,9 @@ for nativearch in $arch1 $arch2; do
mkdir -p "$rootdir/$p" mkdir -p "$rootdir/$p"
done done
# read sources.list content from stdin
cat > "$rootdir/etc/apt/sources.list"
cat << END > "$rootdir/etc/apt/apt.conf" cat << END > "$rootdir/etc/apt/apt.conf"
Apt::Architecture "$nativearch"; Apt::Architecture "$nativearch";
Apt::Architectures "$nativearch"; Apt::Architectures "$nativearch";
@ -74,38 +136,20 @@ END
> "$rootdir/var/lib/dpkg/status" > "$rootdir/var/lib/dpkg/status"
cat << END > "$rootdir/etc/apt/sources.list"
deb [arch=$nativearch] $mirror $dist $components
END
APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get update
> "$rootdir/oldaptnames"
# before downloading packages and before replacing the old Packages # before downloading packages and before replacing the old Packages
# file, copy all old *.deb packages from the mirror to # file, copy all old *.deb packages from the mirror to
# /var/cache/apt/archives so that apt will not re-download *.deb # /var/cache/apt/archives so that apt will not re-download *.deb
# packages that we already have # packages that we already have
if [ -e "$oldmirrordir/dists/$dist/main/binary-$nativearch/Packages.gz" ]; then {
gzip -dc "$oldmirrordir/dists/$dist/main/binary-$nativearch/Packages.gz" \ get_oldaptnames "$oldmirrordir" "dists/$dist/main/binary-$nativearch/Packages.gz"
| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename '' \ if grep --quiet security.debian.org "$rootdir/etc/apt/sources.list"; then
| paste -sd " \n" \ get_oldaptnames "$oldmirrordir" "dists/stable-updates/main/binary-$nativearch/Packages.gz"
| while read name ver arch fname; do get_oldaptnames "$oldcachedir/debian-security" "dists/stable/updates/main/binary-$nativearch/Packages.gz"
if [ ! -e "$oldmirrordir/$fname" ]; then
continue
fi
# apt stores deb files with the colon encoded as %3a while
# mirrors do not contain the epoch at all #645895
case "$ver" in *:*) ver="${ver%%:*}%3a${ver#*:}";; esac
aptname="$rootdir/var/cache/apt/archives/${name}_${ver}_${arch}.deb"
# we have to cp and not mv because other
# distributions might still need this file
# we have to cp and not symlink because apt
# doesn't recognize symlinks
cp --link "$oldmirrordir/$fname" "$aptname"
echo "$aptname" >> "$rootdir/oldaptnames"
done
fi fi
} | sort -u > "$rootdir/oldaptnames"
pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \ pkgs=$(APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get indextargets \
--format '$(FILENAME)' 'Created-By: Packages' "Architecture: $nativearch" \ --format '$(FILENAME)' 'Created-By: Packages' "Architecture: $nativearch" \
@ -120,10 +164,20 @@ END
APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs APT_CONFIG="$rootdir/etc/apt/apt.conf" apt-get --yes install $pkgs
# to be able to also test gpg verification, we need to create a mirror # to be able to also test gpg verification, we need to create a mirror
mkdir -p "$newmirrordir/dists/$dist/" "$newmirrordir/dists/$dist/main/binary-$nativearch/" mkdir -p "$newmirrordir/dists/$dist/main/binary-$nativearch/"
curl --location "$mirror/dists/$dist/Release" > "$newmirrordir/dists/$dist/Release" curl --location "$mirror/dists/$dist/Release" > "$newmirrordir/dists/$dist/Release"
curl --location "$mirror/dists/$dist/Release.gpg" > "$newmirrordir/dists/$dist/Release.gpg" curl --location "$mirror/dists/$dist/Release.gpg" > "$newmirrordir/dists/$dist/Release.gpg"
curl --location "$mirror/dists/$dist/main/binary-$nativearch/Packages.gz" > "$newmirrordir/dists/$dist/main/binary-$nativearch/Packages.gz" curl --location "$mirror/dists/$dist/main/binary-$nativearch/Packages.gz" > "$newmirrordir/dists/$dist/main/binary-$nativearch/Packages.gz"
if grep --quiet security.debian.org "$rootdir/etc/apt/sources.list"; then
mkdir -p "$newmirrordir/dists/stable-updates/main/binary-$nativearch/"
curl --location "$mirror/dists/stable-updates/Release" > "$newmirrordir/dists/stable-updates/Release"
curl --location "$mirror/dists/stable-updates/Release.gpg" > "$newmirrordir/dists/stable-updates/Release.gpg"
curl --location "$mirror/dists/stable-updates/main/binary-$nativearch/Packages.gz" > "$newmirrordir/dists/stable-updates/main/binary-$nativearch/Packages.gz"
mkdir -p "$newcachedir/debian-security/dists/stable/updates/main/binary-$nativearch/"
curl --location "$security_mirror/dists/stable/updates/Release" > "$newcachedir/debian-security/dists/stable/updates/Release"
curl --location "$security_mirror/dists/stable/updates/Release.gpg" > "$newcachedir/debian-security/dists/stable/updates/Release.gpg"
curl --location "$security_mirror/dists/stable/updates/main/binary-$nativearch/Packages.gz" > "$newcachedir/debian-security/dists/stable/updates/main/binary-$nativearch/Packages.gz"
fi
# the deb files downloaded by apt must be moved to their right locations in the # the deb files downloaded by apt must be moved to their right locations in the
# pool directory # pool directory
@ -133,42 +187,18 @@ END
# requires re-creating the heuristic by which the directory is chosen, requires # requires re-creating the heuristic by which the directory is chosen, requires
# stripping the epoch from the filename and will break once mirrors change. # stripping the epoch from the filename and will break once mirrors change.
# This way, it doesn't matter where the mirror ends up storing the package. # This way, it doesn't matter where the mirror ends up storing the package.
> "$rootdir/newaptnames" {
gzip -dc "$newmirrordir/dists/$dist/main/binary-$nativearch/Packages.gz" \ get_newaptnames "$newmirrordir" "dists/$dist/main/binary-$nativearch/Packages.gz";
| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename,MD5sum '' \ if grep --quiet security.debian.org "$rootdir/etc/apt/sources.list"; then
| paste -sd " \n" \ get_newaptnames "$newmirrordir" "dists/stable-updates/main/binary-$nativearch/Packages.gz"
| while read name ver arch fname md5; do get_newaptnames "$newcachedir/debian-security" "dists/stable/updates/main/binary-$nativearch/Packages.gz"
dir="${fname%/*}"
# apt stores deb files with the colon encoded as %3a while
# mirrors do not contain the epoch at all #645895
case "$ver" in *:*) ver="${ver%%:*}%3a${ver#*:}";; esac
aptname="$rootdir/var/cache/apt/archives/${name}_${ver}_${arch}.deb"
if [ -e "$aptname" ]; then
# make sure that we found the right file by checking its hash
echo "$md5 $aptname" | md5sum --check
mkdir -p "$newmirrordir/$dir"
# since we move hardlinks around, the same hardlink might've been
# moved already into the same place by another distribution.
# mv(1) refuses to copy A to B if both are hardlinks of each other.
if [ "$aptname" -ef "$newmirrordir/$fname" ]; then
# both files are already the same so we just need to
# delete the source
rm "$aptname"
else
mv "$aptname" "$newmirrordir/$fname"
fi fi
echo "$aptname" >> "$rootdir/newaptnames" } | sort -u > "$rootdir/newaptnames"
fi
done
rm "$rootdir/var/cache/apt/archives/lock" rm "$rootdir/var/cache/apt/archives/lock"
rmdir "$rootdir/var/cache/apt/archives/partial" rmdir "$rootdir/var/cache/apt/archives/partial"
# remove all packages that were in the old Packages file but not in the # remove all packages that were in the old Packages file but not in the
# new one anymore # new one anymore
sort "$rootdir/oldaptnames" > "$rootdir/tmp"
mv "$rootdir/tmp" "$rootdir/oldaptnames"
sort "$rootdir/newaptnames" > "$rootdir/tmp"
mv "$rootdir/tmp" "$rootdir/newaptnames"
comm -23 "$rootdir/oldaptnames" "$rootdir/newaptnames" | xargs --delimiter="\n" --no-run-if-empty rm comm -23 "$rootdir/oldaptnames" "$rootdir/newaptnames" | xargs --delimiter="\n" --no-run-if-empty rm
# now the apt cache should be empty # now the apt cache should be empty
if [ ! -z "$(ls -1qA "$rootdir/var/cache/apt/archives/")" ]; then if [ ! -z "$(ls -1qA "$rootdir/var/cache/apt/archives/")" ]; then
@ -189,6 +219,20 @@ END
rm "$rootdir/oldaptnames" rm "$rootdir/oldaptnames"
rm "$rootdir/newaptnames" rm "$rootdir/newaptnames"
find "$rootdir" -depth -print0 | xargs -0 rmdir find "$rootdir" -depth -print0 | xargs -0 rmdir
}
for nativearch in $arch1 $arch2; do
for dist in stable testing unstable; do
cat << END | update_cache $dist $nativearch
deb [arch=$nativearch] $mirror $dist $components
END
if [ "$dist" = "stable" ]; then
cat << END | update_cache $dist $nativearch
deb [arch=$nativearch] $mirror $dist $components
deb [arch=$nativearch] $mirror stable-updates main
deb [arch=$nativearch] $security_mirror stable/updates main
END
fi
done done
done done
@ -345,10 +389,19 @@ for dist in stable testing unstable; do
if [ -e "$oldcachedir/debian/dists/$dist" ]; then if [ -e "$oldcachedir/debian/dists/$dist" ]; then
rm --one-file-system --recursive "$oldcachedir/debian/dists/$dist" rm --one-file-system --recursive "$oldcachedir/debian/dists/$dist"
fi fi
if [ "$dist" = "stable" ]; then
if [ -e "$oldcachedir/debian/dists/stable-updates" ]; then
rm --one-file-system --recursive "$oldcachedir/debian/dists/stable-updates"
fi
if [ -e "$oldcachedir/debian-security/dists/stable/updates" ]; then
rm --one-file-system --recursive "$oldcachedir/debian-security/dists/stable/updates"
fi
fi
done done
if [ -e $oldcachedir/debian-unstable.qcow ]; then if [ -e $oldcachedir/debian-unstable.qcow ]; then
rm --one-file-system $oldcachedir/debian-unstable.qcow rm --one-file-system $oldcachedir/debian-unstable.qcow
fi fi
rm --one-file-system --recursive $oldcachedir/debian/pool/main rm --one-file-system --recursive $oldcachedir/debian/pool/main
rm --one-file-system --recursive $oldcachedir/debian-security/pool/updates/main
# now the rest should only be empty directories # now the rest should only be empty directories
find $oldcachedir -depth -print0 | xargs -0 rmdir find $oldcachedir -depth -print0 | xargs -0 rmdir