run.sh 17.2 KB
#!/bin/sh

set -exu

upload() {
	rsync --compress-level=9 -Pha --copy-unsafe-links --delete --rsh "ssh -p 22014" bootstrap.debian.net/ bootstrap.debian.net:/var/www/;
}

now=`date -d "today 00:00" "+%Y%m%dT%H%M%SZ"`

#######################################################
#               Generate main stats                   #
#######################################################

mirror="http://snapshot.debian.org/archive"

curl --retry 4 $mirror/debian/$now/dists/sid/Release > release
curl --retry 4 $mirror/debian/$now/dists/sid/Release.gpg > release.gpg
gpgv --keyring=/etc/apt/trusted.gpg.d/debian-archive-wheezy-automatic.gpg \
	--keyring=/etc/apt/trusted.gpg.d/debian-archive-jessie-automatic.gpg \
	release.gpg release

curl --retry 4 $mirror/debian-ports/$now/dists/sid/Release > ports-release
curl --retry 4 $mirror/debian-ports/$now/dists/sid/Release.gpg > ports-release.gpg
gpgv --keyring=/etc/apt/trusted.gpg.d/debian-ports-archive-2015.gpg \
	--keyring=/etc/apt/trusted.gpg.d/debian-ports-archive-2016.gpg \
	ports-release.gpg ports-release

grep-dctrl -F MD5Sum "" -s MD5Sum -n release > release-md5
grep-dctrl -F MD5Sum "" -s MD5Sum -n ports-release > ports-release-md5

curl --retry 4 $mirror/debian/$now/dists/sid/main/source/Sources.gz > sources.gz

if [ "`md5sum sources.gz | awk '{print $1}'`" != "`grep main/source/Sources.gz release-md5 | awk '{print $1}'`" ] \
|| [ "`wc -c < sources.gz`" != "`grep main/source/Sources.gz release-md5 | awk '{print $2}'`" ]; then
	echo "md5sum or size mismatch"
	exit 1
fi

mainarchs=`grep '^Architectures: ' release | cut -d ' ' -f 2-`

portarchs=`grep '^Architectures: ' ports-release | cut -d ' ' -f 2-`

for arch in $portarchs; do
	# create the directory in case an error log has to be saved
	mkdir -p ./bootstrap.debian.net/$arch
	curl --retry 4 $mirror/debian-ports/$now/dists/sid/main/binary-$arch/Packages.gz > packages_${arch}.gz
	if [ "`md5sum packages_${arch}.gz | awk '{print $1}'`" != "`grep main/binary-$arch/Packages.gz ports-release-md5 | awk '{print $1}'`" ] \
	|| [ "`wc -c < packages_${arch}.gz`" != "`grep main/binary-$arch/Packages.gz ports-release-md5 | awk '{print $2}'`" ]; then
		echo "md5sum or size mismatch"
		exit 1
	fi
	rm -rf ./out
	{ botch-native --verbose --debug --output=./out --drop-b-d-indep --online --allowsrcmismatch --ignoresrclessbin --no-drop --clean --strong --sapsb --no-fixpoint --optgraph $arch packages_${arch}.gz sources.gz 2>&1 && echo "success" || echo "failed"; } | tee log.txt
	exitstatus=`tail -1 log.txt`
	if [ "$exitstatus" != "success" ]; then
		mv log.txt ./bootstrap.debian.net/$arch
		continue
	fi
	# add snapshot timestamp to json output:
	python3 -c "import json,sys; f=open('./out/stats.json');d=json.load(f);f.close();d['timestamp']='$now';f=open('./out/stats.json','w');json.dump(d,f,sort_keys=True,indent=2);f.close();"
	# native.sh was successful, replace old directory with new content
	rm -rf ./bootstrap.debian.net/$arch
	mv ./out ./bootstrap.debian.net/$arch
	# symlink index to stats
	ln -s stats.html ./bootstrap.debian.net/$arch/index.html
done

for arch in $mainarchs; do
	# create the directory in case an error log has to be saved
	mkdir -p ./bootstrap.debian.net/$arch
	curl --retry 4 $mirror/debian/$now/dists/sid/main/binary-$arch/Packages.gz > packages_${arch}.gz
	if [ "`md5sum packages_${arch}.gz | awk '{print $1}'`" != "`grep main/binary-$arch/Packages.gz release-md5 | awk '{print $1}'`" ] \
	|| [ "`wc -c < packages_${arch}.gz`" != "`grep main/binary-$arch/Packages.gz release-md5 | awk '{print $2}'`" ]; then
		echo "md5sum or size mismatch"
		exit 1
	fi
	rm -rf ./out
	{ botch-native --verbose --debug --output=./out --drop-b-d-indep --online --no-drop --clean --strong --sapsb --no-fixpoint --optgraph $arch packages_${arch}.gz sources.gz 2>&1 && echo "success" || echo "failed"; } | tee log.txt
	exitstatus=`tail -1 log.txt`
	if [ "$exitstatus" != "success" ]; then
		mv log.txt ./bootstrap.debian.net/$arch
		continue
	fi
	# add snapshot timestamp to json output:
	python3 -c "import json,sys; f=open('./out/stats.json');d=json.load(f);f.close();d['timestamp']='$now';f=open('./out/stats.json','w');json.dump(d,f,sort_keys=True,indent=2);f.close();"
	# native.sh was successful, replace old directory with new content
	rm -rf ./bootstrap.debian.net/$arch
	mv ./out ./bootstrap.debian.net/$arch
	# symlink index to stats
	ln -s stats.html ./bootstrap.debian.net/$arch/index.html
done

rm -rf ./bootstrap.debian.net/source
mkdir -p ./bootstrap.debian.net/source

python3 generate_html.py "$now" "$mainarchs" "$portarchs"

#######################################################
#              Generate history.html                  #
#######################################################

( cd snapshot; ./download.sh; ./run.sh; )
cp snapshot/history.txt bootstrap.debian.net/
cp snapshot/history_optis.txt bootstrap.debian.net/
cp snapshot/history_optuniv.txt bootstrap.debian.net/
cp snapshot/history_disj.txt bootstrap.debian.net/
cp snapshot/history_cross.txt bootstrap.debian.net/
cp snapshot/history.svg bootstrap.debian.net/
cp snapshot/history_cross.svg bootstrap.debian.net/

#######################################################
#              Generate essential.html                #
#######################################################

rm -f essential-failed
( cd essential && ./run.sh > essential.html || touch ../essential-failed; )
if [ ! -e essential-failed ]; then
	mv essential/essential.html bootstrap.debian.net/essential.html
	rm -rf bootstrap.debian.net/essential
	mkdir -p bootstrap.debian.net/essential
	cp essential/closure-essential-all.list essential/closure-essential-all-src.list \
		essential/closure-essential-noall.list essential/closure-essential-noall-src.list \
		essential/minimal-all-src.list essential/minimal-amd64.list \
		essential/minimal-noall-src.list essential/opt-essential-all.list \
		essential/opt-essential-all-src.list essential/opt-essential-noall.list \
		essential/opt-essential-noall-src.list essential/strong-essential-all.list \
		essential/strong-essential-all-src.list essential/strong-essential-noall.list \
		essential/strong-essential-noall-src.list \
		bootstrap.debian.net/essential
	cp -r essential/y-u-b-d-transitive-essential bootstrap.debian.net/essential
else
	echo "essential failed" >&2
	# if it was not possible to generate the essential package sets for
	# amd64 , then it follows that no coinstallation set of essential
	# exists which in turn means that we can stop processing here
	upload
	exit 0
fi

#######################################################
#             Generate some more lists                #
#######################################################

# we convert amd64 to armhf to avoid m-a:same version skews
botch-convert-arch amd64 armhf packages_amd64.gz packages_armhf
# we remove any m-a:foreign packages from the host-arch package list
# we remove arch:all packages too as they'll come from the build-arch package list
grep-dctrl -X \( -FArchitecture all --or --not -FMulti-Arch foreign \) packages_armhf > packages_armhf-no-ma-foreign
# and add armhf to all source packages
botch-add-arch armhf sources.gz tmp.gz
# remove all source packages that only build Architecture:all packages
zcat tmp.gz | grep-dctrl --not -X -F Architecture all > sources_noall
rm tmp.gz
# generate a binary package list containing all amd64 packages plus all
# armhf packages that are not m-a:foreign plus crossbuild-essential-armhf
zcat packages_amd64.gz | cat - packages_armhf-no-ma-foreign crossbuild-essential-armhf > packages_amd64_armhf_combined

#######################################################
#               Generate cross.html                   #
#######################################################

# extract only the source package name (strip version and src: prefix)
checkonly=`cut -d ' ' -f 1 essential/strong-essential-noall-src.list | cut -d ':' -f 2 | sort -u | tr '\n' ','`
# and remove trailing comma
checkonly="${checkonly%?}"

/usr/bin/time botch-buildcheck-more-problems --latest --explain --failures \
	--deb-native-arch=amd64 --deb-host-arch=armhf --deb-drop-b-d-indep \
	--progress --verbose --verbose --deb-profiles=cross,nocheck,nodoc \
	packages_amd64_armhf_combined sources_noall \
	--checkonly=${checkonly} > cross.yaml \
	|| [ "$?" -lt 64 ] || exit 1

description="<p>Below are two tables showing the two problem classes that"
description="$description prevent cross compilation on the dependency level."
description="$description This example tries to satisfy the transitive strong dependencies of build-essential."
description="$description For a list, read <a href=\"/essential.html\">essential.html</a>"
description="$description or download a machine readable version from"
description="$description <a href=\"/essential/strong-essential-noall-src.list\">here</a>."
description="$description Crossbuild dependencies were checked on amd64 for a dummy"
description="$description architecture generated from amd64 (called armhf here) in"
description="$description current Debian Sid.</a>"
description="$description <p>A machine parsable version can be retrieved in"
description="$description <a href=\"cross.yaml\">dose yaml format</a></p>"

botch-dose2html --wnpp --btsuser debian-cross@lists.debian.org \
	--btstag cross-satisfiability --desc "$description" --timestamp $now \
	--srcsdir cross --packages packages_amd64_armhf_combined \
	cross.yaml cross.html

if [ -s cross.html ]; then
	cp cross.html ./bootstrap.debian.net
	rm -rf ./bootstrap.debian.net/cross
	mv cross ./bootstrap.debian.net
	cp cross.yaml ./bootstrap.debian.net
fi

#######################################################
#             Generate cross_all.html                 #
#######################################################

/usr/bin/time botch-buildcheck-more-problems --latest --explain --failures \
	--deb-native-arch=amd64 --deb-host-arch=armhf --deb-drop-b-d-indep \
	--progress --verbose --verbose --deb-profiles=cross,nocheck,nodoc \
	packages_amd64_armhf_combined sources_noall \
	> cross_all.yaml \
	|| [ "$?" -lt 64 ] || exit 1

description="<p>Below are two tables showing the two problem classes that"
description="$description prevent cross compilation on the dependency level."
description="$description This example tries to satisfy the crossbuild dependencies of"
description="$description all source packages on amd64 as the build architecture for a dummy"
description="$description architecture generated from amd64 (called armhf here) as the host architecture in"
description="$description current Debian sid.</p>"
description="$description <p>A machine parsable version can be retrieved in"
description="$description <a href=\"cross_all.yaml\">dose yaml format</a></p>"

botch-dose2html --wnpp --btsuser debian-cross@lists.debian.org \
	--btstag cross-satisfiability --desc "$description" --timestamp $now \
	--srcsdir cross_all --packages packages_amd64_armhf_combined \
	cross_all.yaml cross_all.html

if [ -s cross_all.html ]; then
	cp cross_all.html ./bootstrap.debian.net
	rm -rf ./bootstrap.debian.net/cross_all
	mv cross_all ./bootstrap.debian.net
	cp cross_all.yaml ./bootstrap.debian.net
fi

#######################################################
#           Generate foreign_install.html             #
#######################################################

/usr/bin/time botch-distcheck-more-problems --progress --verbose --verbose \
	--latest --deb-native-arch=amd64 --deb-foreign-archs=armhf --explain \
	--failures --bg deb://packages_amd64_armhf_combined \
	--fg deb://packages_armhf-no-ma-foreign > foreign_install.yaml \
	|| [ "$?" -lt 64 ] || exit 1

description="<p>Below are two tables showing the two problem classes that"
description="$description prevent installation of foreign architecture binaries</p>"
description="$description <p>We try to install all binary packages from a dummy"
description="$description architecture generated from amd64 (called armhf here) on amd64.</p>"
description="$description <p>A machine parsable version can be retrieved in"
description="$description <a href=\"foreign_install.yaml\">dose yaml format</a></p>"

botch-dose2html --wnpp foreign_install.yaml foreign_install.html \
	--desc "$description" --timestamp $now --srcsdir foreign_install \
	--packages packages_amd64_armhf_combined

if [ -s foreign_install.html ]; then
	cp foreign_install.html ./bootstrap.debian.net
	rm -rf ./bootstrap.debian.net/foreign_install
	mv foreign_install ./bootstrap.debian.net
	cp foreign_install.yaml ./bootstrap.debian.net
fi

#######################################################
#             Generate co_ma_same.html                #
#######################################################

# the idea for this came from Ralf Treinen

# since the armhf packages file was generated from the amd64 packages file,
# we do not need to find the common denominator
zcat packages_amd64.gz | grep-dctrl -F Multi-Arch same -s Package -n | sort | uniq | while read p; do
    echo 'Package:' ${p}-pseudo
    echo 'Version: 0.invalid.0'
    echo 'Architecture: amd64'
    echo 'Depends:' ${p}:amd64, ${p}:armhf
    echo
done > dummy_packages

/usr/bin/time botch-distcheck-more-problems --progress --verbose --verbose \
	--deb-native-arch=amd64 --deb-foreign-archs=armhf \
	--bg deb://packages_amd64_armhf_combined --fg deb://dummy_packages \
	--explain --failures > co_ma_same.yaml \
	|| [ "$?" -lt 64 ] || exit 1

description="<p>Below are two tables showing the two problem classes that"
description="$description prevent co-installation of Multi-Arch:same binaries</p>"
description="$description <p>For every M-A:same package we create a dummy package"
description="$description with the -pseudo prefix which depends on that package"
description="$description for amd64 and a dummy architecture generated from amd64 (called armhf here).</p>"
description="$description <p>A machine parsable version can be retrieved in"
description="$description <a href=\"co_ma_same.yaml\">dose yaml format</a></p>"

botch-dose2html --wnpp co_ma_same.yaml co_ma_same.html \
	--desc "$description" --timestamp $now --srcsdir co_ma_same \
	--packages packages_amd64_armhf_combined

if [ -s co_ma_same.html ]; then
	cp co_ma_same.html ./bootstrap.debian.net
	rm -rf ./bootstrap.debian.net/co_ma_same
	mv co_ma_same ./bootstrap.debian.net
	cp co_ma_same.yaml ./bootstrap.debian.net
fi

#######################################################
#            Generate ma_interpreter.html             #
#######################################################

dose-ceve --deb-native-arch=amd64 -G pkg -T grml deb://packages_amd64.gz debsrc://sources.gz > ma_interpreter.xml

botch-multiarch-interpreter-problem --packages packages_amd64.gz ma_interpreter.xml > ma_interpreter.html

if [ -s ma_interpreter.html ]; then
	cp ma_interpreter.html ./bootstrap.debian.net
fi

#######################################################
#           Generate cross_cheated.html               #
#######################################################

(cd botch; { botch-cross --verbose --debug --online --drop-b-d-indep --optgraph amd64 armhf ../packages_amd64.gz ../sources.gz 2>&1; echo $?; } | tee tmp; )
exitstatus=`tail -1 botch/tmp`
if [ "$exitstatus" -eq 0 ]; then
	cp botch/order1.lst bootstrap.debian.net/cross_cheated_order1.lst
	cp botch/order2.lst bootstrap.debian.net/cross_cheated_order2.lst
	cp botch/stats.html bootstrap.debian.net/cross_cheated_stats.html
	cp botch/ma.diff bootstrap.debian.net/cross_cheated_ma.diff
fi
cp botch/tmp bootstrap.debian.net/cross_cheated.log
rm -f botch/ma.diff

#######################################################
#          Generate importance_metric.html            #
#######################################################

rm -f popcon_source.gz
UNIVERSE="--deb-native-arch=amd64 packages_amd64.gz sources.gz"

botch-create-graph $UNIVERSE --strongtype > strongbuildgraph.xml
botch-buildgraph2srcgraph strongbuildgraph.xml $UNIVERSE > strongsrcgraph.xml
botch-create-graph $UNIVERSE --closuretype > closurebuildgraph.xml
botch-buildgraph2srcgraph closurebuildgraph.xml $UNIVERSE > closuresrcgraph.xml
botch-calcportsmetric --online strongsrcgraph.xml closuresrcgraph.xml > importance_metric_all.txt
if [ -s importance_metric_all.txt ]; then
	mv importance_metric_all.txt bootstrap.debian.net/importance_metric_all.txt
fi

UNIVERSE="$UNIVERSE --drop-b-d-indep"
zcat packages_amd64.gz | grep-dctrl -FArchitecture all > available-all
botch-packages-union available-all essential/minimal-amd64 available
botch-create-graph $UNIVERSE --available=available --strongtype > strongbuildgraph.xml
botch-buildgraph2srcgraph strongbuildgraph.xml $UNIVERSE > strongsrcgraph.xml
botch-create-graph $UNIVERSE --available=available --closuretype > closurebuildgraph.xml
botch-buildgraph2srcgraph closurebuildgraph.xml $UNIVERSE > closuresrcgraph.xml
botch-calcportsmetric --online strongsrcgraph.xml closuresrcgraph.xml > importance_metric_noall.txt
if [ -s importance_metric_noall.txt ]; then
	mv importance_metric_noall.txt bootstrap.debian.net/importance_metric_noall.txt
fi

rm -f strongsrcgraph.xml closuresrcgraph.xml strongbuildgraph.xml closurebuildgraph.xml
rm -f available available-all

#######################################################
#                       Upload                        #
#######################################################

upload

# run every monday at 03:00 (to be sure that the snapshot at midnight exists):
# while sleep $(($(echo -e "next monday 03:00:00\nnow" | date -f - +%s-)0)); do annotate-output "+%FT%T" time ./run.sh 2>&1 | tee log.`date +%Y%m%d`; done