josch/mmdebstrap
2
3
Fork 6
Code Issues 8 Pull requests 1 Projects Releases Wiki Activity

coverage.sh: allow running without qemu

Browse source 
On machines without kvm using qemu is not practical. Now, most tests can
be run without a container. Also add support for missing unshare, proot
and binfmt.
This commit is contained in:
Johannes 'josch' Schauer 2018-12-05 10:33:03 +01:00
parent 6634f4f49f
commit 2618a587a0
Signed by: josch
GPG key ID: F2CBA5C78FBD83E1
3 changed files with 321 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

300
coverage.sh
View file

@ -9,8 +9,12 @@ mirrordir="./shared/debian"
# we use -f because the file might not exist # we use -f because the file might not exist
rm -f shared/cover_db.img rm -f shared/cover_db.img
# prepare image for cover_db : "${HAVE_QEMU:=yes}"
guestfish -N shared/cover_db.img=disk:100M -- mkfs vfat /dev/sda
if [ "$HAVE_QEMU" = "yes" ]; then
# prepare image for cover_db
guestfish -N shared/cover_db.img=disk:100M -- mkfs vfat /dev/sda
fi
# only copy if necessary # only copy if necessary
if [ ! -e shared/mmdebstrap ] || [ mmdebstrap -nt shared/mmdebstrap ]; then if [ ! -e shared/mmdebstrap ] || [ mmdebstrap -nt shared/mmdebstrap ]; then
@ -44,6 +48,15 @@ SOURCE_DATE_EPOCH=$(date --date="$(grep-dctrl -s Date -n '' "$mirrordir/dists/un
# for traditional sort order that uses native byte values # for traditional sort order that uses native byte values
export LC_ALL=C export LC_ALL=C
: "${HAVE_UNSHARE:=yes}"
: "${HAVE_PROOT:=yes}"
: "${HAVE_BINFMT:=yes}"
defaultmode="auto"
if [ "$HAVE_UNSHARE" != "yes" ]; then
defaultmode="root"
fi
# by default, use the mmdebstrap executable in the current directory together # by default, use the mmdebstrap executable in the current directory together
# with perl Devel::Cover but allow to overwrite this # with perl Devel::Cover but allow to overwrite this
: "${CMD:=perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap}" : "${CMD:=perl -MDevel::Cover=-silent,-nogcov ./mmdebstrap}"
@ -56,6 +69,12 @@ for dist in stable testing unstable; do
continue continue
fi fi
print_header "mode=root,variant=$variant: check against debootstrap $dist" print_header "mode=root,variant=$variant: check against debootstrap $dist"
if [ ! -e "shared/cache/debian-$dist-$variant.tar" ]; then
echo "shared/cache/debian-$dist-$variant.tar does not exist. Skipping..."
continue
fi
cat << END > shared/test.sh cat << END > shared/test.sh
#!/bin/sh #!/bin/sh
set -eu set -eu
@ -138,8 +157,15 @@ diff --no-dereference --brief --recursive /tmp/debian-$dist-debootstrap /tmp/deb
# #
# we cannot use this (yet) because it cannot copy with paths that have [ or @ in them # we cannot use this (yet) because it cannot copy with paths that have [ or @ in them
#fmtree -c -p /tmp/debian-$dist-debootstrap -k flags,gid,link,mode,size,time,uid | sudo fmtree -p /tmp/debian-$dist-mm #fmtree -c -p /tmp/debian-$dist-debootstrap -k flags,gid,link,mode,size,time,uid | sudo fmtree -p /tmp/debian-$dist-mm
rm /tmp/debian-$dist-mm.tar
rm -r /tmp/debian-$dist-debootstrap /tmp/debian-$dist-mm
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh SUDO
fi
done done
done done
@ -150,8 +176,13 @@ set -eu
export LC_ALL=C export LC_ALL=C
$CMD --mode=root --variant=apt unstable /tmp/debian-unstable $mirror $CMD --mode=root --variant=apt unstable /tmp/debian-unstable $mirror
tar -C /tmp/debian-unstable --one-file-system -c . | tar -t | sort > tar1.txt tar -C /tmp/debian-unstable --one-file-system -c . | tar -t | sort > tar1.txt
rm -r /tmp/debian-unstable
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh SUDO
fi
print_header "mode=root,variant=apt: test progress bars on fake tty" print_header "mode=root,variant=apt: test progress bars on fake tty"
cat << END > shared/test.sh cat << END > shared/test.sh
@ -161,8 +192,13 @@ export LC_ALL=C
script -qfc "$CMD --mode=root --variant=apt unstable /tmp/unstable-chroot.tar $mirror" /dev/null script -qfc "$CMD --mode=root --variant=apt unstable /tmp/unstable-chroot.tar $mirror" /dev/null
tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh SUDO
fi
print_header "mode=root,variant=apt: existing empty directory" print_header "mode=root,variant=apt: existing empty directory"
cat << END > shared/test.sh cat << END > shared/test.sh
@ -173,8 +209,13 @@ mkdir /tmp/debian-unstable
$CMD --mode=root --variant=apt unstable /tmp/debian-unstable $mirror $CMD --mode=root --variant=apt unstable /tmp/debian-unstable $mirror
tar -C /tmp/debian-unstable --one-file-system -c . | tar -t | sort > tar2.txt tar -C /tmp/debian-unstable --one-file-system -c . | tar -t | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm -r /tmp/debian-unstable
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh SUDO
fi
print_header "mode=unshare,variant=apt: create tarball" print_header "mode=unshare,variant=apt: create tarball"
cat << END > shared/test.sh cat << END > shared/test.sh
@ -186,19 +227,31 @@ sysctl -w kernel.unprivileged_userns_clone=1
runuser -u user -- $CMD --mode=unshare --variant=apt unstable /tmp/unstable-chroot.tar $mirror runuser -u user -- $CMD --mode=unshare --variant=apt unstable /tmp/unstable-chroot.tar $mirror
tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
echo "HAVE_QEMU != yes -- Skipping test..."
fi
print_header "mode=auto,variant=apt: read from stdin, write to stdout" print_header "mode=$defaultmode,variant=apt: read from stdin, write to stdout"
cat << END > shared/test.sh cat << END > shared/test.sh
#!/bin/sh #!/bin/sh
set -eu set -eu
export LC_ALL=C export LC_ALL=C
echo "deb $mirror unstable main" | $CMD --variant=apt > /tmp/unstable-chroot.tar echo "deb $mirror unstable main" | $CMD --mode=$defaultmode --variant=apt > /tmp/unstable-chroot.tar
tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
elif [ "$defaultmode" = "root" ]; then
./run_null.sh SUDO
else
./run_null.sh
fi
print_header "mode=auto,variant=apt: default mirror" print_header "mode=auto,variant=apt: default mirror"
cat << END > shared/test.sh cat << END > shared/test.sh
@ -206,11 +259,18 @@ cat << END > shared/test.sh
set -eu set -eu
export LC_ALL=C export LC_ALL=C
echo "127.0.0.1 deb.debian.org" >> /etc/hosts echo "127.0.0.1 deb.debian.org" >> /etc/hosts
$CMD --variant=apt unstable /tmp/unstable-chroot.tar $CMD --mode=$defaultmode --variant=apt unstable /tmp/unstable-chroot.tar
tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
elif [ "$defaultmode" = "root" ]; then
./run_null.sh SUDO
else
./run_null.sh
fi
print_header "mode=auto,variant=apt: pass distribution but implicitly write to stdout" print_header "mode=auto,variant=apt: pass distribution but implicitly write to stdout"
cat << END > shared/test.sh cat << END > shared/test.sh
@ -218,33 +278,52 @@ cat << END > shared/test.sh
set -eu set -eu
export LC_ALL=C export LC_ALL=C
echo "127.0.0.1 deb.debian.org" >> /etc/hosts echo "127.0.0.1 deb.debian.org" >> /etc/hosts
$CMD --variant=apt unstable > /tmp/unstable-chroot.tar $CMD --mode=$defaultmode --variant=apt unstable > /tmp/unstable-chroot.tar
tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
echo "HAVE_QEMU != yes -- Skipping test..."
fi
print_header "mode=auto,variant=apt: mirror is -" print_header "mode=auto,variant=apt: mirror is -"
cat << END > shared/test.sh cat << END > shared/test.sh
#!/bin/sh #!/bin/sh
set -eu set -eu
export LC_ALL=C export LC_ALL=C
echo "deb $mirror unstable main" | $CMD --variant=apt unstable /tmp/unstable-chroot.tar - echo "deb $mirror unstable main" | $CMD --mode=$defaultmode --variant=apt unstable /tmp/unstable-chroot.tar -
tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
elif [ "$defaultmode" = "root" ]; then
./run_null.sh SUDO
else
./run_null.sh
fi
print_header "mode=auto,variant=apt: mirror is deb..." print_header "mode=auto,variant=apt: mirror is deb..."
cat << END > shared/test.sh cat << END > shared/test.sh
#!/bin/sh #!/bin/sh
set -eu set -eu
export LC_ALL=C export LC_ALL=C
$CMD --variant=apt unstable /tmp/unstable-chroot.tar "deb $mirror unstable main" $CMD --mode=$defaultmode --variant=apt unstable /tmp/unstable-chroot.tar "deb $mirror unstable main"
tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
elif [ "$defaultmode" = "root" ]; then
./run_null.sh SUDO
else
./run_null.sh
fi
print_header "mode=auto,variant=apt: mirror is real file" print_header "mode=auto,variant=apt: mirror is real file"
cat << END > shared/test.sh cat << END > shared/test.sh
@ -252,22 +331,36 @@ cat << END > shared/test.sh
set -eu set -eu
export LC_ALL=C export LC_ALL=C
echo "deb $mirror unstable main" > /tmp/sources.list echo "deb $mirror unstable main" > /tmp/sources.list
$CMD --variant=apt unstable /tmp/unstable-chroot.tar /tmp/sources.list $CMD --mode=$defaultmode --variant=apt unstable /tmp/unstable-chroot.tar /tmp/sources.list
tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm /tmp/unstable-chroot.tar /tmp/sources.list
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
elif [ "$defaultmode" = "root" ]; then
./run_null.sh SUDO
else
./run_null.sh
fi
print_header "mode=auto,variant=apt: no mirror but data on stdin" print_header "mode=auto,variant=apt: no mirror but data on stdin"
cat << END > shared/test.sh cat << END > shared/test.sh
#!/bin/sh #!/bin/sh
set -eu set -eu
export LC_ALL=C export LC_ALL=C
echo "deb $mirror unstable main" | $CMD --variant=apt unstable /tmp/unstable-chroot.tar echo "deb $mirror unstable main" | $CMD --mode=$defaultmode --variant=apt unstable /tmp/unstable-chroot.tar
tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt tar -tf /tmp/unstable-chroot.tar | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
elif [ "$defaultmode" = "root" ]; then
./run_null.sh SUDO
else
./run_null.sh
fi
print_header "mode=root,variant=apt: add foreign architecture" print_header "mode=root,variant=apt: add foreign architecture"
cat << END > shared/test.sh cat << END > shared/test.sh
@ -279,8 +372,13 @@ $CMD --mode=root --variant=apt --architectures=amd64,armhf unstable /tmp/debian-
rm /tmp/debian-unstable/var/lib/dpkg/arch rm /tmp/debian-unstable/var/lib/dpkg/arch
tar -C /tmp/debian-unstable --one-file-system -c . | tar -t | sort > tar2.txt tar -C /tmp/debian-unstable --one-file-system -c . | tar -t | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm -r /tmp/debian-unstable
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh SUDO
fi
print_header "mode=root,variant=apt: test --aptopt" print_header "mode=root,variant=apt: test --aptopt"
cat << END > shared/test.sh cat << END > shared/test.sh
@ -292,8 +390,13 @@ echo "Acquire::Check-Valid-Until false;" | cmp /tmp/debian-unstable/etc/apt/apt.
rm /tmp/debian-unstable/etc/apt/apt.conf.d/99mmdebstrap rm /tmp/debian-unstable/etc/apt/apt.conf.d/99mmdebstrap
tar -C /tmp/debian-unstable --one-file-system -c . | tar -t | sort > tar2.txt tar -C /tmp/debian-unstable --one-file-system -c . | tar -t | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm -r /tmp/debian-unstable
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh SUDO
fi
print_header "mode=root,variant=apt: test --dpkgopt" print_header "mode=root,variant=apt: test --dpkgopt"
cat << END > shared/test.sh cat << END > shared/test.sh
@ -305,8 +408,13 @@ echo "path-exclude=/usr/share/doc/*" | cmp /tmp/debian-unstable/etc/dpkg/dpkg.cf
rm /tmp/debian-unstable/etc/dpkg/dpkg.cfg.d/99mmdebstrap rm /tmp/debian-unstable/etc/dpkg/dpkg.cfg.d/99mmdebstrap
tar -C /tmp/debian-unstable --one-file-system -c . | tar -t | sort > tar2.txt tar -C /tmp/debian-unstable --one-file-system -c . | tar -t | sort > tar2.txt
grep -v '^./usr/share/doc/.' tar1.txt | diff -u - tar2.txt grep -v '^./usr/share/doc/.' tar1.txt | diff -u - tar2.txt
rm -r /tmp/debian-unstable
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh SUDO
fi
print_header "mode=root,variant=apt: test --include" print_header "mode=root,variant=apt: test --include"
cat << END > shared/test.sh cat << END > shared/test.sh
@ -322,8 +430,13 @@ rm /tmp/debian-unstable/var/lib/dpkg/info/doc-debian.list
rm /tmp/debian-unstable/var/lib/dpkg/info/doc-debian.md5sums rm /tmp/debian-unstable/var/lib/dpkg/info/doc-debian.md5sums
tar -C /tmp/debian-unstable --one-file-system -c . | tar -t | sort > tar2.txt tar -C /tmp/debian-unstable --one-file-system -c . | tar -t | sort > tar2.txt
diff -u tar1.txt tar2.txt diff -u tar1.txt tar2.txt
rm -r /tmp/debian-unstable
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh SUDO
fi
# test all variants # test all variants
@ -335,8 +448,13 @@ set -eu
export LC_ALL=C export LC_ALL=C
$CMD --mode=root --variant=$variant unstable /tmp/unstable-chroot.tar $mirror $CMD --mode=root --variant=$variant unstable /tmp/unstable-chroot.tar $mirror
tar -tf /tmp/unstable-chroot.tar | sort > "$variant.txt" tar -tf /tmp/unstable-chroot.tar | sort > "$variant.txt"
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh SUDO
fi
# check if the other modes produce the same result in each variant # check if the other modes produce the same result in each variant
for mode in unshare fakechroot proot; do for mode in unshare fakechroot proot; do
# fontconfig doesn't install reproducibly because differences # fontconfig doesn't install reproducibly because differences
@ -356,20 +474,35 @@ END
;; ;;
esac esac
print_header "mode=$mode,variant=$variant: create tarball" print_header "mode=$mode,variant=$variant: create tarball"
if [ "$mode" = "unshare" ] && [ "$HAVE_UNSHARE" != "yes" ]; then
echo "HAVE_UNSHARE != yes -- Skipping test..."
continue
fi
if [ "$mode" = "proot" ] && [ "$HAVE_PROOT" != "yes" ]; then
echo "HAVE_PROOT != yes -- Skipping test..."
continue
fi
cat << END > shared/test.sh cat << END > shared/test.sh
#!/bin/sh #!/bin/sh
set -eu set -eu
export LC_ALL=C export LC_ALL=C
adduser --gecos user --disabled-password user [ "\$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1 && adduser --gecos user --disabled-password user
[ "$mode" = unshare ] && sysctl -w kernel.unprivileged_userns_clone=1 [ "$mode" = unshare ] && sysctl -w kernel.unprivileged_userns_clone=1
runuser -u user -- $CMD --mode=$mode --variant=$variant unstable /tmp/unstable-chroot.tar $mirror prefix=
[ "\$(id -u)" -eq 0 ] && prefix="runuser -u user --"
\$prefix $CMD --mode=$mode --variant=$variant unstable /tmp/unstable-chroot.tar $mirror
# in fakechroot mode, we use a fake ldconfig, so we have to # in fakechroot mode, we use a fake ldconfig, so we have to
# artificially add some files # artificially add some files
{ tar -tf /tmp/unstable-chroot.tar; { tar -tf /tmp/unstable-chroot.tar;
[ "$mode" = "fakechroot" ] && printf "./etc/ld.so.cache\n./var/cache/ldconfig/\n"; [ "$mode" = "fakechroot" ] && printf "./etc/ld.so.cache\n./var/cache/ldconfig/\n";
} | sort | diff -u "./$variant.txt" - } | sort | diff -u "./$variant.txt" -
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh
fi
# Devel::Cover doesn't survive mmdebstrap re-exec-ing itself # Devel::Cover doesn't survive mmdebstrap re-exec-ing itself
# with fakechroot, thus, we do an additional run where we # with fakechroot, thus, we do an additional run where we
# explicitly run mmdebstrap with fakechroot from the start # explicitly run mmdebstrap with fakechroot from the start
@ -379,13 +512,20 @@ END
#!/bin/sh #!/bin/sh
set -eu set -eu
export LC_ALL=C export LC_ALL=C
adduser --gecos user --disabled-password user [ "\$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1 && adduser --gecos user --disabled-password user
runuser -u user -- fakechroot fakeroot $CMD --mode=$mode --variant=$variant unstable /tmp/unstable-chroot.tar $mirror prefix=
[ "\$(id -u)" -eq 0 ] && prefix="runuser -u user --"
\$prefix fakechroot fakeroot $CMD --mode=$mode --variant=$variant unstable /tmp/unstable-chroot.tar $mirror
{ tar -tf /tmp/unstable-chroot.tar; { tar -tf /tmp/unstable-chroot.tar;
printf "./etc/ld.so.cache\n./var/cache/ldconfig/\n"; printf "./etc/ld.so.cache\n./var/cache/ldconfig/\n";
} | sort | diff -u "./$variant.txt" - } | sort | diff -u "./$variant.txt" -
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh
fi
fi fi
done done
# some variants are equal and some are strict superset of the last # some variants are equal and some are strict superset of the last
@ -435,23 +575,25 @@ done
# test extract variant also with chrootless mode # test extract variant also with chrootless mode
for mode in root unshare fakechroot proot chrootless; do for mode in root unshare fakechroot proot chrootless; do
prefix=
if [ "$mode" = "fakechroot" ]; then
# Devel::Cover doesn't survive mmdebstrap re-exec-ing itself
# with fakechroot, thus, we explicitly run mmdebstrap with
# fakechroot from the start
prefix="runuser -u user -- fakechroot fakeroot"
elif [ "$mode" != "root" ]; then
prefix="runuser -u user --"
fi
print_header "mode=$mode,variant=extract: unpack doc-debian" print_header "mode=$mode,variant=extract: unpack doc-debian"
if [ "$mode" = "unshare" ] && [ "$HAVE_UNSHARE" != "yes" ]; then
echo "HAVE_UNSHARE != yes -- Skipping test..."
continue
fi
if [ "$mode" = "proot" ] && [ "$HAVE_PROOT" != "yes" ]; then
echo "HAVE_PROOT != yes -- Skipping test..."
continue
fi
cat << END > shared/test.sh cat << END > shared/test.sh
#!/bin/sh #!/bin/sh
set -eu set -eu
export LC_ALL=C export LC_ALL=C
adduser --gecos user --disabled-password user [ "\$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1 && adduser --gecos user --disabled-password user
[ "$mode" = unshare ] && sysctl -w kernel.unprivileged_userns_clone=1 [ "$mode" = unshare ] && sysctl -w kernel.unprivileged_userns_clone=1
$prefix $CMD --mode=$mode --variant=extract --include=doc-debian unstable /tmp/debian-unstable $mirror prefix=
[ "\$(id -u)" -eq 0 ] && [ "$mode" != "root" ] && prefix="runuser -u user --"
[ "$mode" = "fakechroot" ] && prefix="\$prefix fakechroot fakeroot"
\$prefix $CMD --mode=$mode --variant=extract --include=doc-debian unstable /tmp/debian-unstable $mirror
# delete contents of doc-debian # delete contents of doc-debian
rm /tmp/debian-unstable/usr/share/doc-base/debian-* rm /tmp/debian-unstable/usr/share/doc-base/debian-*
rm -r /tmp/debian-unstable/usr/share/doc/debian rm -r /tmp/debian-unstable/usr/share/doc/debian
@ -485,7 +627,13 @@ rm -f /tmp/debian-unstable/dev/zero
# the rest should be empty directories that we can rmdir recursively # the rest should be empty directories that we can rmdir recursively
find /tmp/debian-unstable -depth -print0 | xargs -0 rmdir find /tmp/debian-unstable -depth -print0 | xargs -0 rmdir
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
elif [ "$mode" = "root" ]; then
./run_null.sh SUDO
else
./run_null.sh
fi
done done
print_header "mode=chrootless,variant=custom: install doc-debian" print_header "mode=chrootless,variant=custom: install doc-debian"
@ -493,8 +641,10 @@ cat << END > shared/test.sh
#!/bin/sh #!/bin/sh
set -eu set -eu
export LC_ALL=C export LC_ALL=C
adduser --gecos user --disabled-password user [ "\$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1 && adduser --gecos user --disabled-password user
runuser -u user -- $CMD --mode=chrootless --variant=custom --include=doc-debian unstable /tmp/debian-unstable $mirror prefix=
[ "\$(id -u)" -eq 0 ] && prefix="runuser -u user --"
\$prefix $CMD --mode=chrootless --variant=custom --include=doc-debian unstable /tmp/debian-unstable $mirror
# delete contents of doc-debian # delete contents of doc-debian
rm /tmp/debian-unstable/usr/share/doc-base/debian-* rm /tmp/debian-unstable/usr/share/doc-base/debian-*
rm -r /tmp/debian-unstable/usr/share/doc/debian rm -r /tmp/debian-unstable/usr/share/doc/debian
@ -523,7 +673,11 @@ rm /tmp/debian-unstable/var/lib/dpkg/info/doc-debian.list
# the rest should be empty directories that we can rmdir recursively # the rest should be empty directories that we can rmdir recursively
find /tmp/debian-unstable -depth -print0 | xargs -0 rmdir find /tmp/debian-unstable -depth -print0 | xargs -0 rmdir
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh
fi
# test foreign architecture with all modes # test foreign architecture with all modes
# create directory in sudo mode # create directory in sudo mode
@ -536,30 +690,41 @@ set -eu
export LC_ALL=C export LC_ALL=C
$CMD --mode=root --variant=essential unstable /tmp/unstable-chroot.tar $mirror $CMD --mode=root --variant=essential unstable /tmp/unstable-chroot.tar $mirror
tar -tf /tmp/unstable-chroot.tar | sort > tar1.txt tar -tf /tmp/unstable-chroot.tar | sort > tar1.txt
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
else
./run_null.sh SUDO
fi
# FIXME: once fakechroot and proot are fixed, we can switch to variant=apt # FIXME: once fakechroot and proot are fixed, we can switch to variant=apt
# FIXME: cannot test fakechroot or proot because of # FIXME: cannot test fakechroot or proot because of
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=909637 # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=909637
for mode in root unshare fakechroot proot; do for mode in root unshare fakechroot proot; do
prefix=
if [ "$mode" = "fakechroot" ]; then
# Devel::Cover doesn't survive mmdebstrap re-exec-ing itself
# with fakechroot, thus, we explicitly run mmdebstrap with
# fakechroot from the start
prefix="runuser -u user -- fakechroot fakeroot"
elif [ "$mode" != "root" ]; then
prefix="runuser -u user --"
fi
print_header "mode=$mode,variant=essential: create armhf tarball" print_header "mode=$mode,variant=essential: create armhf tarball"
if [ "$HAVE_BINFMT" != "yes" ]; then
echo "HAVE_BINFMT != yes -- Skipping test..."
continue
fi
if [ "$mode" = "unshare" ] && [ "$HAVE_UNSHARE" != "yes" ]; then
echo "HAVE_UNSHARE != yes -- Skipping test..."
continue
fi
if [ "$mode" = "proot" ] && [ "$HAVE_PROOT" != "yes" ]; then
echo "HAVE_PROOT != yes -- Skipping test..."
continue
fi
cat << END > shared/test.sh cat << END > shared/test.sh
#!/bin/sh #!/bin/sh
set -eu set -eu
export LC_ALL=C export LC_ALL=C
adduser --gecos user --disabled-password user [ "\$(id -u)" -eq 0 ] && ! id -u user > /dev/null 2>&1 && adduser --gecos user --disabled-password user
[ "$mode" = unshare ] && sysctl -w kernel.unprivileged_userns_clone=1 [ "$mode" = unshare ] && sysctl -w kernel.unprivileged_userns_clone=1
$prefix $CMD --mode=$mode --variant=essential --architectures=armhf unstable /tmp/unstable-chroot.tar $mirror prefix=
[ "\$(id -u)" -eq 0 ] && [ "$mode" != "root" ] && prefix="runuser -u user --"
[ "$mode" = "fakechroot" ] && prefix="\$prefix fakechroot fakeroot"
\$prefix $CMD --mode=$mode --variant=essential --architectures=armhf unstable /tmp/unstable-chroot.tar $mirror
# we ignore differences between architectures by ignoring some files # we ignore differences between architectures by ignoring some files
# and renaming others # and renaming others
# in fakechroot mode, we use a fake ldconfig, so we have to # in fakechroot mode, we use a fake ldconfig, so we have to
@ -584,8 +749,15 @@ $prefix $CMD --mode=$mode --variant=essential --architectures=armhf unstable /tm
| grep -v '^./usr/share/man/man8/x86_64.8.gz$'; | grep -v '^./usr/share/man/man8/x86_64.8.gz$';
[ "$mode" = "proot" ] && printf "./etc/ld.so.preload\n"; [ "$mode" = "proot" ] && printf "./etc/ld.so.preload\n";
} | sort | diff -u - tar2.txt } | sort | diff -u - tar2.txt
rm /tmp/unstable-chroot.tar
END END
./run_qemu.sh if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
elif [ "$mode" = "root" ]; then
./run_null.sh SUDO
else
./run_null.sh
fi
done done
# test if auto mode picks the right mode # test if auto mode picks the right mode
@ -594,8 +766,10 @@ done
# test tty output # test tty output
guestfish add-ro shared/cover_db.img : run : mount /dev/sda / : tar-out / - \ if [ "$HAVE_QEMU" = "yes" ]; then
| tar -C shared/cover_db --extract guestfish add-ro shared/cover_db.img : run : mount /dev/sda / : tar-out / - \
| tar -C shared/cover_db --extract
fi
if [ -e shared/cover_db/runs ]; then if [ -e shared/cover_db/runs ]; then
cover -nogcov -report html_basic shared/cover_db cover -nogcov -report html_basic shared/cover_db

99
make_mirror.sh
View file

@ -13,6 +13,8 @@ if [ "$arch1" = "$arch2" ]; then
fi fi
components=main components=main
: "${HAVE_QEMU:=yes}"
if [ -e "$mirrordir/dists/unstable/Release" ]; then if [ -e "$mirrordir/dists/unstable/Release" ]; then
http_code=$(curl --output /dev/null --silent --location --head --time-cond "$mirrordir/dists/unstable/Release" --write-out '%{http_code}' "$mirror/dists/unstable/Release") http_code=$(curl --output /dev/null --silent --location --head --time-cond "$mirrordir/dists/unstable/Release" --write-out '%{http_code}' "$mirror/dists/unstable/Release")
case "$http_code" in case "$http_code" in
@ -155,20 +157,21 @@ done
# now fill the cache with new content # now fill the cache with new content
mkdir -p "$cachedir" mkdir -p "$cachedir"
# We must not use any --dpkgopt here because any dpkg options still if [ "$HAVE_QEMU" = "yes" ]; then
# leak into the chroot with chrootless mode. # We must not use any --dpkgopt here because any dpkg options still
# We do not use our own package cache here because # leak into the chroot with chrootless mode.
# - it doesn't (and shouldn't) contain the extra packages # We do not use our own package cache here because
# - it doesn't matter if the base system is from a different mirror timestamp # - it doesn't (and shouldn't) contain the extra packages
# Write tarfile via redirection because in the Debian package autopkgtest # - it doesn't matter if the base system is from a different mirror timestamp
# ./mmdebstrap is a suid binary and we do not want the tarfile being owned # Write tarfile via redirection because in the Debian package autopkgtest
# by root. # ./mmdebstrap is a suid binary and we do not want the tarfile being owned
tmpdir="$(mktemp -d)" # by root.
./mmdebstrap --variant=apt --architectures=amd64,armhf --mode=unshare \ tmpdir="$(mktemp -d)"
--include=linux-image-amd64,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,proot,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,debootstrap,libfakechroot:armhf,libfakeroot:armhf \ ./mmdebstrap --variant=apt --architectures=amd64,armhf --mode=unshare \
unstable > "$tmpdir/debian-unstable.tar" --include=linux-image-amd64,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,proot,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,debootstrap,libfakechroot:armhf,libfakeroot:armhf \
unstable > "$tmpdir/debian-unstable.tar"
cat << END > "$tmpdir/extlinux.conf" cat << END > "$tmpdir/extlinux.conf"
default linux default linux
timeout 0 timeout 0
@ -177,7 +180,7 @@ kernel /vmlinuz
append initrd=/initrd.img root=/dev/sda1 rw console=ttyS0,115200 append initrd=/initrd.img root=/dev/sda1 rw console=ttyS0,115200
serial 0 115200 serial 0 115200
END END
cat << END > "$tmpdir/mmdebstrap.service" cat << END > "$tmpdir/mmdebstrap.service"
[Unit] [Unit]
Description=mmdebstrap worker script Description=mmdebstrap worker script
@ -188,15 +191,15 @@ ExecStart=/worker.sh
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target
END END
# here is something crazy: # here is something crazy:
# as we run mmdebstrap, the process ends up being run by different users with # as we run mmdebstrap, the process ends up being run by different users with
# different privileges (real or fake). But for being able to collect # different privileges (real or fake). But for being able to collect
# Devel::Cover data, they must all share a single directory. The only way that # Devel::Cover data, they must all share a single directory. The only way that
# I found to make this work is to mount the database directory with a # I found to make this work is to mount the database directory with a
# filesystem that doesn't support ownership information at all and a umask that # filesystem that doesn't support ownership information at all and a umask that
# gives read/write access to everybody. # gives read/write access to everybody.
# https://github.com/pjcj/Devel--Cover/issues/223 # https://github.com/pjcj/Devel--Cover/issues/223
cat << 'END' > "$tmpdir/worker.sh" cat << 'END' > "$tmpdir/worker.sh"
#!/bin/sh #!/bin/sh
mount -t 9p -o trans=virtio,access=any mmdebstrap /mnt mount -t 9p -o trans=virtio,access=any mmdebstrap /mnt
( (
@ -216,34 +219,35 @@ mount -t 9p -o trans=virtio,access=any mmdebstrap /mnt
umount /mnt umount /mnt
systemctl poweroff systemctl poweroff
END END
chmod +x "$tmpdir/worker.sh" chmod +x "$tmpdir/worker.sh"
cat << 'END' > "$tmpdir/mini-httpd" cat << 'END' > "$tmpdir/mini-httpd"
START=1 START=1
DAEMON_OPTS="-h 127.0.0.1 -p 80 -u nobody -dd /mnt -i /var/run/mini-httpd.pid -T UTF-8" DAEMON_OPTS="-h 127.0.0.1 -p 80 -u nobody -dd /mnt -i /var/run/mini-httpd.pid -T UTF-8"
END END
cat << 'END' > "$tmpdir/hosts" cat << 'END' > "$tmpdir/hosts"
127.0.0.1 localhost 127.0.0.1 localhost
END END
#libguestfs-test-tool #libguestfs-test-tool
#export LIBGUESTFS_DEBUG=1 LIBGUESTFS_TRACE=1 #export LIBGUESTFS_DEBUG=1 LIBGUESTFS_TRACE=1
guestfish -N "$tmpdir/debian-unstable.img"=disk:2G -- \ guestfish -N "$tmpdir/debian-unstable.img"=disk:2G -- \
part-disk /dev/sda mbr : \ part-disk /dev/sda mbr : \
part-set-bootable /dev/sda 1 true : \ part-set-bootable /dev/sda 1 true : \
mkfs ext2 /dev/sda1 : \ mkfs ext2 /dev/sda1 : \
mount /dev/sda1 / : \ mount /dev/sda1 / : \
tar-in "$tmpdir/debian-unstable.tar" / : \ tar-in "$tmpdir/debian-unstable.tar" / : \
extlinux / : \ extlinux / : \
copy-in "$tmpdir/extlinux.conf" / : \ copy-in "$tmpdir/extlinux.conf" / : \
mkdir-p /etc/systemd/system/multi-user.target.wants : \ mkdir-p /etc/systemd/system/multi-user.target.wants : \
ln-s ../mmdebstrap.service /etc/systemd/system/multi-user.target.wants/mmdebstrap.service : \ ln-s ../mmdebstrap.service /etc/systemd/system/multi-user.target.wants/mmdebstrap.service : \
copy-in "$tmpdir/mmdebstrap.service" /etc/systemd/system/ : \ copy-in "$tmpdir/mmdebstrap.service" /etc/systemd/system/ : \
copy-in "$tmpdir/worker.sh" / : \ copy-in "$tmpdir/worker.sh" / : \
copy-in "$tmpdir/mini-httpd" /etc/default : \ copy-in "$tmpdir/mini-httpd" /etc/default : \
copy-in "$tmpdir/hosts" /etc/ : copy-in "$tmpdir/hosts" /etc/ :
rm "$tmpdir/extlinux.conf" "$tmpdir/worker.sh" "$tmpdir/mini-httpd" "$tmpdir/hosts" "$tmpdir/debian-unstable.tar" "$tmpdir/mmdebstrap.service" rm "$tmpdir/extlinux.conf" "$tmpdir/worker.sh" "$tmpdir/mini-httpd" "$tmpdir/hosts" "$tmpdir/debian-unstable.tar" "$tmpdir/mmdebstrap.service"
qemu-img convert -O qcow2 "$tmpdir/debian-unstable.img" "$cachedir/debian-unstable.qcow" qemu-img convert -O qcow2 "$tmpdir/debian-unstable.img" "$cachedir/debian-unstable.qcow"
rm "$tmpdir/debian-unstable.img" rm "$tmpdir/debian-unstable.img"
rmdir "$tmpdir" rmdir "$tmpdir"
fi
mirror="http://127.0.0.1/debian" mirror="http://127.0.0.1/debian"
SOURCE_DATE_EPOCH=$(date --date="$(grep-dctrl -s Date -n '' "$mirrordir/dists/unstable/Release")" +%s) SOURCE_DATE_EPOCH=$(date --date="$(grep-dctrl -s Date -n '' "$mirrordir/dists/unstable/Release")" +%s)
@ -256,7 +260,8 @@ set -eu
export LC_ALL=C export LC_ALL=C
debootstrap --merged-usr --variant=$variant $dist /tmp/debian-$dist-debootstrap $mirror debootstrap --merged-usr --variant=$variant $dist /tmp/debian-$dist-debootstrap $mirror
tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system -C /tmp/debian-$dist-debootstrap -c . > "cache/debian-$dist-$variant.tar" tar --sort=name --mtime=@$SOURCE_DATE_EPOCH --clamp-mtime --numeric-owner --one-file-system -C /tmp/debian-$dist-debootstrap -c . > "cache/debian-$dist-$variant.tar"
rm -r /tmp/debian-$dist-debootstrap
END END
./run_qemu.sh ./run_qemu.sh SUDO
done done
done done

32
run_null.sh Executable file
View file

@ -0,0 +1,32 @@
#!/bin/sh
set -eu
cachedir="./shared/cache"
SUDO=
while [ "$#" -gt 0 ]; do
key="$1"
case "$key" in
SUDO)
SUDO=sudo
;;
*)
echo "Unknown argument: $key"
exit 1
;;
esac
shift
done
# subshell so that we can cd without effecting the rest
(
set +e
cd ./shared;
$SUDO sh ./test.sh;
echo $?;
) 2>&1 | tee shared/result.txt | head --lines=-1
if [ "$(tail --lines=1 shared/result.txt)" -ne 0 ]; then
echo "test.sh failed"
exit 1
fi