Compare commits

..

6 commits

Author SHA1 Message Date
6818d2ed40
release 1.5.1 2024-06-03 10:46:29 +02:00
9f8172bbc0
Switch variant apt from 'apt-get dist-upgrade' to apt patterns
The dist-upgrade method will fail to install essential packages but
still exit successfully without reporting an error, see #1072218
2024-06-03 08:04:03 +02:00
767e8a8bfb
run_qemu.sh: allow setting MMDEBSTRAP_TESTS_DEBUG and add auto-detection for amlogic a311d bananapi 2024-06-03 08:04:03 +02:00
abc66c5dc7
increase additional number of blocks from 1.1 to 1.2 times the computed number 2024-06-03 08:04:02 +02:00
f3ea5f2676
add flock on temporary directory in /tmp
Since systemd 256~rc3-3, /tmp is regularly cleaned up, removing files
older than 10 days. Since a rootfs contains files with timestamps
potentially much older than that, we exclude our temporary directory by
adding an exclusive lock on it which will stop systemd-tmpfiles from
cleaning up anything in it.

Thanks: Peter Pentchev <roam@ringlet.net>
2024-06-03 08:03:04 +02:00
5f491f2955
make_mirror.sh: force systemd to not mount /tmp as tmpfs
Since systemd 256~rc3-3, /tmp is mounted as tmpfs by default. This
breaks our tests because /tmp is mounted with nodev which makes it
impossible for debootstrap to mknod.
2024-06-02 08:44:28 +02:00
14 changed files with 88 additions and 76 deletions

View file

@ -1,3 +1,9 @@
1.5.1 (2024-06-03)
------------------
- in root and unshare mode, run 'mount --make-rprivate /' before bind-mounting
- switch apt variant from using 'apt-get dist-upgrade' to apt patterns
1.5.0 (2024-05-14)
------------------

View file

@ -550,6 +550,7 @@ END
--customize-hook='copy-in "'"$tmpdir"'/worker.sh" /' \
--customize-hook='echo 127.0.0.1 localhost > "$1/etc/hosts"' \
--customize-hook='printf "START=1\nDAEMON_OPTS=\"-h 127.0.0.1 -p 80 -u nobody -dd /mnt/cache -i /var/run/mini-httpd.pid -T UTF-8\"\n" > "$1/etc/default/mini-httpd"' \
--customize-hook='touch "$1/etc/systemd/system/tmp.mount"' \
"$mirror"
kill $PROXYPID

View file

@ -23,7 +23,7 @@
use strict;
use warnings;
our $VERSION = '1.5.0';
our $VERSION = '1.5.1';
use English;
use Getopt::Long;
@ -36,7 +36,8 @@ use File::Find;
use Cwd qw(abs_path getcwd);
require "syscall.ph"; ## no critic (Modules::RequireBarewordIncludes)
require "sys/ioctl.ph"; ## no critic (Modules::RequireBarewordIncludes)
use Fcntl qw(S_IFCHR S_IFBLK FD_CLOEXEC F_GETFD F_SETFD);
use Fcntl
qw(S_IFCHR S_IFBLK FD_CLOEXEC F_GETFD F_SETFD LOCK_EX O_RDONLY O_DIRECTORY);
use List::Util qw(any none);
use POSIX
qw(SIGINT SIGHUP SIGPIPE SIGTERM SIG_BLOCK SIG_UNBLOCK strftime isatty);
@ -2069,19 +2070,16 @@ sub run_setup() {
# for authentication, use the keyrings from the host
print $conf "Dir::Etc::Trusted \"$options->{apttrusted}\";\n";
print $conf "Dir::Etc::TrustedParts \"$options->{apttrustedparts}\";\n";
if ($options->{variant} ne 'apt') {
# apt considers itself essential. Thus, when generating an EDSP
# document for an external solver, it will add the Essential:yes field
# to the apt package stanza. This is unnecessary for any other variant
# than 'apt' because in all other variants we compile the set of
# packages we consider essential ourselves and for the 'essential'
# variant it would even be wrong to add apt. This workaround is only
# needed when apt is used with an external solver but doesn't hurt
# otherwise and we don't have a good way to figure out whether apt is
# using an external solver or not short of parsing the --aptopt
# options.
print $conf "pkgCacheGen::ForceEssential \",\";\n";
}
# apt considers itself essential. Thus, when generating an EDSP document
# for an external solver, it will add the Essential:yes field to the apt
# package stanza. This is unnecessary because we compile the set of
# packages we consider essential ourselves and for the 'essential' variant
# it would even be wrong to add apt. This workaround is only needed when
# apt is used with an external solver but doesn't hurt otherwise and we
# don't have a good way to figure out whether apt is using an external
# solver or not short of parsing the --aptopt options.
print $conf "pkgCacheGen::ForceEssential \",\";\n";
close $conf;
# We put certain configuration items in their own configuration file
@ -2487,27 +2485,8 @@ sub run_download() {
dryrun => $options->{dryrun},
},
);
} elsif ($options->{variant} eq 'apt') {
# if we just want to install Essential:yes packages, apt and their
# dependencies then we can make use of libapt treating apt as
# implicitly essential. An upgrade with the (currently) empty status
# file will trigger an installation of the essential packages plus apt.
#
# 2018-09-02, #debian-dpkg on OFTC, times in UTC+2
# 23:39 < josch> I'll just put it in my script and if it starts
# breaking some time I just say it's apt's fault. :P
# 23:42 < DonKult> that is how it usually works, so yes, do that :P (<-
# and please add that line next to it so you can
# remind me in 5+ years that I said that after I wrote
# in the bugreport: "Are you crazy?!? Nobody in his
# right mind would even suggest depending on it!")
@dl_debs = run_apt_download_progress({
APT_ARGV => ['dist-upgrade'],
dryrun => $options->{dryrun},
},
);
} elsif (any { $_ eq $options->{variant} }
('essential', 'standard', 'important', 'required', 'buildd')) {
('essential', 'apt', 'standard', 'important', 'required', 'buildd')) {
# 2021-06-07, #debian-apt on OFTC, times in UTC+2
# 17:27 < DonKult> (?essential includes 'apt' through)
# 17:30 < josch> DonKult: no, because pkgCacheGen::ForceEssential ",";
@ -2981,8 +2960,14 @@ sub run_install() {
my $options = shift;
my @pkgs_to_install = (@{ $options->{include} });
if ($options->{variant} eq 'extract') {
error "must not be called with variant extract";
}
if (none { $_ eq $options->{variant} } ('custom', 'essential')) {
push @pkgs_to_install, 'apt';
}
if ($options->{variant} eq 'buildd') {
push @pkgs_to_install, 'build-essential', 'apt';
push @pkgs_to_install, 'build-essential';
}
if (any { $_ eq $options->{variant} }
('required', 'important', 'standard')) {
@ -4454,8 +4439,9 @@ sub approx_disk_usage {
# image. See https://bugs.debian.org/1005857
find({ wanted => $scan_installed_size, no_chdir => 1 }, $directory);
# because the above is only a heuristic we add 10% extra for good measure
return int($installed_size * 1.1);
# the above is only a heuristic and especially ext4 will consume quite a
# few more blocks than the heuristic above is going to compute
return int($installed_size * 1.2);
}
sub main() {
@ -5759,6 +5745,7 @@ sub main() {
$blocksize = 1048576;
}
my $rootdir_handle;
if (any { $_ eq $options->{format} }
('tar', 'squashfs', 'ext2', 'ext4', 'null')) {
if ($options->{format} ne 'null') {
@ -5789,6 +5776,13 @@ sub main() {
# directory
$options->{root} = tempdir('mmdebstrap.XXXXXXXXXX', TMPDIR => 1);
info "using $options->{root} as tempdir";
# add an flock on the temporary directory to prevent cleanup by systemd
# see section Age in tmpfiles.d(5)
sysopen($rootdir_handle, $options->{root}, O_RDONLY | O_DIRECTORY)
or error "Failed to sysopen $options->{root}: $!\n";
flock($rootdir_handle, LOCK_EX)
or error "Unable to flock $options->{root}: $!\n";
# in unshare and root mode, other users than the current user need to
# access the rootfs, most prominently, the _apt user. Thus, make the
# temporary directory world readable.
@ -7074,14 +7068,10 @@ considered for selection of C<Essential:yes> packages.
=item B<apt>
The B<essential> set plus apt. This variant uses the fact that B<apt> treats
itself as essential and thus running C<apt-get dist-upgrade> without any
packages installed will install the B<essential> set plus B<apt>. If you just
want B<essential> and B<apt>, then this variant is faster than using the
B<essential> variant and adding B<apt> via C<--include> because all packages
get installed at once. The downside of this variant is, that if it should
happen that an B<essential> package is not installable, then it will just get
ignored without throwing an error.
The B<essential> set plus apt.
It is roughly equivalent to running mmdebstrap with
--variant=essential --include="apt"
=item B<buildd>
@ -7410,15 +7400,11 @@ the B<setup> step. This can be disabled using B<--skip=update>.
=item B<download>
In the B<extract> and B<custom> variants, C<apt-get install> is used to
download all the packages requested via the B<--include> option. The B<apt>
variant uses the fact that libapt treats the C<apt> packages as implicitly
essential to download only all C<Essential:yes> packages plus apt using
C<apt-get dist-upgrade>. In the remaining variants, all Packages files
downloaded by the B<update> step are inspected to find the C<Essential:yes>
package set as well as all packages of the required priority. If I<SUITE> is a
non-empty string, then only packages from the archive with suite or codename
matching I<SUITE> will be considered for selection of C<Essential:yes>
packages.
download all the packages requested via the B<--include> option. In the
remaining variants, apt patterns are used to find the C<Essential:yes> packages
from the native architecture. If I<SUITE> is a non-empty string, then only
packages from the archive with suite or codename matching I<SUITE> will be
considered for selection of C<Essential:yes> packages.
=item B<mount>

View file

@ -4,6 +4,7 @@ set -eu
: "${DEFAULT_DIST:=unstable}"
: "${cachedir:=./shared/cache}"
: "${MMDEBSTRAP_TESTS_DEBUG:=no}"
tmpdir="$(mktemp -d)"
cleanup() {
@ -29,20 +30,42 @@ fi
touch shared/output.txt
setpriv --pdeathsig TERM tail -f shared/output.txt &
# to connect to serial use:
# minicom -D 'unix#/tmp/ttyS0'
#
# or this (quit with ctrl+q):
# socat stdin,raw,echo=0,escape=0x11 unix-connect:/tmp/ttyS0
ret=0
timeout --foreground 40m debvm-run --image="$(realpath "$cachedir")/debian-$DEFAULT_DIST.ext4" -- \
-nic none \
-m 4G -snapshot \
set -- timeout --foreground 40m \
debvm-run --image="$(realpath "$cachedir")/debian-$DEFAULT_DIST.ext4" \
--
cpuname=$(lscpu | awk '/Model name:/ {print $3}' | tr '\n' '+')
ncpu=$(lscpu | awk '/Core\(s\) per socket:/ {print $4}' | tr '\n' '+')
if [ "$cpuname" = "Cortex-A53+Cortex-A73+" ] && [ "$ncpu" = "2+4+" ]; then
# crude detection of the big.LITTLE heterogeneous setup of cores on the
# amlogic a311d bananapi
#
# https://lists.nongnu.org/archive/html/qemu-devel/2020-10/msg08494.html
# https://gitlab.com/qemu-project/qemu/-/issues/239
# https://segments.zhan.science/posts/kvm_on_pinehone_pro/#trouble-with-heterogeneous-architecture
set -- taskset --cpu-list 2,3,4,5 "$@" -smp 4
fi
set -- "$@" -nic none -m 4G -snapshot
if [ "$MMDEBSTRAP_TESTS_DEBUG" = "no" ]; then
# to connect to serial use:
# minicom -D 'unix#/tmp/ttyS0'
# or this (quit with ctrl+q):
# socat stdin,raw,echo=0,escape=0x11 unix-connect:/tmp/ttyS0
set -- "$@" \
-monitor unix:/tmp/monitor,server,nowait \
-serial unix:/tmp/ttyS0,server,nowait \
-serial unix:/tmp/ttyS1,server,nowait \
-virtfs local,id=mmdebstrap,path="$(pwd)/shared",security_model=none,mount_tag=mmdebstrap \
>"$tmpdir/log" 2>&1 || ret=$?
-serial unix:/tmp/ttyS1,server,nowait
fi
set -- "$@" -virtfs local,id=mmdebstrap,path="$(pwd)/shared",security_model=none,mount_tag=mmdebstrap
ret=0
if [ "$MMDEBSTRAP_TESTS_DEBUG" = "no" ]; then
"$@" >"$tmpdir/log" 2>&1 || ret=$?
else
"$@" 2>&1 | tee "$tmpdir/log" || ret=$?
fi
if [ "$ret" -ne 0 ]; then
cat "$tmpdir/log"
exit $ret

View file

@ -5,4 +5,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
{{ CMD }} --mode={{ MODE }} --variant=essential \
--include '?or(?exact-name(dummy-does-not-exist),?exact-name(apt))' \
{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -
tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -

View file

@ -6,4 +6,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
--include '?narrow(?archive(^{{ DIST }}$),?essential)' \
--include apt \
{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -
{ tar -tf /tmp/debian-chroot.tar; echo ./var/lib/apt/extended_states; } | sort | diff -u tar1.txt -

View file

@ -6,6 +6,6 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
--include "$(tr '\n' ',' < pkglist.txt)" \
--aptopt='APT::Solver "aspcud"' \
{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
tar -tf /tmp/debian-chroot.tar | sort \
{ tar -tf /tmp/debian-chroot.tar; echo ./var/lib/apt/extended_states; } | sort \
| grep -v '^./etc/apt/apt.conf.d/99mmdebstrap$' \
| diff -u tar1.txt -

View file

@ -17,5 +17,4 @@ tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort \
| grep -v '^./usr/share/lintian/overrides/tzdata' \
| grep -v '^./usr/share/zoneinfo' \
| grep -v '^./var/lib/dpkg/info/tzdata.' \
| grep -v '^./var/lib/apt/extended_states$' \
| diff -u tar1.txt -

View file

@ -6,7 +6,6 @@ trap "rm -rf /tmp/debian-chroot" EXIT INT TERM
rm /tmp/debian-chroot/usr/share/doc-base/doc-debian.debian-*
rm -r /tmp/debian-chroot/usr/share/doc/debian
rm -r /tmp/debian-chroot/usr/share/doc/doc-debian
rm /tmp/debian-chroot/var/lib/apt/extended_states
rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.list
rm /tmp/debian-chroot/var/lib/dpkg/info/doc-debian.md5sums
tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -

View file

@ -36,7 +36,6 @@ rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
rm /tmp/debian-chroot/usr/share/file/magic.mgc
rm /tmp/debian-chroot/usr/share/misc/magic.mgc
rm /tmp/debian-chroot/var/lib/apt/extended_states
rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/

View file

@ -37,7 +37,6 @@ rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/changelog.gz
rm /tmp/debian-chroot/usr/share/doc/libmagic-mgc/copyright
rm /tmp/debian-chroot/usr/share/file/magic.mgc
rm /tmp/debian-chroot/usr/share/misc/magic.mgc
rm /tmp/debian-chroot/var/lib/apt/extended_states
rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.list
rm /tmp/debian-chroot/var/lib/dpkg/info/libmagic-mgc.md5sums
rmdir /tmp/debian-chroot/usr/share/doc/libmagic-mgc/

View file

@ -16,6 +16,7 @@ I: running apt-get update...
I: downloading packages with apt...
I: extracting archives...
I: installing essential packages...
I: installing remaining packages inside the chroot...
I: cleaning package lists and apt cache...
LOG
tail --lines=1 /tmp/log | grep '^I: success in .* seconds$'

View file

@ -11,7 +11,6 @@ rm /tmp/debian-chroot/etc/localtime
rm /tmp/debian-chroot/etc/timezone
rm -r /tmp/debian-chroot/usr/share/doc/tzdata
rm -r /tmp/debian-chroot/usr/share/zoneinfo
rm /tmp/debian-chroot/var/lib/apt/extended_states
for p in doc-debian tzdata; do
for f in list md5sums config postinst postrm templates preinst prerm; do
[ -e "/tmp/debian-chroot/var/lib/dpkg/info/$p.$f" ] || continue

View file

@ -6,4 +6,4 @@ trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get update' \
--essential-hook='APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-get --yes install apt' \
{{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -
tar -tf /tmp/debian-chroot.tar | sort | diff -u tar1.txt -