add --dry-run and --simulate

This commit is contained in:
Johannes 'josch' Schauer 2020-01-10 11:44:15 +01:00
parent 9441184bf1
commit 2cb6438454
Signed by: josch
GPG key ID: F2CBA5C78FBD83E1
2 changed files with 299 additions and 102 deletions

View file

@ -66,7 +66,7 @@ if [ ! -e shared/mmdebstrap ] || [ mmdebstrap -nt shared/mmdebstrap ]; then
fi fi
starttime= starttime=
total=115 total=136
skipped=0 skipped=0
runtests=0 runtests=0
i=1 i=1
@ -1792,10 +1792,103 @@ else
runtests=$((runtests+1)) runtests=$((runtests+1))
fi fi
print_header "mode=$defaultmode,variant=apt: create directory --dry-run"
cat << END > shared/test.sh
#!/bin/sh
set -eu
export LC_ALL=C.UTF-8
$CMD --mode=$defaultmode --dry-run --variant=apt --setup-hook="exit 1" --essential-hook="exit 1" --customize-hook="exit 1" $DEFAULT_DIST /tmp/debian-chroot $mirror
rm /tmp/debian-chroot/dev/console
rm /tmp/debian-chroot/dev/fd
rm /tmp/debian-chroot/dev/full
rm /tmp/debian-chroot/dev/null
rm /tmp/debian-chroot/dev/ptmx
rm /tmp/debian-chroot/dev/random
rm /tmp/debian-chroot/dev/stderr
rm /tmp/debian-chroot/dev/stdin
rm /tmp/debian-chroot/dev/stdout
rm /tmp/debian-chroot/dev/tty
rm /tmp/debian-chroot/dev/urandom
rm /tmp/debian-chroot/dev/zero
rm /tmp/debian-chroot/etc/apt/sources.list
rm /tmp/debian-chroot/etc/fstab
rm /tmp/debian-chroot/etc/hostname
rm /tmp/debian-chroot/etc/resolv.conf
rm /tmp/debian-chroot/var/lib/apt/lists/lock
rm /tmp/debian-chroot/var/lib/dpkg/available
rm /tmp/debian-chroot/var/lib/dpkg/cmethopt
rm /tmp/debian-chroot/var/lib/dpkg/status
# the rest should be empty directories that we can rmdir recursively
find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir
END
if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
runtests=$((runtests+1))
elif [ "$defaultmode" = "root" ]; then
./run_null.sh SUDO
runtests=$((runtests+1))
else
./run_null.sh
runtests=$((runtests+1))
fi
# test all --dry-run variants
for variant in extract custom essential apt; do
for mode in root unshare fakechroot proot chrootless; do
print_header "mode=$mode,variant=$variant: create tarball --dry-run"
if [ "$mode" = "unshare" ] && [ "$HAVE_UNSHARE" != "yes" ]; then
echo "HAVE_UNSHARE != yes -- Skipping test..." >&2
skipped=$((skipped+1))
continue
fi
if [ "$mode" = "proot" ] && [ "$HAVE_PROOT" != "yes" ]; then
echo "HAVE_PROOT != yes -- Skipping test..." >&2
skipped=$((skipped+1))
continue
fi
cat << END > shared/test.sh
#!/bin/sh
set -eu
export LC_ALL=C.UTF-8
prefix=
include=
if [ "\$(id -u)" -eq 0 ] && [ "$mode" != root ]; then
# this must be qemu
if ! id -u user >/dev/null 2>&1; then
adduser --gecos user --disabled-password user
fi
if [ "$mode" = unshare ]; then
sysctl -w kernel.unprivileged_userns_clone=1
fi
prefix="runuser -u user --"
if [ "$mode" = extract ] || [ "$mode" = custom ]; then
include="--include=\$(cat pkglist.txt | tr '\n' ',')"
fi
fi
\$prefix $CMD --mode=$mode \$include --dry-run --variant=$variant $DEFAULT_DIST /tmp/debian-chroot.tar $mirror
if [ -e /tmp/debian-chroot.tar ]; then
echo "/tmp/debian-chroot.tar must not be created with --dry-run" >&2
exit 1
fi
END
if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
runtests=$((runtests+1))
elif [ "$mode" = "root" ]; then
./run_null.sh SUDO
runtests=$((runtests+1))
else
./run_null.sh
runtests=$((runtests+1))
fi
done
done
# test all variants # test all variants
for variant in essential apt required minbase buildd important debootstrap - standard; do for variant in essential apt required minbase buildd important debootstrap - standard; do
print_header "mode=root,variant=$variant: create directory" print_header "mode=root,variant=$variant: create tarball"
cat << END > shared/test.sh cat << END > shared/test.sh
#!/bin/sh #!/bin/sh
set -eu set -eu

View file

@ -1071,6 +1071,11 @@ sub run_hooks {
return; return;
} }
if ($options->{dryrun}) {
info "not running ${name}-hooks because of --dry-run";
return;
}
my $runner = sub { my $runner = sub {
foreach my $script (@{ $options->{"${name}_hook"} }) { foreach my $script (@{ $options->{"${name}_hook"} }) {
if ($script if ($script
@ -1520,11 +1525,18 @@ sub setup {
# (essential variant) then we have to compute the package set ourselves. # (essential variant) then we have to compute the package set ourselves.
# Same if we want to install priority based variants. # Same if we want to install priority based variants.
if (any { $_ eq $options->{variant} } ('extract', 'custom')) { if (any { $_ eq $options->{variant} } ('extract', 'custom')) {
if ($options->{dryrun}) {
info "simulate downloading packages with apt...";
} else {
info "downloading packages with apt..."; info "downloading packages with apt...";
}
run_apt_progress({ run_apt_progress({
ARGV => [ ARGV => [
'apt-get', '--yes', 'apt-get',
'-oApt::Get::Download-Only=true', 'install' '--yes',
'-oApt::Get::Download-Only=true',
$options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
'install'
], ],
PKGS => [@pkgs_to_install], PKGS => [@pkgs_to_install],
}); });
@ -1542,11 +1554,18 @@ sub setup {
# remind me in 5+ years that I said that after I wrote # remind me in 5+ years that I said that after I wrote
# in the bugreport: "Are you crazy?!? Nobody in his # in the bugreport: "Are you crazy?!? Nobody in his
# right mind would even suggest depending on it!") # right mind would even suggest depending on it!")
if ($options->{dryrun}) {
info "simulate downloading packages with apt...";
} else {
info "downloading packages with apt..."; info "downloading packages with apt...";
}
run_apt_progress({ run_apt_progress({
ARGV => [ ARGV => [
'apt-get', '--yes', 'apt-get',
'-oApt::Get::Download-Only=true', 'dist-upgrade' '--yes',
'-oApt::Get::Download-Only=true',
$options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
'dist-upgrade'
], ],
}); });
} elsif ( } elsif (
@ -1648,11 +1667,18 @@ sub setup {
debug " $pkg"; debug " $pkg";
} }
if ($options->{dryrun}) {
info "simulate downloading packages with apt...";
} else {
info "downloading packages with apt..."; info "downloading packages with apt...";
}
run_apt_progress({ run_apt_progress({
ARGV => [ ARGV => [
'apt-get', '--yes', 'apt-get',
'-oApt::Get::Download-Only=true', 'install' '--yes',
'-oApt::Get::Download-Only=true',
$options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
'install'
], ],
PKGS => [keys %ess_pkgs], PKGS => [keys %ess_pkgs],
}); });
@ -1660,9 +1686,9 @@ sub setup {
error "unknown variant: $options->{variant}"; error "unknown variant: $options->{variant}";
} }
# extract the downloaded packages # collect the .deb files that were downloaded by apt
my @essential_pkgs; my @essential_pkgs;
{ if (!$options->{dryrun}) {
my $apt_archives = "/var/cache/apt/archives/"; my $apt_archives = "/var/cache/apt/archives/";
opendir my $dh, "$options->{root}/$apt_archives" opendir my $dh, "$options->{root}/$apt_archives"
or error "cannot read $apt_archives"; or error "cannot read $apt_archives";
@ -1677,7 +1703,6 @@ sub setup {
push @essential_pkgs, $deb; push @essential_pkgs, $deb;
} }
close $dh; close $dh;
}
if (scalar @essential_pkgs == 0) { if (scalar @essential_pkgs == 0) {
# check if a file:// URI was used # check if a file:// URI was used
@ -1686,12 +1711,13 @@ sub setup {
or error "cannot start apt-get indextargets: $!"; or error "cannot start apt-get indextargets: $!";
while (my $uri = <$pipe_apt>) { while (my $uri = <$pipe_apt>) {
if ($uri =~ /^file:\/\//) { if ($uri =~ /^file:\/\//) {
error error "nothing got downloaded -- use copy:// instead of"
"nothing got downloaded -- use copy:// instead of file://"; . " file://";
} }
} }
error "nothing got downloaded"; error "nothing got downloaded";
} }
}
# We have to extract the packages from @essential_pkgs either if we run in # We have to extract the packages from @essential_pkgs either if we run in
# chrootless mode and extract variant or in any other mode. # chrootless mode and extract variant or in any other mode.
@ -1700,6 +1726,8 @@ sub setup {
if ( $options->{mode} eq 'chrootless' if ( $options->{mode} eq 'chrootless'
and $options->{variant} ne 'extract') { and $options->{variant} ne 'extract') {
# nothing to do # nothing to do
} elsif ($options->{dryrun}) {
info "skip extracting packages because of --dry-run";
} else { } else {
info "extracting archives..."; info "extracting archives...";
print_progress 0.0; print_progress 0.0;
@ -1736,7 +1764,11 @@ sub setup {
} }
if ($options->{mode} eq 'chrootless') { if ($options->{mode} eq 'chrootless') {
if ($options->{dryrun}) {
info "simulate installing packages...";
} else {
info "installing packages..."; info "installing packages...";
}
# FIXME: the dpkg config from the host is parsed before the command # FIXME: the dpkg config from the host is parsed before the command
# line arguments are parsed and might break this mode # line arguments are parsed and might break this mode
# Example: if the host has --path-exclude set, then this will also # Example: if the host has --path-exclude set, then this will also
@ -1745,7 +1777,8 @@ sub setup {
'-oDPkg::Options::=--force-not-root', '-oDPkg::Options::=--force-not-root',
'-oDPkg::Options::=--force-script-chrootless', '-oDPkg::Options::=--force-script-chrootless',
'-oDPkg::Options::=--root=' . $options->{root}, '-oDPkg::Options::=--root=' . $options->{root},
'-oDPkg::Options::=--log=' . "$options->{root}/var/log/dpkg.log" '-oDPkg::Options::=--log=' . "$options->{root}/var/log/dpkg.log",
$options->{dryrun} ? '-oAPT::Get::Simulate=true' : (),
); );
if (defined $options->{qemu}) { if (defined $options->{qemu}) {
# The binfmt support on the outside is used, so qemu needs to know # The binfmt support on the outside is used, so qemu needs to know
@ -2002,6 +2035,9 @@ sub setup {
# we need --force-depends because dpkg does not take Pre-Depends # we need --force-depends because dpkg does not take Pre-Depends
# into account and thus doesn't install them in the right order # into account and thus doesn't install them in the right order
# And the --predep-package option is broken: #539133 # And the --predep-package option is broken: #539133
if ($options->{dryrun}) {
info "simulate installing packages...";
} else {
info "installing packages..."; info "installing packages...";
run_chroot( run_chroot(
sub { sub {
@ -2016,10 +2052,12 @@ sub setup {
}, },
$options $options
); );
}
# if the path-excluded option was added to the dpkg config, # if the path-excluded option was added to the dpkg config,
# reinstall all packages # reinstall all packages
if (-e "$options->{root}/etc/dpkg/dpkg.cfg.d/99mmdebstrap") { if ((!$options->{dryrun})
and -e "$options->{root}/etc/dpkg/dpkg.cfg.d/99mmdebstrap") {
open(my $fh, '<', open(my $fh, '<',
"$options->{root}/etc/dpkg/dpkg.cfg.d/99mmdebstrap") "$options->{root}/etc/dpkg/dpkg.cfg.d/99mmdebstrap")
or error "cannot open /etc/dpkg/dpkg.cfg.d/99mmdebstrap: $!"; or error "cannot open /etc/dpkg/dpkg.cfg.d/99mmdebstrap: $!";
@ -2096,15 +2134,29 @@ sub setup {
$? == 0 or error "apt-get indextargets failed"; $? == 0 or error "apt-get indextargets failed";
if (scalar @pkgs_to_install_from_outside > 0) { if (scalar @pkgs_to_install_from_outside > 0) {
if ($options->{dryrun}) {
info 'simulate downloading '
. (join ', ', @pkgs_to_install_from_outside) . "...";
} else {
info 'downloading ' info 'downloading '
. (join ', ', @pkgs_to_install_from_outside) . "..."; . (join ', ', @pkgs_to_install_from_outside) . "...";
}
run_apt_progress({ run_apt_progress({
ARGV => [ ARGV => [
'apt-get', '--yes', 'apt-get',
'-oApt::Get::Download-Only=true', 'install' '--yes',
'-oApt::Get::Download-Only=true',
$options->{dryrun}
? '-oAPT::Get::Simulate=true'
: (),
'install'
], ],
PKGS => [@pkgs_to_install_from_outside], PKGS => [@pkgs_to_install_from_outside],
}); });
if ($options->{dryrun}) {
info 'simulate installing '
. (join ', ', @pkgs_to_install_from_outside) . "...";
} else {
my @debs_to_install; my @debs_to_install;
my $apt_archives = "/var/cache/apt/archives/"; my $apt_archives = "/var/cache/apt/archives/";
opendir my $dh, "$options->{root}/$apt_archives" opendir my $dh, "$options->{root}/$apt_archives"
@ -2121,14 +2173,16 @@ sub setup {
} }
close $dh; close $dh;
if (scalar @debs_to_install == 0) { if (scalar @debs_to_install == 0) {
warning "nothing got downloaded -- maybe the packages" warning
"nothing got downloaded -- maybe the packages"
. " were already installed?"; . " were already installed?";
} else { } else {
# we need --force-depends because dpkg does not take # we need --force-depends because dpkg does not take
# Pre-Depends into account and thus doesn't install # Pre-Depends into account and thus doesn't install
# them in the right order # them in the right order
info 'installing ' info 'installing '
. (join ', ', @pkgs_to_install_from_outside) . "..."; . (join ', ', @pkgs_to_install_from_outside)
. "...";
run_dpkg_progress({ run_dpkg_progress({
ARGV => [ ARGV => [
@chrootcmd, 'env', @chrootcmd, 'env',
@ -2143,16 +2197,21 @@ sub setup {
} }
} }
} }
}
if (!$options->{dryrun}) {
run_chroot( run_chroot(
sub { sub {
info info "installing remaining packages inside the"
"installing remaining packages inside the chroot..."; . " chroot...";
run_apt_progress({ run_apt_progress({
ARGV => [ ARGV => [
@chrootcmd, 'env', @chrootcmd,
'--unset=APT_CONFIG', '--unset=TMPDIR', 'env',
'apt-get', '--yes', '--unset=APT_CONFIG',
'--unset=TMPDIR',
'apt-get',
'--yes',
'install' 'install'
], ],
PKGS => [@pkgs_to_install], PKGS => [@pkgs_to_install],
@ -2160,7 +2219,17 @@ sub setup {
}, },
$options $options
); );
} else {
info "simulate installing remaining packages inside the"
. " chroot...";
run_apt_progress({
ARGV => [
'apt-get', '--yes',
'-oAPT::Get::Simulate=true', 'install'
],
PKGS => [@pkgs_to_install],
});
}
} }
} else { } else {
error "unknown variant: $options->{variant}"; error "unknown variant: $options->{variant}";
@ -2536,6 +2605,7 @@ sub main() {
setup_hook => [], setup_hook => [],
essential_hook => [], essential_hook => [],
customize_hook => [], customize_hook => [],
dryrun => 0,
}; };
my $logfile = undef; my $logfile = undef;
Getopt::Long::Configure('default', 'bundling', 'auto_abbrev', Getopt::Long::Configure('default', 'bundling', 'auto_abbrev',
@ -2589,6 +2659,8 @@ sub main() {
'setup-hook=s@' => \$options->{setup_hook}, 'setup-hook=s@' => \$options->{setup_hook},
'essential-hook=s@' => \$options->{essential_hook}, 'essential-hook=s@' => \$options->{essential_hook},
'customize-hook=s@' => \$options->{customize_hook}, 'customize-hook=s@' => \$options->{customize_hook},
'simulate' => \$options->{dryrun},
'dry-run' => \$options->{dryrun},
) or pod2usage(-exitval => 2, -verbose => 1); ) or pod2usage(-exitval => 2, -verbose => 1);
if (defined($logfile)) { if (defined($logfile)) {
@ -2600,6 +2672,14 @@ sub main() {
. " with some debootstrap wrappers."; . " with some debootstrap wrappers.";
} }
if ($options->{dryrun}) {
foreach my $hook ('setup', 'essential', 'customize') {
if (scalar @{ $options->{"${hook}_hook"} } > 0) {
warning "In dry-run mode, --$hook-hook options have no effect";
}
}
}
my @valid_variants = ( my @valid_variants = (
'extract', 'custom', 'essential', 'apt', 'extract', 'custom', 'essential', 'apt',
'required', 'minbase', 'buildd', 'important', 'required', 'minbase', 'buildd', 'important',
@ -3249,10 +3329,17 @@ sub main() {
# try to fail early if target tarball or squashfs image cannot be # try to fail early if target tarball or squashfs image cannot be
# opened for writing # opened for writing
if ($options->{target} ne '-') { if ($options->{target} ne '-') {
if ($options->{dryrun}) {
if (-e $options->{target}) {
info "not overwriting $options->{target} because in"
. " dry-run mode";
}
} else {
open my $fh, '>', $options->{target} open my $fh, '>', $options->{target}
or error "cannot open $options->{target} for writing: $!"; or error "cannot open $options->{target} for writing: $!";
close $fh; close $fh;
} }
}
# since the output is a tarball, we create the rootfs in a temporary # since the output is a tarball, we create the rootfs in a temporary
# directory # directory
$options->{root} $options->{root}
@ -3493,7 +3580,9 @@ sub main() {
close $childsock; close $childsock;
if ($options->{maketar} or $options->{makesqfs}) { if ($options->{dryrun}) {
info "simulate creating tarball...";
} elsif ($options->{maketar} or $options->{makesqfs}) {
info "creating tarball..."; info "creating tarball...";
# redirect tar output to the writing end of the pipe so that # redirect tar output to the writing end of the pipe so that
@ -3914,7 +4003,9 @@ sub main() {
close $parentsock; close $parentsock;
if ($options->{maketar} or $options->{makesqfs}) { if ($options->{dryrun}) {
# nothing to do
} elsif ($options->{maketar} or $options->{makesqfs}) {
# we use eval() so that error() doesn't take this process down and # we use eval() so that error() doesn't take this process down and
# thus leaves the setup() process without a parent # thus leaves the setup() process without a parent
eval { eval {
@ -3927,7 +4018,8 @@ sub main() {
my @argv = (); my @argv = ();
if ($options->{makesqfs}) { if ($options->{makesqfs}) {
push @argv, 'tar2sqfs', push @argv, 'tar2sqfs',
'--quiet', '--no-skip', '--force', '--exportable', '--quiet', '--no-skip', '--force',
'--exportable',
'--compressor', 'xz', '--compressor', 'xz',
'--block-size', '1048576', '--block-size', '1048576',
$options->{target}; $options->{target};
@ -4299,8 +4391,20 @@ equivalent:
--architectures=amd64,armhf,mipsel --architectures=amd64,armhf,mipsel
--arch=amd64 --arch="armhf mipsel" --arch=armhf,mipsel --arch=amd64 --arch="armhf mipsel" --arch=armhf,mipsel
=item B<--simulate>, B<--dry-run>
Run apt-get with B<--simulate>. Only the package cache is initialized but no
binary packages are downloaded or installed. Use this option to quickly check
whether a package selection within a certain suite and variant can in principle
be installed as far as their dependencies go. If the output is a tarball, then
no output is produced. If the output is a directory, then the directory will be
left populated with the skeleton files and directories necessary for apt to run
in it.
=begin comment =begin comment
No hooks are executed in with B<--simulate> or B<--dry-run>.
=item B<--setup-hook>=I<command> =item B<--setup-hook>=I<command>
Execute arbitrary I<command>s right after initial setup (directory creation, Execute arbitrary I<command>s right after initial setup (directory creation,