Instead of Dpkg::Index use a primitive deb822 parser and shave off another 10 seconds

This commit is contained in:
Johannes 'josch' Schauer 2018-09-23 21:15:12 +02:00
parent 16d9b413f7
commit 1e9817574c
Signed by untrusted user: josch
GPG key ID: F2CBA5C78FBD83E1
2 changed files with 36 additions and 30 deletions

View file

@ -26,7 +26,7 @@ Summary:
- more than one mirror possible
- security and updates mirror included for Debian stable chroots
- 2-3 times faster
- 3-6 times faster
- chroot with apt in 11 seconds
- gzipped tarball with apt is 27M small
- bit-by-bit reproducible output
@ -40,14 +40,14 @@ with debootstrap since 2009 (See #543819 and #762222). Since mmdebstrap uses
apt internally, support for multiple mirrors comes for free and stable or
oldstable **chroots will include security and updates mirrors**.
A side-effect of using apt is being **2-3 times faster** than debootstrap. The
A side-effect of using apt is being **3-6 times faster** than debootstrap. The
timings were carried out on a laptop with an Intel Core i5-5200U.
| variant | mmdebstrap | debootstrap |
| ------- | ---------- | ------------ |
| minbase | 25.25 s | 51.47 s |
| buildd | 30.99 s | 59.38 s |
| - | 29.85 s | 127.18 s |
| minbase | 14.18 s | 51.47 s |
| buildd | 20.55 s | 59.38 s |
| - | 18.98 s | 127.18 s |
Apt considers itself an `Essential: yes` package. This feature allows one to
create a chroot containing just the `Essential: yes` packages and apt (and

View file

@ -22,7 +22,6 @@ use File::Copy;
use File::Path qw(make_path remove_tree);
use File::Temp qw(tempfile tempdir);
use Cwd qw(abs_path);
use Dpkg::Index;
require "syscall.ph";
use Fcntl qw(S_IFCHR S_IFBLK FD_CLOEXEC F_GETFD F_SETFD);
use List::Util qw(any none);
@ -732,49 +731,56 @@ sub setup {
chomp $fname;
open (my $pipe_cat, '-|', '/usr/lib/apt/apt-helper', 'cat-file', $fname) or die "cannot start apt-helper cat-file: $!";
my $key_func = sub {
return $_[0]->{Package} . ' ' . $_[0]->{Version} . ' ' . $_[0]->{Architecture};
};
my $index = Dpkg::Index->new(get_key_func=>$key_func);
$index->parse($pipe_cat, 'apt-helper cat-file') or die "failed to parse";
foreach my $key ($index->get_keys()) {
my $cdata = $index->get_by_key($key);
my $pkgname = $cdata->{Package};
my $arch = $cdata->{Architecture} // '';
my $ess = $cdata->{Essential} // '';
my $pkgname;
my $ess = '';
my $prio = 'optional';
while (my $line = <$pipe_cat>) {
chomp $line;
# Dpkg::Index takes 10 seconds to parse a typical Packages
# file. Thus we instead use a simple parser that just retrieve
# the information we need.
if ($line ne "") {
if ($line =~ /^Package: (.*)/) {
$pkgname = $1;
} elsif ($line =~ /^Essential: yes$/) {
$ess = 'yes'
} elsif ($line =~ /^Priority: (.*)/) {
$prio = $1;
}
next;
}
# the line is empty, thus a package stanza just finished
# processing and we can handle it now
if ($ess eq 'yes') {
$ess_pkgs{$pkgname} = ();
}
my $prio = $cdata->{Priority} // 'optional';
if ($options->{variant} eq 'essential') {
} elsif ($options->{variant} eq 'essential') {
# for this variant we are only interested in the
# essential packages
next;
} elsif (any { $_ eq $options->{variant} } ('standard', 'important', 'required', 'buildd', 'minbase')) {
if ($prio eq 'optional' or $prio eq 'extra') {
# always ignore packages of priority optional and extra
next;
} elsif ($prio eq 'standard') {
if (any { $_ eq $options->{variant} } ('important', 'required', 'buildd', 'minbase')) {
next;
if (none { $_ eq $options->{variant} } ('important', 'required', 'buildd', 'minbase')) {
$pkgs_to_install{$pkgname} = ();
}
} elsif ($prio eq 'important') {
if (any { $_ eq $options->{variant} } ('required', 'buildd', 'minbase')) {
next;
if (none { $_ eq $options->{variant} } ('required', 'buildd', 'minbase')) {
$pkgs_to_install{$pkgname} = ();
}
} elsif ($prio eq 'required') {
# required packages are part of all sets except
# essential and apt
$pkgs_to_install{$pkgname} = ();
} else {
die "unknown priority: $prio";
}
} else {
die "unknown variant: $options->{variant}";
}
$pkgs_to_install{$pkgname} = ();
# reset values
undef $pkgname;
$ess = '';
$prio = 'optional';
}
close $pipe_cat;
@ -1805,7 +1811,7 @@ This section lists some differences to debootstrap.
=item * Multiple ways to operate as non-root: fakechroot, proot, unshare
=item * 2-3 times faster
=item * 3-6 times faster
=item * Can create a chroot with only Essential:yes packages and their dependencies