diff --git a/multistrap b/multistrap index 189baaa..a00fbe4 100755 --- a/multistrap +++ b/multistrap @@ -1,4 +1,5 @@ #!/usr/bin/perl +# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab # Copyright (C) 2009-2015 Neil Williams # Copyright (C) 2015-2017 Johannes Schauer @@ -16,76 +17,60 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +package multistrap; + +# This allows one to use this file as an application with main() as the entry +# point as well as a module to allow unit testing +__PACKAGE__->main() unless caller(); + use strict; use warnings; use IO::File; -use Config::Auto; +use Config::IniFiles; use Cwd qw (realpath getcwd); use File::Basename; -use Parse::Debian::Packages; +use Parse::Debian::Packages; # FIXME: use Dpkg::Index instead use POSIX qw(locale_h); -use Locale::gettext; +use Dpkg::Gettext; use File::Copy; - -main(); +use List::Util qw(any all none); +use Text::Wrap; +use Getopt::Long; +use Pod::Usage; sub main { - use vars qw/ @aptsources %packages $str $retries $dir $arch $foreign - $unpack $sourcedir @debootstrap %suites %components %sources %keys - $preffile $file $tidy $noauth %keyrings $deflist @extrapkgs @includes - $setupsh $configsh $omitrequired $omitpreinst @reinstall $tgzname @check - $explicit_suite $allow_recommends %omitdebsrc @dsclist %flatfile - %important $addimportant @debconf %hooks $warn_count @foreignarches - $ignorenative $markauto $default_release/; - setlocale(LC_MESSAGES, ""); textdomain("multistrap"); my $progname = basename($0); - $default_release = "*"; - $unpack = "true"; - %omitdebsrc=(); - my $dryrun; - my $use_shortcut; - while( @ARGV ) { - $_= shift( @ARGV ); - last if m/^--$/; - if (!/^-/) { - unshift(@ARGV,$_); - last; - } elsif (/^(-\?|-h|--help|--version)$/) { - &usageversion(); - exit( 0 ); - } elsif (/^(-s|--shortcut)$/) { - $use_shortcut = shift(@ARGV); - } elsif (/^(-f|--file)$/) { - $file = shift(@ARGV); - } elsif (/^(-a|--arch)$/) { - $arch = shift(@ARGV); - } elsif (/^(-d|--dir)$/) { - $dir = shift(@ARGV); - $dir .= ($dir =~ m:/$:) ? '' : "/"; - } elsif (/^(--tidy-up)$/) { - $tidy++; - } elsif (/^(--source-dir)$/) { - $sourcedir = shift (@ARGV); - $sourcedir .= ($sourcedir =~ m:/$:) ? '' : "/"; - $sourcedir = (-d $sourcedir) ? $sourcedir : undef; - } elsif (/^(--no-auth)$/) { - $noauth++; - } elsif (/^(--dry-run|--simulate)$/) { - $dryrun++; - } else { - die "$progname: "._g("Unknown option")." $_.\n"; - } + + my $options = {}; + # The long option must come before the short option because the first + # option will become the key in the $options hash. + # + # --man is a hidden option (not documented) + GetOptions ($options, 'help|h', 'man', 'simulate|dry-run', 'shortcut|s=s', + 'file|f=s', 'arch|a=s', 'directory|d=s', 'tidy-up!', 'source-dir=s', 'auth!' + ) or pod2usage(2); + pod2usage(1) if ($options->{help}); + pod2usage(-exitval => 0, -verbose => 2) if ($options->{man}); + pod2usage(-message => "Mandatory argument -f or --file is missing.\n", + -exitval => 1, -verbose => 1) if (! exists $options->{file}); + + if (exists $options->{shortcut} && exists $options->{file}) { + die (_g("Options --shortcut and --file are mutually exclusive\n")); } - if (defined $use_shortcut) { - my $short = "/usr/share/multistrap/".$use_shortcut.".conf"; - $file = $short if (-f $short); - $short = "/etc/multistrap.d/".$use_shortcut.".conf"; - $file = $short if (-f $short); + + my $file; + if (exists $options->{shortcut} && defined $options->{shortcut}) { + # FIXME: use ~/.config/multistrap and XDG paths as well + my $short = "/usr/share/multistrap/".$options->{shortcut}.".conf"; + $file = $short if (-f $short); + $short = "/etc/multistrap.d/".$options->{shortcut}.".conf"; + $file = $short if (-f $short); + } elsif (exists $options->{file} && defined $options->{file}) { } if (not defined $file) { - die (sprintf (_g("Need a configuration file - use %s -f\n"), $progname)); + die (sprintf (_g("Need a configuration file - use %s -f\n"), $progname)); } my $cachedir = "var/cache/apt/"; # archives @@ -93,38 +78,51 @@ sub main { my $etcdir = "etc/apt/"; # sources my $dpkgdir = "var/lib/dpkg/"; # state - my $cfgdir=dirname($file); - cascade($file); + # The "config" is read from configuration files. + # The "options" come from the command line. + # The "settings" are the config and options together. + my $config_tree = parse_ini($file); + my $config = get_config_from_tree($config_tree); + my $settings = resolve_settings(); + # Overwrite settings from the configuration file with settings from the + # command line + if (exists $options->{arch}) { + $settings->{general}{arch} = $options->{arch}; + } + if (exists $options->{directory}) { + $settings->{general}{directory} = $options->{directory}; + } + if (exists $options->{'tidy-up'}) { + $settings->{general}{cleanup} = $options->{'tidy-up'}; + } + if (exists $options->{'source-dir'}) { + $settings->{general}{retainsources} = $options->{'source-dir'}; + } + if (exists $options->{auth}) { + $settings->{general}{noauth} = !$options->{auth}; + } # Translators: fields are programname, include file. - printf (_g("%s using %s\n"), $progname, $file); my $host = `dpkg --print-architecture`; chomp($host); - foreach my $inc (@includes) { - cascade($inc); + if ($settings->{general}{omitrequired} and $settings->{general}{addimportant}) { + warn("\n"._g("Error: Cannot set 'add Priority: important' when packages ". + "of 'Priority: required' are being omitted.\n")); + if (defined $options->{simulate}) { + warn("\n"); + &dump_settings; + exit 0; + } + exit (7); } - if (defined $omitrequired and defined $addimportant) { - warn("\n"._g("Error: Cannot set 'add Priority: important' when packages ". - "of 'Priority: required' are being omitted.\n")); - if (scalar @includes > 0) { - my $plural = ngettext("Please also check the included configuration file:", - "Please also check the included configuration files:", scalar @includes); - warn (sprintf("%s '%s'\n", $plural, join ("', '", sort @includes))); - } - if (defined $dryrun) { - warn("\n"); - &dump_config; - exit 0; - } - exit (7); - } - uniq_sort (\@debootstrap); - uniq_sort (\@aptsources); - if (defined $dryrun) { - &dump_config; - exit 0; + if (defined $options->{simulate}) { + &dump_settings; + exit 0; } + my @debootstrap = uniq_sort (@{$settings->{general}{debootstrap}}, @{$settings->{general}{bootstrap}}); + my @aptsources = uniq_sort (@{$settings->{general}{aptsources}}); # Translators: fields are: programname, configfile. printf (_g("%s using %s\n"), $progname, $file); + my $arch = $settings->{general}{arch}; if ((not defined $arch) or ($arch eq "")) { $arch = $host; printf (_g("Defaulting architecture to native: %s\n"),$arch); @@ -133,9 +131,11 @@ sub main { } else { printf (_g("Using foreign architecture: %s\n"), $arch); } - $foreign++ if (($host ne $arch) or (defined $ignorenative)); + my $foreign; + $foreign++ if (($host ne $arch) or ($settings->{general}{ignorenative})); + my $dir = $settings->{general}{directory}; if (not defined $dir or not defined $arch) { - &dump_config; + &dump_settings; exit 3; } unless (keys %sources) { @@ -261,6 +261,7 @@ sub main { } } my $k; + # FIXME: remove duplicates from %keyrings foreach my $pkg (values %keyrings) { next if (not defined $pkg); next if ("" eq "$pkg"); @@ -291,6 +292,8 @@ sub main { File::Copy::copy "${xdir}/usr/share/keyrings/${gpg}", "${dir}${etcdir}trusted.gpg.d/"; } system ("rm -rf ${xdir}"); + # FIXME: if the globbing was too aggressive, then this + # will remove files that are needed later unlink ($file); } } @@ -320,7 +323,7 @@ sub main { $config_str .= " -o Dir::Etc=" . shellescape("${dir}${etcdir}"); $config_str .= " -o Dir::Etc::Parts=" . shellescape("${dir}${etcdir}apt.conf.d/"); $config_str .= " -o Dir::Etc::PreferencesParts=" . shellescape("${dir}${etcdir}preferences.d/"); - $config_str .= " -o APT::Default-Release=" . shellescape($default_release); + $config_str .= " -o APT::Default-Release=" . shellescape($default_release) if (defined $default_release); # if (not defined $preffile); if (defined $deflist) { my $sourcesname = "sources.list.d/multistrap.sources.list"; @@ -377,7 +380,7 @@ sub main { } chomp($str); @s = split (/ /, $str); - uniq_sort (\@s); + @s = uniq_sort (@s); $str = join (' ', @s); print "$apt_get -y install $str\n"; $retval = 0; @@ -898,7 +901,7 @@ sub handle_source_packages { } } } - uniq_sort (\@dsclist); + @dsclist = uniq_sort (@dsclist); my $olddir = getcwd(); chdir ($sourcedir); if (scalar @dsclist > 0) { @@ -1151,179 +1154,490 @@ will be created - it is not packed into a .tgz once complete. or die ("$progname: ". _g("failed to write usage:") . "$!\n"); } -sub cascade { - $file = shift; - my @req_arches=(); - my $config = Config::Auto::parse($file, format => 'ini'); - if (not defined $config or (scalar keys %$config) == 0) { - my $progname = basename($0); - die ("$progname: ". sprintf(_g("Failed to parse '%s'!\n"), $file)); - } - my $type; - my $value; - foreach my $key (%$config) { - $type = lc($key) if (ref $key ne "HASH"); - $value = $key if (ref $key eq "HASH"); - $keys{$type} = $value; - } - foreach my $section (sort keys %keys) { - if ($section eq "general") { - $arch = $keys{$section}{'arch'} - if (defined $keys{$section}{'arch'} and (not defined $arch)); - $dir = $keys{$section}{'directory'} - if (defined $keys{$section}{'directory'} and (not defined $dir)); - # support the original value but replace by new value. - $unpack = "false" if (defined $keys{$section}{'forceunpack'} and (lc($keys{$section}{'forceunpack'}) ne "true")); - $unpack = "false" if (defined $keys{$section}{'unpack'} and (lc($keys{$section}{'unpack'} ne "true"))); - $markauto++ if ((defined $keys{$section}{'markauto'}) and (lc($keys{$section}{'markauto'}) eq "true")); - $configsh = $keys{$section}{'configscript'} - if (defined $keys{$section}{'configscript'} and (not defined $configsh)); - $tgzname = $keys{$section}{'tarballname'} - if (defined $keys{$section}{'tarballname'} and (not defined $tgzname)); - chomp($tgzname) if (defined $tgzname); - undef $tgzname if (defined $tgzname and $tgzname eq ''); - if ((defined $configsh) and ($configsh eq '')) { - undef $configsh - } - if ((defined $configsh) and (not -x $configsh)) { - my $configmsg = sprintf (_g("INF: '%s' exists but is not executable - ignoring.\n"), $configsh); - undef $configsh; - warn $configmsg; - $warn_count++; - } - $setupsh = $keys{$section}{'setupscript'} - if (defined $keys{$section}{'setupscript'} and (not defined $setupsh)); - undef $setupsh if ((defined $setupsh) and (not -x $setupsh)); - $omitrequired++ if (defined $keys{$section}{'omitrequired'} and (lc($keys{$section}{'omitrequired'}) eq "true")); - $addimportant++ if (defined $keys{$section}{'addimportant'} and (lc($keys{$section}{'addimportant'}) eq "true")); - $omitpreinst++ if (defined $keys{$section}{'omitpreinst'} and ($keys{$section}{'omitpreinst'} eq "true")); - $tidy++ if ((defined $keys{$section}{'cleanup'}) and ($keys{$section}{'cleanup'} eq "true")); - $noauth++ if ((defined $keys{$section}{'noauth'}) and ($keys{$section}{'noauth'} eq "true")); - $ignorenative++ if ((defined $keys{$section}{'ignorenativearch'}) and - (lc($keys{$section}{'ignorenativearch'}) eq 'true')); - $preffile = $keys{$section}{'aptpreferences'} - if (defined $keys{$section}{'aptpreferences'} and (not defined $preffile)); - undef $preffile if ((defined $preffile) and (not -f $preffile)); - $sourcedir = $keys{$section}{'retainsources'} - if ((defined $keys{$section}{'retainsources'}) and (-d $keys{$section}{'retainsources'})); - $explicit_suite++ if ((defined $keys{$section}{'explicitsuite'}) and - ($keys{$section}{'explicitsuite'} eq "true")); - $allow_recommends++ if ((defined $keys{$section}{'allowrecommends'}) and - ($keys{$section}{'allowrecommends'} eq "true")); - $default_release = $keys{$section}{'aptdefaultrelease'} - if (defined $keys{$section}{'aptdefaultrelease'}); - my @p = split(' ', $keys{$section}{'debconfseed'}) - if (defined $keys{$section}{'debconfseed'}); - foreach my $f (@p) { - my $fl = realpath ($f); - next if ($fl eq ""); - next if (not -f $fl); - chomp ($fl); - push @debconf, $fl; - } - my @h = split(' ', $keys{$section}{'hookdir'}) - if (defined $keys{$section}{'hookdir'}); - foreach my $f (@h) { - opendir (HOOKS, "$f") or next; - my @hookfiles=grep(!m:\.\.?$:, readdir HOOKS); - closedir(HOOKS); - foreach my $hf (@hookfiles) { - my $fl = realpath ("$f/$hf"); - next if (($fl eq "") or (not -f $fl) or (not -x $fl)); - push (@{$hooks{'A'}}, $fl) if ($hf =~ /^completion/); - push (@{$hooks{'D'}}, $fl) if ($hf =~ /^download/); - push (@{$hooks{'N'}}, $fl) if ($hf =~ /^native/); - } - } - my @ma = split(' ',$keys{$section}{'multiarch'}) - if (defined $keys{$section}{'multiarch'}); - push @foreignarches, @ma; - my @d=(); - @d = split(' ', lc($keys{$section}{'debootstrap'})) - if (defined $keys{$section}{'debootstrap'}); - push @debootstrap, @d; - my @b = split(' ', lc($keys{$section}{'bootstrap'})) - if (defined $keys{$section}{'bootstrap'}); - push @debootstrap, @b; - my @a=(); - if (exists $keys{$section}{'aptsources'}) { - @a = split (' ', lc($keys{$section}{'aptsources'})); - } - push @aptsources, @a; - my @i = split (' ', $keys{$section}{'include'}) - if (defined $keys{$section}{'include'}); - foreach my $inc (@i) { - # look for the full filepath or try same directory as current conf. - if (not -f $inc) { - my $cfgdir=dirname($file); - my $chk = realpath ("$cfgdir/$inc"); - chomp ($chk) if (defined $chk); - $inc = $chk if (-f $chk); - } - if (not -f $inc) { - my $dirmsg = sprintf (_g("ERR: Cannot find include file: '%s' for '%s'"), $inc, $file); - die ("$dirmsg\n"); - } - } - push @includes, @i; - } else { - $sources{$section}=$keys{$section}{'source'}; - # don't set suite or component if URL is of apt-ftparchive trailing-slash form - # regexp is: optional string in '[]', string without '[' or ']', string ending in '/' - $flatfile{$section}++ if (($sources{$section} =~ /^(\[.*\] )*[^\[\]]+ .+\/$/)); - if ((exists $keys{$section}{'architecture'}) and - ($keys{$section}{'architecture'} ne "")) { - my $frgn_arch = $keys{$section}{'architecture'}; - my @tmp=(); - if (ref ($keys{$section}{'packages'}) eq 'ARRAY') { - foreach my $p (@{$keys{$section}{'packages'}}) { - push @tmp, "$p:$frgn_arch"; - push @req_arches, $frgn_arch; - } - } else { - foreach my $p (split(' ', $keys{$section}{'packages'})) { - push @tmp, "$p:$frgn_arch"; - push @req_arches, $frgn_arch; - } - } - $packages{$section} = join(' ', @tmp); - } else { - if (ref ($keys{$section}{'packages'}) eq 'ARRAY') { - $packages{$section}=join(' ', @{$keys{$section}{'packages'}}); - } else { - $packages{$section}=join(' ', $keys{$section}{'packages'}); - } - } - $suites{$section}=$keys{$section}{'suite'} - if (not exists $suites{$section} and not exists $flatfile{$section}); - $components{$section}=$keys{$section}{'components'} - if (not exists $components{$section} and not exists $flatfile{$section}); - $omitdebsrc{$section}=$section if ((defined $keys{$section}{'omitdebsrc'}) - and ($keys{$section}{'omitdebsrc'} eq "true")); - push @reinstall, split (/ /, $keys{$section}{'reinstall'}) - if (defined $keys{$section}{'reinstall'}); - $components{$section}='main' if (not defined $components{$section}); - $keyrings{$section}=$keys{$section}{'keyring'} if (not exists $keyrings{$section}); - push @extrapkgs, split (' ', $keys{$section}{'additional'}) - if (defined $keys{$section}{'additional'}); - } - } - my %archchk=(); - foreach my $farch (@foreignarches) { - $archchk{$farch}++; - } - foreach my $req (@req_arches) { - if (not exists $archchk{$req}) { - # Translators: %1 and %2 are the same value here - the erroneous architecture name - my $reqmsg = sprintf (_g("ERR: Misconfiguration in: 'architecture' option. ". - "Packages of architecture=%s requested but '%s' is not included in the multiarch=". - join (" ", @foreignarches) . " option.\n"), $req, $req); - warn $reqmsg; - die ("\n"); - } - } - uniq_sort (\@reinstall); - uniq_sort (\@extrapkgs); +my $general_spec = { + arch => { + type => 'string', + help => 'Native architecture'}, + directory => { + type => 'string', + help => 'Output directory' + }, + cleanup => { + type => 'bool', + default => 1, + help => 'remove apt cache data' + }, + noauth => { + type => 'bool', + default => 0, + help => 'Allow the use of unauthenticated repositories' + }, + unpack => { + type => 'bool', + default => 1, + help => 'Extract all downloaded archives' + }, + explicitsuite => { + type => 'bool', + default => 0, + help => 'Suite explicitly selected instead of using latest versions.' + }, + aptsources => { + type => 'section', + list => 1, + help => '' + }, + bootstrap => { + type => 'section', + list => 1, + help => '' + }, + omitrequired => { + type => 'bool', + default => 0, + help => '' + }, + addimportant => { + type => 'bool', + default => 0, + help => '' + }, + debootstrap => { + type => 'section', + list => 1, + default => [], + help => '' + }, + bootstrap => { + type => 'section', + list => 1, + default => [], + help => '' + }, + ignorenative => { + type => 'bool', + default => 0, + help => '' + }, + retainsources => { + type => 'string', + help => '' + }, +}; + +my $section_spec = { + packages => { + type => 'stringlist', + list => 1, + default => [], + help => '' + }, + source => { + type => 'string', + list => 1, + help => '' + }, + keyring => { + type => 'string', + list => 1, + default => [], + help => '' + }, + suite => { + type => 'string', + help => '' + }, + omitdebsrc => { + type => 'bool', + default => 0, + help => '' + }, +}; + +sub get_inclduegraph_from_tree { + my $config_tree = shift; + + if (!exists $config_tree->{general}{include}) { + return []; + } + # Traverse the tree in depth-first-search order. + # + # If the same file occurs in multiple branches of the tree, then the + # resulting graph will be a directed acyclic graph and not a tree + # anymore. + my $includegraph = []; + sub visit { + my $acc = shift; + my $n = shift; + # The origin of the includes in this file must exactly be one file + # and not the result of a merge of two or more files. + if (scalar @{$n->[2]} != 1) { + die "Include statements were merged but that is forbidden"; + } + # Add an edge from the filename of this node to all files that it + # included. + my $f = $n->[2]->[0]; + for my $i (@{$n->[0]}) { + # Make the filename absolute instead of relative to the + # current file + push @{$acc}, [$f, dirname($f) . '/' . $i]; + } + # Recurse. + for my $c (@{$n->[1]}) { + visit($acc, $c); + } + }; + visit($includegraph, $config_tree->{general}{include}); + return $includegraph; +} + +sub get_config_from_tree { + my $config_tree = shift; + + my $config = {}; + foreach my $section (keys %{$config_tree}) { + my $spec; + if ($section eq "general") { + $spec = $general_spec; + } else { + $spec = $section_spec; + } + # First fill the default with the configuration from the spec. + while (my ($k, $v) = each %{$spec}) { + # Do not set values from the spec that do not have a default + if (! exists $v->{default}) { + next; + } + $config->{$section}{$k} = $v->{default}; + } + # Then overwrite the default values with what was read from the config. + while (my ($k, $v) = each %{$config_tree->{$section}}) { + if (! exists $spec->{$k}) { + printf("unknown property: $k\n"); + next; + } + # The "include" parameter of the "general" section is the only one + # where we are interested in more values than from the root node of + # the config tree. We do not handle it here. + if ($section eq 'general' && $k eq 'include') { + next; + } + # Make sure that non-list-type values contain no more than one element + if (scalar @{$v->[0]} > 1 && $spec->{$k}{list} != 1) { + die "property $k must not be a list"; + } + my @value; + # Convert and validate config settings. + if ($spec->{$k}{type} eq 'string') { + @value = @{$v->[0]}; + } elsif ($spec->{$k}{type} eq 'bool') { + my @valid_bool = ('true', 'false', 'yes', 'no', '1', '0'); + # Check if the given value can be interpreted as a boolean. + foreach my $b (@{$v->[0]}) { + if (none {lc($b) eq $_} @valid_bool) { + die "property $k is not a valid boolean"; + } + } + # Check if the given value evaluates to true. + sub is_true { + my $s = shift; + return any { $s eq $_ } ('true', 'yes', '1'); + } + @value = map { is_true(lc($_)) } @{$v->[0]}; + } elsif ($spec->{$k}{type} eq 'section') { + # Check if the given section name matches an existing section. + foreach my $s (@{$v->[0]}) { + foreach my $t (split /\s+/, $s) { + if (lc($t) eq "general") { + die "section name $t forbidden"; + } + if (! exists $config_tree->{lc($t)}) { + die "case-insensitive section name $t not found"; + } + } + } + @value = map { lc } (map { split /\s+/ } @{$v->[0]}); + } elsif ($spec->{$k}{type} eq 'stringlist') { + @value = map { split /\s+/ } @{$v->[0]}; + } else { + die "invalid type: $spec->{$k}{type}"; + } + if (exists $spec->{$k}{list} && $spec->{$k}{list} == 1) { + # If this is a list-type value, store it as an array reference + $config->{$section}{$k} = [@value]; + } else { + # If this is a non-list value, store its first (and only) value as + # a simple scalar + $config->{$section}{$k} = $value[0]; + } + } + } + return $config; +} + +# Write a representation of the include graph in dot format to standard output +sub dump_includegraph { + my $includegraph = shift; + + print "digraph g {\n"; + my %mapping = (); + my $num_verts = 0; + foreach my $e (@{$includegraph}) { + my ($v1, $v2) = @{$e}; + if (! exists $mapping{$v1}) { + $mapping{$v1} = $num_verts; + $num_verts += 1; + } + if (! exists $mapping{$v2}) { + $mapping{$v2} = $num_verts; + $num_verts += 1; + } + } + foreach my $v (sort keys %mapping) { + my $i = $mapping{$v}; + print " $i [label=\"$v\"];\n"; + } + foreach my $e (@{$includegraph}) { + my ($v1, $v2) = @{$e}; + my $i1 = $mapping{$v1}; + my $i2 = $mapping{$v2}; + print " $i1 -> $i2;\n"; + } + print "}\n"; + return $includegraph; +} + +sub dump_settings { + my $settings = shift; + + # Get a representation of the configuration sections such that the + # "general" section comes first and is followed by the others in sorted + # order. + my @sections = sort grep !/^general$/ (keys %{$settings}) + if (exists $settings->{general}) { + unshift @sections, "general"; + } + + sub value_formatter { + my $val = shift; + my $type = shift; + if ($type eq "bool") { + if ($val) { + return "true"; + } else { + return "false"; + } + } elsif ($type eq "section") { + return lc($val); + } elsif ($type eq "stringlist") { + return join " ", @{$val}; + } else { + return $val; + } + }; + + foreach my $section (@sections) { + my $spec; + if ($section eq "general") { + $spec = $general_spec; + } else { + $spec = $section_spec; + } + print("[$section]\n"); + foreach my $k (sort keys %{$settings->{$section}}) { + my $v = $settings->{$section}{$k}; + my $type = $spec->{$k}{type}; + my $t = $spec->{$k}{help}; + if (exists $spec->{$k}{default}) { + $t .= " (default: "; + $t .= value_formatter($spec->{$k}{default}, $type); + $t .= ")"; + } + $t .= "\n"; + print(wrap('# ', '# ', $t)); + if (ref $v eq 'ARRAY') { + foreach my $e (@{$v}) { + print("$k=" . value_formatter($e, $type) . "\n"); + } + } else { + print("$k=" . value_formatter($v, $type) . "\n"); + } + } + } +} + +# Given a config.ini, recursively traverses all included ini files and returns +# a hash which represents a merge of the included ini file tree. +# +# Note, that the resulting data structure will even be a tree if the same ini +# is included by multiple siblings. +# +# The first argument is the ini file to parse. +# +# To prevent cycles, the remaining arguments are the set of ini files that +# make the path of the current config to the root to prevent cycles. +# +# The merging is done such that all configuration values that are specified in +# more than one descendant, are represented as nested array refs representing +# the transitive reduction of the configuration file tree that they appeared +# in. We don't use nested hash refs because those would not remain in order. +# +# Nodes in the tree are represented as tuples (array refs) where the first +# element is the list of values stored in the current node and the second +# element is its list of children. +# +# Example: +# +# complex.ini: +# [general] +# include=blub.ini +# include=bla.ini +# property=1 +# +# branch1.ini: +# [general] +# include=shared.ini +# property=2 +# +# branch2.ini +# [general] +# include=intermediate.ini +# property=3 +# +# intermediate.ini +# [general] +# include=shared.ini +# foo=bar +# +# shared.ini +# [general] +# property=4 +# +# Result: +# +# my %config = { +# general => { +# include => [ +# [ 'branch1.ini', 'branch2.ini' ], +# [ +# [ ['shared.ini'], [] ], +# [ ['intermediate.ini'], [ +# [ ['shared.ini'], [] ] +# ] ] +# ] +# ], +# property => [ +# [ '1' ], +# [ +# [ ['2'], [ [ [ '4' ], [] ] ] ], +# [ ['3'], [ [ [ '4' ], [] ] ] ], +# ] +# ], +# foo => [ [ 'bar' ], [] ] +# } +# } +# +# Observations: +# +# - The full include tree is seen in $config{'global'}{'include'}. +# - Part of the tree is seen in $config{'global'}{'property'}. There is no +# "node" for intermediate.ini because it didn't contain the property. +# - $config{'global'}{'foo'} is a simple scalar because it only occurred once +sub parse_ini { + my $file = shift; + my @seen_includes = @_; + # if this is the first call, then seen_includes might be empty. Then + # add ourselves + if (scalar @seen_includes == 0) { + @seen_includes = ($file); + } + my $progname = basename($0); + printf STDERR (_g("%s using %s\n"), $progname, $file); + tie (my %ini, 'Config::IniFiles', ( + -file => $file, + -nocase => 1, + -allowedcommentchars => '#', + -handle_trailing_comment => 1)) + || die sprintf(_g("Failed to parse '%s'!\n"), $file); + # Go through all included configs, parse them and put the values from + # the results into the SECOND tuple element (the children of this + # config) + my $config; + if (exists $ini{'general'}{'include'}) { + my @includes; + if (ref ($ini{'general'}{'include'}) eq 'ARRAY') { + @includes = @{$ini{'general'}{'include'}}; + } else { + @includes = ($ini{'general'}{'include'}); + } + foreach my $include (@includes) { + if (any { $_ eq $include } @seen_includes) { + die "$include was included already. Cyclic or duplicate includes detected."; + } + my $newini = parse_ini(dirname($file).'/'.$include, uniq_sort(@seen_includes, $include)); + # merge this configuration into the ones that were read so far + foreach my $section (keys %{$newini}) { + # FIXME: we would like to use "each" but there is + # #849298 + foreach my $parameter (keys %{$newini->{$section}}) { + my $value = $newini->{$section}{$parameter}; + if (exists $config->{$section}{$parameter}) { + push @{$config->{$section}{$parameter}[1]}, $value; + push @{$config->{$section}{$parameter}[2]}, $include; + } else { + # parameter doesn't exist, so just copy it + $config->{$section}{$parameter} = [ undef, [$value], [$include] ]; + } + } + } + } + } + # Go through this config and put the read values into the FIRST tuple + # element + foreach my $section (keys %ini) { + foreach my $parameter (keys %{$ini{$section}}) { + my $value = $ini{$section}{$parameter}; + if (ref $value ne 'ARRAY') { + $value = [$value]; + } + if (exists $config->{$section}{$parameter}) { + $config->{$section}{$parameter}[0] = $value; + $config->{$section}{$parameter}[2] = [$file]; + } else { + $config->{$section}{$parameter} = [ $value, [], [$file] ]; + } + } + } + # Go through all config parameters at this level (we don't recurse + # here) and apply a transformation on nodes that were not filled by + # this config file + foreach my $section (keys %{$config}) { + foreach my $parameter (keys %{$config->{$section}}) { + my $value = $config->{$section}{$parameter}; + # only operate on this node if its value is not set + if (defined $value->[0]) { + next; + } + if (scalar @{$value->[1]} == 1) { + # if this node only has a single child, replace this node + # by the child + $config->{$section}{$parameter} = $value->[1]->[0]; + } else { + # concatenate the values of all leave nodes to the value of + # this node + my @leaves = grep {scalar @{$_->[1]} == 0} @{$value->[1]}; + my @nonleaves = grep {scalar @{$_->[1]} != 0} @{$value->[1]}; + $config->{$section}{$parameter} = [ + # make sure to dereference the leave values so + # that we do not get a nested list + [map({@{$_->[0]}} @leaves)], + [@nonleaves], + [map({@{$_->[2]}} @leaves)] + ] + } + } + } + return $config; } sub system_fatal { @@ -1347,186 +1661,86 @@ sub mkdir_fatal { } } -sub _g { - return gettext(shift); -} - sub uniq_sort { - my $aryref = shift; - my %uniq = (); - foreach my $i (@$aryref) { - $uniq{$i}++; - } - @$aryref = sort keys %uniq; + my %uniq; + @uniq{@_} = (); + return sort keys %uniq; } -sub dump_config { - my $msg; - if (not defined $dir or not defined $arch) { - $msg = sprintf(_g("The supplied configuration file '%s'". - " cannot be parsed correctly."), $file); - warn ("\n$msg\n\n"); - } - my $plural; - @check=(); - push @check, @debootstrap; - push @check, @aptsources; - uniq_sort (\@check); - foreach my $sect (@check) { - if (not exists $keys{$sect}) { - $msg .= sprintf (_g("ERR: The '%s' section is not defined.\n"), $sect); - } - } - if (scalar @includes > 0) { - $plural = ngettext("Including configuration file from:", - "Including configuration files from:", scalar @includes); - printf ("include:\t%s '%s'\n", $plural, join ("', '", sort @includes)); - } else { - printf ("include:\t\t"._g("No included configuration files.\n")); - } - undef $plural; - print "\n"; - # explain the bootstrap section details explicitly and just refer to - # those for the apt sources. - foreach my $sect_name (@check) { - next unless (defined $packages{$sect_name}); - printf ("Section name:\t$sect_name\n"); - print "\tsource:\t\t$sources{$sect_name}\n"; - my @sorted = split(/ /, $packages{$sect_name}); - uniq_sort (\@sorted); - print "\tsuite:\t\t$suites{$sect_name}\n" if (not exists $flatfile{$sect_name}); - print "\tcomponents:\t$components{$sect_name}\n" if (not exists $flatfile{$sect_name}); - # only list packages in a bootstrapping section - if (not grep(/^$sect_name$/i, @debootstrap)) { - printf ("\t%s\n",_g("Not listed as a 'Bootstrap' section.")); - print "\n"; - next; - } - print "\tpackages:\t".join(" ", @sorted)."\n"; - print "\n"; - } - $plural = ngettext("Section to install", "Sections to install", scalar @debootstrap); - printf ("%s:\t%s\n", $plural, join(" ", sort @debootstrap)); - $plural = ngettext("Section for updates", "Sections for updates", scalar @aptsources); - printf ("%s:\t%s\n", $plural, join(" ", sort @aptsources)); - my @srcdump=(); - foreach my $src (sort keys %sources) { - next if ((!grep(/^$src$/i, @aptsources)) or (!grep(/^$src$/i, @debootstrap))); - push @srcdump, $sources{$src}; - } - my $srcmsg="omitdebsrc\t\t"._g("Omit deb-src from sources.list for sections:"); - if (scalar keys %omitdebsrc == 0) { - $srcmsg .= sprintf(" %s",_g("None.")); - } else { - foreach my $omit (sort keys %omitdebsrc) { - $srcmsg .= " " . $omitdebsrc{$omit} if (defined $omitdebsrc{$omit}); - } - } - print "$srcmsg\n"; - if (defined $explicit_suite) { - printf("explicitsuite:\t\t"._g("Explicit suite selection: Yes\n")); - } else { - printf("explicitsuite:\t\t"._g("Explicit suite selection: No - let apt use latest.\n")); - } - if (defined $allow_recommends) { - printf("allowrecommends:\t"._g("Recommended packages are added to the selection.\n")); - } else { - printf("allowrecommends:\t"._g("Recommended packages are ignored.\n")); - } - if ($default_release ne "*") { - printf("aptdefaultrelease:\t"."APT::Default-Release: ".$default_release."\n"); - } - if (defined $markauto) { - printf("markauto:\t\t"._g("Marking dependency packages as auto-installed.\n")); - } - $plural = ngettext("Debconf preseed file", "Debconf preseed files", scalar @debconf); - printf("%s:\t%s\n", $plural, join(" ", sort @debconf)) if (scalar @debconf > 0); - if (defined ($hooks{'D'} and scalar @{$hooks{'D'}} > 0)) { - # Translators: leaving the plural blank to keep the lines shorter. - $plural = ngettext ("Download hook: ", "", scalar @{$hooks{'D'}}); - print "download hooks:\t\t$plural".join (", ", sort @{$hooks{'D'}})."\n"; - } - if (defined ($hooks{'N'} and scalar @{$hooks{'N'}} > 0)) { - # Translators: leaving the plural blank to keep the lines shorter. - $plural = ngettext ("Native hook: ", "", scalar @{$hooks{'N'}}); - print "native hooks:\t\t$plural".join (", ", sort @{$hooks{'N'}})."\n"; - } - if (defined ($hooks{'A'} and scalar @{$hooks{'A'}} > 0)) { - # Translators: leaving the plural blank to keep the lines shorter. - $plural = ngettext ("Completion hook: ", "", scalar @{$hooks{'A'}}); - print "completion hooks:\t$plural".join (", ", sort @{$hooks{'A'}})."\n"; - } - $plural = ngettext ("Extra Package: ", "Extra Packages: ", scalar @extrapkgs); - print "additional:\t\t$plural".join (", ", sort @extrapkgs)."\n" if (scalar @extrapkgs > 0); - print "reinstall:\t\t".join (", ", sort (@reinstall))."\n" if (scalar @reinstall > 0); - if (defined $arch and $arch ne "") { - printf ("Architecture:\t\t"._g("Architecture to download: %s\n"), $arch); - } else { - my $host = `dpkg --print-architecture`; - chomp($host); - $msg .= sprintf(_g("Cannot determine architecture from '%s'. Using %s.\n"), $file, $host); - } - if (scalar (@foreignarches) > 0) { - $plural = ngettext("Foreign architecture", "Foreign architectures", scalar @foreignarches); - printf ("MultiArch:\t\t%s: %s\n", $plural, join(" ", sort @foreignarches)); - } - if (defined $dir and $dir ne "") { - printf ("dir:\t\t\t"._g("Output directory: '%s'\n"), $dir); - } else { - $msg .= sprintf(_g("Cannot determine directory from '%s'.\n"), $file); - } - if ($unpack eq "true") { - printf ("unpack:\t\t\t"._g("extract all downloaded archives: %s\n"), $unpack); - } else { - printf ("unpack:\t\t\t"._g("extract all downloaded archives: %s\n"), "false"); - } - print "configscript:\t\t$configsh\n" if (defined $configsh); - printf ("setupscript:\t\t%s: %s",_g("Script to be run after unpacking"),"$setupsh\n") if (defined $setupsh); - if (defined $omitrequired) { - printf ("omitrequired:\t\t%s\n",_g("'Priority required' packages are not included.")); - } else { - printf ("omitrequired:\t\t%s\n",_g("'Priority: required' packages are included.")); - } - if (defined $addimportant) { - printf("addimportant:\t\t"._g("'Priority: important' packages are included.\n")); - } else { - printf("addimportant:\t\t"._g("'Priority: important' packages are ignored.\n")); - } - if (defined $tidy) { - printf ("cleanup:\t\t"._g("remove apt cache data: true\n")); - } else { - printf ("cleanup:\t\t"._g("remove apt cache data: false\n")); - } - if (defined $noauth) { - printf ("noauth:\t\t\t"._g("allow the use of unauthenticated repositories: true\n")); - } else { - printf ("noauth:\t\t\t"._g("allow the use of unauthenticated repositories: false\n")); - } - if (defined $sourcedir) { - printf ("retainsources:\t"._g("Sources will be retained in: %s\n"), $sourcedir); - } - if (defined $tgzname) { - printf ("tarballname:\t\t"._g("Tarball name: '%s'\n"), $tgzname); - } - if (not defined $foreign or not defined $ignorenative) { - if (defined $omitpreinst) { - printf ("omitpreinst:\t\t"._g("Preinst scripts are not executed.\n")); - } else { - printf ("omitpreinst:\t\t"._g("Preinst scripts are executed with the install argument.\n")); - } - printf ("ignorenativearch:\t"._g("Packages will be configured.\n")); - } else { - printf ("omitpreinst:\t\t"._g("Preinst scripts are not executed.\n")); - printf ("ignorenativearch:\t"._g("Packages will not be configured.\n")); - } - if (defined $preffile) { - printf ("aptpreferences:\t\t"._g("Apt preferences file to use: '%s'\n"), $preffile); - } else { - printf ("aptpreferences:\t\t"._g("No apt preferences file. Default release: *\n")); - } - print "\n"; - if (defined $msg) { - warn ("\n$msg\n"); - exit 1; - } -} +__END__ + +=head1 NAME + +multistrap - multiple repository bootstraps + +=head1 SYNOPSIS + + multistrap [-a ARCH] [-d DIR] -f CONFIG_FILE + multistrap [--simulate] -f CONFIG_FILE + multistrap -?|-h|--help|--version + +=head1 OPTIONS + +=head2 General Options + +=over 8 + +=item B<-?|-h|--help|--version> + +output the help text and exit successfully. + +=item B<--dry-run> B<--simulate> + +collate all the configuration settings and output a bare summary. + +=back + +=head2 Configuration Options + +These options overwrite values from the given configuration file which is +documented in L. + +=over 8 + +=item B<-a|--arch> + +architecture of the packages to put into the multistrap. + +=item B<-d|--dir> + +directory into which the bootstrap will be installed. + +=item B<-f|--file> + +configuration file for multistrap [required] + +=item B<-s|--shortcut> + +shortened version of -f for files in known locations without the .conf suffix. +Searched locations are F, F and +F<~/.config/multistrap>. + +=item B<--tidy-up> + +remove apt cache data, downloaded Packages files and the apt package cache. +Same as cleanup=true. + +=item B<--no-auth> + +allow the use of unauthenticated repositories. Same as noauth=true + +=item B<--source-dir> DIR + +move the contents of var/cache/apt/archives/ from inside the chroot to the +specified external directory, then add the Debian source packages for each +used binary. Same as retainsources=DIR If the specified directory does not +exist, nothing is done. Requires --tidy-up in order to calculate the full list +of source packages, including dependencies. + +=back + +=head1 DESCRIPTION + +blubber + +=cut diff --git a/t/config.t b/t/config.t new file mode 100755 index 0000000..8cd334a --- /dev/null +++ b/t/config.t @@ -0,0 +1,103 @@ +#!/usr/bin/perl + +# Copyright (C) 2015-2017 Johannes Schauer +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +use strict; +use warnings; + +use Test::More tests => 6; + +require "./multistrap"; + +my %tests = ( + complex => { + general => { + include => [ + [ 'branch1.ini', 'branch2.ini' ], + [ + [ ['shared.ini'], [], ['t/data/branch1.ini'] ], + [ + ['intermediate.ini'], + [ [ ['shared.ini'], [], ['t/data/intermediate.ini'] ] ], + ['t/data/branch2.ini'] + ] + ], + ['t/data/complex.ini'] + ], + property => [ + ['1'], + [ + [ + ['2'], [ [ ['4'], [], ['t/data/shared.ini'] ] ], + ['t/data/branch1.ini'] + ], + [ + ['3'], [ [ ['4'], [], ['t/data/shared.ini'] ] ], + ['t/data/branch2.ini'] + ], + ], + ['t/data/complex.ini'] + ], + foo => [ ['bar'], [], ['t/data/intermediate.ini'] ] + } + }, + shared => { + general => { property => [ ['4'], [], ['t/data/shared.ini'] ] } + }, + intermediate => { + general => { + include => [ ['shared.ini'], [], ['t/data/intermediate.ini'] ], + property => [ ['4'], [], ['t/data/shared.ini'] ], + foo => [ ['bar'], [], ['t/data/intermediate.ini'] ] + } + }, + branch1 => { + general => { + include => [ ['shared.ini'], [], ['t/data/branch1.ini'] ], + property => [ + ['2'], [ [ ['4'], [], ['t/data/shared.ini'] ] ], + ['t/data/branch1.ini'] + ] + } + }, + branch2 => { + general => { + include => [ + ['intermediate.ini'], + [ [ ['shared.ini'], [], ['t/data/intermediate.ini'] ] ], + ['t/data/branch2.ini'] + ], + property => [ + ['3'], [ [ ['4'], [], ['t/data/shared.ini'] ] ], + ['t/data/branch2.ini'] + ], + foo => [ ['bar'], [], ['t/data/intermediate.ini'] ] + } + }, + concat => { + general => { + property => [ + [ '4', '1' ], [], [ 't/data/shared.ini', 't/data/simple.ini' ] + ], + include => + [ [ 'shared.ini', 'simple.ini' ], [], ['t/data/concat.ini'] ] + } + } +); + +use Data::Dumper; +print(Data::Dumper->Dump([multistrap::cascade("t/data/concat.ini")])); + +foreach my $k ( sort keys %tests ) { + is_deeply( multistrap::cascade("t/data/$k.ini"), $tests{$k}, "$k.ini" ); +} diff --git a/t/critic.t b/t/critic.t new file mode 100755 index 0000000..3219f34 --- /dev/null +++ b/t/critic.t @@ -0,0 +1,21 @@ +#!/usr/bin/perl + +# Copyright (C) 2015-2017 Johannes Schauer +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +use strict; +use warnings; + +use Test::Perl::Critic (-severity => 1); +use Test::More tests => 1; + +critic_ok("./multistrap"); diff --git a/t/data/branch1.ini b/t/data/branch1.ini new file mode 100644 index 0000000..266fc7b --- /dev/null +++ b/t/data/branch1.ini @@ -0,0 +1,3 @@ +[general] +include=shared.ini +property=2 diff --git a/t/data/branch2.ini b/t/data/branch2.ini new file mode 100644 index 0000000..8f090aa --- /dev/null +++ b/t/data/branch2.ini @@ -0,0 +1,3 @@ +[general] +include=intermediate.ini +property=3 diff --git a/t/data/complex.ini b/t/data/complex.ini new file mode 100644 index 0000000..c29ac27 --- /dev/null +++ b/t/data/complex.ini @@ -0,0 +1,4 @@ +[general] +include=branch1.ini +include=branch2.ini +property=1 diff --git a/t/data/concat.ini b/t/data/concat.ini new file mode 100644 index 0000000..2463a6a --- /dev/null +++ b/t/data/concat.ini @@ -0,0 +1,3 @@ +[general] +include=shared.ini +include=simple.ini diff --git a/t/data/intermediate.ini b/t/data/intermediate.ini new file mode 100644 index 0000000..edbaaf9 --- /dev/null +++ b/t/data/intermediate.ini @@ -0,0 +1,3 @@ +[general] +include=shared.ini +foo=bar diff --git a/t/data/shared.ini b/t/data/shared.ini new file mode 100644 index 0000000..4bf71a0 --- /dev/null +++ b/t/data/shared.ini @@ -0,0 +1,2 @@ +[general] +property=4 diff --git a/t/data/simple.ini b/t/data/simple.ini new file mode 100644 index 0000000..14342ec --- /dev/null +++ b/t/data/simple.ini @@ -0,0 +1,2 @@ +[general] +property=1 diff --git a/t/perltidy.t b/t/perltidy.t new file mode 100644 index 0000000..dc21d82 --- /dev/null +++ b/t/perltidy.t @@ -0,0 +1,5 @@ +#!/usr/bin/perl + +use Test::PerlTidy; + +run_tests(perltidyrc => 't/perltidyrc');