From 38e93256e41e85fb885a128631c010bebe4dfca9 Mon Sep 17 00:00:00 2001 From: codehelp Date: Mon, 23 Mar 2009 22:28:50 +0000 Subject: [PATCH] pbuilder/multistrap : Support Secure Apt. git-svn-id: http://emdebian.org/svn/current@5727 563faec7-e20c-0410-992a-a66f704d0ccd --- em_multistrap | 194 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 150 insertions(+), 44 deletions(-) diff --git a/em_multistrap b/em_multistrap index 9a39c3f..2443260 100755 --- a/em_multistrap +++ b/em_multistrap @@ -20,14 +20,15 @@ use Config::Auto; use File::Basename; use Parse::Debian::Packages; use strict; -use vars qw/ $progname $ourversion %scripts $dstrap $script $extra +use warnings; +use vars qw/ $progname $ourversion $dstrap $extra @archives $deb $cachedir $config_str %packages $retval $str $retries - $dir $include $arch $foreign $suite $url $unpack $option %options + $dir $include $arch $foreign $suite $url $unpack @e $sourcesname $libdir $dpkgdir @debootstrap %suites $mirror $etcdir $repo @dirs @touch %sources $section %keys $host $key $value $type - $file $config $tidy /; + $file $config $tidy $noauth $keyring %keyrings /; $progname = basename($0); -$ourversion = "0.0.4"; +$ourversion = "0.0.5"; $unpack = "true"; while( @ARGV ) { @@ -54,6 +55,9 @@ while( @ARGV ) { elsif (/^(--tidy-up)$/) { $tidy++; } + elsif (/^(--no-auth)$/) { + $noauth++; + } else { die "$progname: Unknown option $_.\n"; } @@ -71,8 +75,7 @@ foreach $key (%$config) %sources=(); %packages=(); %suites=(); -%options=(); -%scripts=(); +%keyrings=(); foreach $section (sort keys %keys) { if ($section eq "general") @@ -84,7 +87,10 @@ foreach $section (sort keys %keys) if (defined $keys{$section}{'forceunpack'}); $unpack = lc($keys{$section}{'unpack'}) if (defined $keys{$section}{'unpack'}); - $tidy++ if (defined $keys{$section}{'cleanup'}); + $tidy++ if ((defined $keys{$section}{'cleanup'}) and + ($keys{$section}{'cleanup'} eq "true")); + $noauth++ if ((defined $keys{$section}{'noauth'}) and + ($keys{$section}{'noauth'} eq "true")); @debootstrap = split(' ', lc($keys{$section}{'debootstrap'})); } else @@ -92,13 +98,14 @@ foreach $section (sort keys %keys) $sources{$section}=$keys{$section}{'source'}; $packages{$section}=$keys{$section}{'packages'}; $suites{$section}=$keys{$section}{'suite'}; - $scripts{$section}=$keys{$section}{'script'}; + $keyrings{$section}=$keys{$section}{'keyring'}; } } print "$progname $ourversion using $file\n"; $host = `dpkg-architecture -qDEB_BUILD_ARCH`; chomp ($host); $foreign++ if ($host ne $arch); +print "$progname building '$arch' multistrap on '$host'\n"; $cachedir = "var/cache/apt/"; # archives $libdir = "var/lib/apt/"; # lists $etcdir = "etc/apt/"; # sources @@ -110,7 +117,7 @@ system ("mkdir -p ${dir}${libdir}"); system ("mkdir -p ${dir}${dpkgdir}"); system ("mkdir -p ${dir}etc/apt/sources.list.d/"); @dirs = qw/ alternatives info parts updates/; -@touch = qw/ diversion statoverride status lock/; +@touch = qw/ diversions statoverride status lock/; foreach my $dpkgd (@dirs) { if (not -d "${dir}${dpkgdir}$dpkgd") { mkdir "${dir}${dpkgdir}$dpkgd"; @@ -118,10 +125,13 @@ foreach my $dpkgd (@dirs) { } foreach my $file (@touch) { utime(time, time, "${dir}${dpkgdir}/$file") or ( - open(F, ">${dir}${dpkgdir}/$file") && close F ) + open(F, ">${dir}${dpkgdir}/$file") && close F ); } -unlink ("${dir}etc/apt/sources.list.d/sources.list") - if (-f "${dir}etc/apt/sources.list.d/sources.list"); +utime(time, time, "${dir}etc/shells") or + (open(F, ">${dir}etc/shells") && close F ); + +unlink ("${dir}etc/apt/sources.list.d/multistrap.sources.list") + if (-f "${dir}etc/apt/sources.list.d/multistrap.sources.list"); unlink ("${dir}etc/apt/sources.list") if (-f "${dir}etc/apt/sources.list"); @@ -144,7 +154,7 @@ foreach $repo (sort keys %suites) } if (-d "${dir}etc/apt/") { - open (SOURCES, ">>${dir}etc/apt/sources.list.d/sources.list") + open (SOURCES, ">>${dir}etc/apt/sources.list.d/multistrap.sources.list") or die "Cannot open sources list $!"; $mirror = $sources{$repo}; $suite = $suites{$repo}; @@ -156,22 +166,33 @@ END } } +foreach $keyring (sort values %keyrings) +{ + my $e=`printenv`; + my $str = ($e =~ /\nUSER=root\n/) ? "" : "sudo "; + $str = (-f "/usr/bin/sudo") ? "$str" : ""; + print "I: Installing $keyring\n"; + system ("$str apt-get install $keyring"); +} + $config_str = ''; $config_str .= " -o Apt::Architecture=$arch"; -$config_str .= " -o Apt::Get::AllowUnauthenticated=true"; +$config_str .= " -o Apt::Get::AllowUnauthenticated=true" + if (defined $noauth); $config_str .= " -o Apt::Get::Download-Only=true"; $config_str .= " -o Apt::Install-Recommends=false"; $config_str .= " -o Dir=$dir"; $config_str .= " -o Dir::Etc=${dir}${etcdir}"; -$sourcesname = "sources.list.d/$repo.sources.list"; +$sourcesname = "sources.list.d/multistrap.sources.list"; $config_str .= " -o Dir::Etc::SourceList=${dir}${etcdir}$sourcesname"; $config_str .= " -o Dir::State=${dir}${libdir}"; $config_str .= " -o Dir::State::Status=${dir}${dpkgdir}/status"; $config_str .= " -o Dir::Cache=${dir}${cachedir}"; system ("apt-get $config_str update"); -$str = join (' ', values %packages); +$str = join (' ', values %packages) . " "; +chomp($str); +$str .= join (' ', values %keyrings) . " "; chomp($str); -$str .= " "; my $required = &get_required_debs; $str .= join (' ', @$required); chomp($str); @@ -182,9 +203,8 @@ die ("apt download failed. Exit value: ".($retval/256)."\n") &force_unpack if ($unpack eq "true"); system ("touch ${dir}${libdir}lists/lock"); -(not defined $tidy) ? system ("apt-get $config_str update") : &tidy_apt; -system ("rm -rf ${dir}debootstrap"); &native if (not defined ($foreign)); +(not defined $tidy) ? system ("apt-get $config_str update") : &tidy_apt; print "\nMultistrap system installed successfully in $dir.\n\n"; exit 0; @@ -308,6 +328,8 @@ sub force_unpack sub tidy_apt { print "I: Tidying up apt cache and list data.\n"; + unlink ("${dir}etc/apt/sources.list") + if (-f "${dir}etc/apt/sources.list"); opendir (DEBS, "${dir}${libdir}lists/") or die ("Cannot read apt lists directory.\n"); my @lists=grep(!m:\.\.?$:, readdir DEBS); @@ -346,7 +368,10 @@ sub tidy_apt sub native { print "I: Native mode - configuring unpacked packages . . .\n"; - system ("sudo chroot $dir dpkg --configure -a"); + my $e=`printenv`; + my $str = ($e =~ /\nUSER=root\n/) ? "" : "sudo"; + $str = (-f "/usr/bin/sudo") ? "$str" : ""; + system ("$str chroot $dir dpkg --configure -a"); } sub get_required_debs @@ -397,6 +422,8 @@ Command: Options: -a|--arch ARCHITECTURE: override the configuration file architecture. -d|--dir PATH: override the configuration file directory. + --no-auth: do not use Secure Apt for any repositories + --tidy-up: remove apt cache data and downloaded archives. -?|-h|--help: print this usage message and exit --version: print this usage message and exit @@ -408,16 +435,22 @@ Example configuration: [General] arch=armel directory=/opt/multistrap/ +# same as --tidy-up option if set to true +cleanup=true +# same as --no-auth option if set to true +noauth=false # extract all downloaded archives (default is true) unpack=true # the order of sections is not important. +# the debootstrap option determines which repository +# is used to calculate the list of Priority: required packages. debootstrap=Debian [Debian] packages= source=http://ftp.uk.debian.org/debian +keyring=debian-archive-keyring suite=lenny -script= This will result in a completely normal debootstrap of Debian lenny from the specified mirror, for armel in /opt/multistrap/. @@ -425,7 +458,8 @@ the specified mirror, for armel in /opt/multistrap/. 'Architecture' and 'directory' can be overridden on the command line. Specify a package to extend the debootstap to include that package and -all dependencies. +all dependencies. Dependencies will be calculated by apt so as to use +only the most recent suitable version from all configured repositories. General settings: @@ -447,56 +481,128 @@ em_multistrap - extends debootstrap for multiple repository support em_multistrap [-a ARCH] [-d DIR] -f CONFIG_FILE em_multistrap -?|-h|--help|--version +=head1 Options + +(These options can also be set in the configuration file.) + +--tidy-up - remove apt cache data, downloaded Packages files and +the apt package cache. Same as cleanup=true. + +--no-auth - allow the use of unauthenticated repositories. Same +as noauth=true + =head1 Description -em_multistrap extends debootstrap to provide support for multiple -repositories, using a configuration file to specify the relevant suites, -debootstrap options, architecture, extra packages and the mirror to use -for each repository. +em_multistrap provides a debootstrap-like method based on apt and +extended to provide support for multiple repositories, using a +configuration file to specify the relevant suites, architecture, +extra packages and the mirror to use for each debootstrap. + +The aim is to create a complete debootstrap with all packages +installed and configured, instead of just the base system. Example configuration: [General] arch=armel directory=/opt/multistrap/ + # same as --tidy-up option if set to true + cleanup=true + # same as --no-auth option if set to true + noauth=false # extract all downloaded archives (default is true) unpack=true # the order of sections is not important. + # the debootstrap option determines which repository + # is used to calculate the list of Priority: required packages. debootstrap=Debian [Debian] packages= source=http://ftp.uk.debian.org/debian + keyring=debian-archive-keyring suite=lenny - script= - options= This will result in a completely normal debootstrap of Debian lenny from -the specified mirror, for armel in /opt/multistrap/. - -'Architecture' and 'directory' can be overridden on the command line. - -Section names are case-insensitive. +the specified mirror, for armel in '/opt/multistrap/'. Specify a package to extend the multistrap to include that package and all dependencies. +Specify more debootstraps by adding new sections. Section names are used +in the debootstrap general option. + +Section names are case-insensitive. + +e.g. change + + debootstrap=Debian + +to + + debootstrap=Grip + +then add the new section for Grip: + + [Grip] + packages=locales + keyring=emdebian-archive-keyring + source=http://www.emdebian.org/grip + suite=lenny + +Setting Grip instead of Debian in the debootstrap option, as above, +will provide a base system from Emdebian Grip 1.0 and locate any +missing dependencies in Debian 5.0 Lenny, allowing you to add any +package(s) you need from Debian that are not yet in Emdebian Grip. + +All dependencies are resolved only by apt, using all configured +repositories, to use only the most recent and most suitable +dependencies. Note that multistrap turns off Install-Recommends +so if the multistrap needs a package that is only a Recommended +dependency, the recommended package needs to be specified in the +packages line explicitly. + +'Architecture' and 'directory' can be overridden on the command line. +Other general options have command line options, except debootstrap +itself. =head1 General settings: 'directory' specifies the top level directory where the debootstrap will be created - it is not packed into a .tgz once complete. -Note that em_multistrap deliberately turns off Install-Recommends. +As with debootstrap, em_multistrap will continue after errors. -As with debootstrap, em_multistrap will continue after errors although -you may want to purge $directory/var/cache/apt/archives from time to -time. - -em_multistrap does not currently implement machine:variant support +em_multistrap does not currently implement the machine:variant support used in Emdebian but the build directory is not packed up at the end of the run so other scripts can be used to implement customisations. +=head1 Secure Apt + +To use authenticated apt repositories, multistrap either needs to be +able to install an appropriate keyring package from the existing apt +sources *outside the multistrap environment* or have the relevant keys +already configured using apt-key *on the host system*. + +If relevant packages exist, specify them in the 'keyring' option for +each repository. em_multistrap will then check that apt has already +installed this package so that the repository can be authenticated +before any packages are downloaded from it. + +Note that *all* repositories to be used with multistrap must be +authenticated or apt will fail. Similarly, secure apt can only be +disabled for all repositories (by using the --no-auth command line +option or setting the general noauth option in the configuration +file), even if only one repository does not have a suitable keyring +available. Not all packages need keyring packages, if you configure +apt-key appropriately. + +The keyring package(s) will also be installed inside the multistrap +environment to match the installed apt sources for the multistrap. + +All configuration of apt-key needs to be done for the machine +running multistrap itself. + =head1 State multistrap is stateless - if the directory exists, it will simply @@ -505,11 +611,8 @@ proceed as normal and apt will try to pick up where it left off. =head1 Native mode - multistrap em_multistrap was not intended for native support, it was developed for -cross architeeture support. In order for multiple repositories to be -used, em_multistrap sets the --foreign option with debootstrap even if -debootstrap would be able to complete the installation of each single -repository. The reason for this is so that only the packages selected -by apt are actually unpacked. +cross architecture support. In order for multiple repositories to be +used, em_multistrap only unpacks the packages selected by apt. In native mode, various post-multistrap operations are likely to be needed that debootstrap would do for you: @@ -522,4 +625,7 @@ needed that debootstrap would do for you: locales to your multistrap configuration file in the 'packages' option. +Eventually, multistrap will either replace em_multistrap or call +em_multistrap with --arch and take over native duties. + =cut