pbuilder/multistrap : Support Secure Apt.

git-svn-id: http://emdebian.org/svn/current@5727 563faec7-e20c-0410-992a-a66f704d0ccd

em_multistrap

@@ -20,14 +20,15 @@ use Config::Auto;
 use File::Basename;
 use Parse::Debian::Packages;
 use strict;
-use vars qw/ $progname $ourversion %scripts $dstrap $script $extra 
+use warnings;
+use vars qw/ $progname $ourversion $dstrap $extra 
  @archives $deb $cachedir $config_str %packages $retval $str $retries
- $dir $include $arch $foreign $suite $url $unpack $option %options
+ $dir $include $arch $foreign $suite $url $unpack
  @e $sourcesname $libdir $dpkgdir @debootstrap %suites $mirror $etcdir
  $repo @dirs @touch %sources $section %keys $host $key $value $type
- $file $config $tidy /;
+ $file $config $tidy $noauth $keyring %keyrings /;
 $progname = basename($0);
-$ourversion = "0.0.4";
+$ourversion = "0.0.5";
 $unpack = "true";
 
 while( @ARGV ) {
@@ -54,6 +55,9 @@ while( @ARGV ) {
 	elsif (/^(--tidy-up)$/) {
 		$tidy++;
 	}
+	elsif (/^(--no-auth)$/) {
+		$noauth++;
+	}
 	else {
 		die "$progname: Unknown option $_.\n";
 	}
@@ -71,8 +75,7 @@ foreach $key (%$config)
 %sources=();
 %packages=();
 %suites=();
-%options=();
+%keyrings=();
-%scripts=();
 foreach $section (sort keys %keys)
 {
 	if ($section eq "general")
@@ -84,7 +87,10 @@ foreach $section (sort keys %keys)
 			if (defined $keys{$section}{'forceunpack'});
 		$unpack = lc($keys{$section}{'unpack'})
 			if (defined $keys{$section}{'unpack'});
-		$tidy++ if (defined $keys{$section}{'cleanup'});
+		$tidy++ if ((defined $keys{$section}{'cleanup'}) and
+			($keys{$section}{'cleanup'} eq "true"));
+		$noauth++ if ((defined $keys{$section}{'noauth'}) and
+			($keys{$section}{'noauth'} eq "true"));
 		@debootstrap = split(' ', lc($keys{$section}{'debootstrap'}));
 	}
 	else
@@ -92,13 +98,14 @@ foreach $section (sort keys %keys)
 		$sources{$section}=$keys{$section}{'source'};
 		$packages{$section}=$keys{$section}{'packages'};
 		$suites{$section}=$keys{$section}{'suite'};
-		$scripts{$section}=$keys{$section}{'script'};
+		$keyrings{$section}=$keys{$section}{'keyring'};
 	}
 }
 print "$progname $ourversion using $file\n";
 $host = `dpkg-architecture -qDEB_BUILD_ARCH`;
 chomp ($host);
 $foreign++ if ($host ne $arch);
+print "$progname building '$arch' multistrap on '$host'\n";
 $cachedir = "var/cache/apt/"; # archives
 $libdir = "var/lib/apt/";     # lists
 $etcdir = "etc/apt/";         # sources
@@ -110,7 +117,7 @@ system ("mkdir -p ${dir}${libdir}");
 system ("mkdir -p ${dir}${dpkgdir}");
 system ("mkdir -p ${dir}etc/apt/sources.list.d/");
 @dirs = qw/ alternatives info parts updates/;
-@touch = qw/ diversion statoverride status lock/;
+@touch = qw/ diversions statoverride status lock/;
 foreach my $dpkgd (@dirs) {
 	if (not -d "${dir}${dpkgdir}$dpkgd") {
 		mkdir "${dir}${dpkgdir}$dpkgd";
@@ -118,10 +125,13 @@ foreach my $dpkgd (@dirs) {
 }
 foreach my $file (@touch) {
 	utime(time, time, "${dir}${dpkgdir}/$file") or (
-		open(F, ">${dir}${dpkgdir}/$file") && close F )
+		open(F, ">${dir}${dpkgdir}/$file") && close F );
 }
-unlink ("${dir}etc/apt/sources.list.d/sources.list")
+utime(time, time, "${dir}etc/shells") or 
-	if (-f "${dir}etc/apt/sources.list.d/sources.list");
+	(open(F, ">${dir}etc/shells") && close F );
+
+unlink ("${dir}etc/apt/sources.list.d/multistrap.sources.list")
+	if (-f "${dir}etc/apt/sources.list.d/multistrap.sources.list");
 unlink ("${dir}etc/apt/sources.list")
 	if (-f "${dir}etc/apt/sources.list");
 
@@ -144,7 +154,7 @@ foreach $repo (sort keys %suites)
 	}
 	if (-d "${dir}etc/apt/")
 	{
-		open (SOURCES, ">>${dir}etc/apt/sources.list.d/sources.list") 
+		open (SOURCES, ">>${dir}etc/apt/sources.list.d/multistrap.sources.list") 
 			or die "Cannot open sources list $!";
 		$mirror = $sources{$repo};
 		$suite = $suites{$repo};
@@ -156,22 +166,33 @@ END
 	}
 }
 
+foreach $keyring (sort values %keyrings)
+{
+	my $e=`printenv`;
+	my $str = ($e =~ /\nUSER=root\n/) ? "" : "sudo ";
+	$str = (-f "/usr/bin/sudo") ? "$str" : "";
+	print "I: Installing $keyring\n";
+	system ("$str apt-get install $keyring");
+}
+
 $config_str = '';
 $config_str .= " -o Apt::Architecture=$arch";
-$config_str .= " -o Apt::Get::AllowUnauthenticated=true";
+$config_str .= " -o Apt::Get::AllowUnauthenticated=true"
+	if (defined $noauth);
 $config_str .= " -o Apt::Get::Download-Only=true";
 $config_str .= " -o Apt::Install-Recommends=false";
 $config_str .= " -o Dir=$dir";
 $config_str .= " -o Dir::Etc=${dir}${etcdir}";
-$sourcesname = "sources.list.d/$repo.sources.list";
+$sourcesname = "sources.list.d/multistrap.sources.list";
 $config_str .= " -o Dir::Etc::SourceList=${dir}${etcdir}$sourcesname";
 $config_str .= " -o Dir::State=${dir}${libdir}";
 $config_str .= " -o Dir::State::Status=${dir}${dpkgdir}/status";
 $config_str .= " -o Dir::Cache=${dir}${cachedir}";
 system ("apt-get $config_str update");
-$str = join (' ', values %packages);
+$str = join (' ', values %packages) . " ";
+chomp($str);
+$str .= join (' ', values %keyrings) . " ";
 chomp($str);
-$str .= " ";
 my $required = &get_required_debs;
 $str .= join (' ', @$required);
 chomp($str);
@@ -182,9 +203,8 @@ die ("apt download failed. Exit value: ".($retval/256)."\n")
 
 &force_unpack if ($unpack eq "true");
 system ("touch ${dir}${libdir}lists/lock");
-(not defined $tidy) ? system ("apt-get $config_str update") : &tidy_apt;
-system ("rm -rf ${dir}debootstrap");
 &native if (not defined ($foreign));
+(not defined $tidy) ? system ("apt-get $config_str update") : &tidy_apt;
 print "\nMultistrap system installed successfully in $dir.\n\n";
 exit 0;
 
@@ -308,6 +328,8 @@ sub force_unpack
 sub tidy_apt
 {
 	print "I: Tidying up apt cache and list data.\n";
+	unlink ("${dir}etc/apt/sources.list")
+		if (-f "${dir}etc/apt/sources.list");
 	opendir (DEBS, "${dir}${libdir}lists/")
 		or die ("Cannot read apt lists directory.\n");
 	my @lists=grep(!m:\.\.?$:, readdir DEBS);
@@ -346,7 +368,10 @@ sub tidy_apt
 sub native
 {
 	print "I: Native mode - configuring unpacked packages . . .\n";
-	system ("sudo chroot $dir dpkg --configure -a");
+	my $e=`printenv`;
+	my $str = ($e =~ /\nUSER=root\n/) ? "" : "sudo";
+	$str = (-f "/usr/bin/sudo") ? "$str" : "";
+	system ("$str chroot $dir dpkg --configure -a");
 }
 
 sub get_required_debs
@@ -397,6 +422,8 @@ Command:
 Options:
  -a|--arch ARCHITECTURE: override the configuration file architecture.
  -d|--dir PATH:          override the configuration file directory.
+    --no-auth:           do not use Secure Apt for any repositories
+    --tidy-up:           remove apt cache data and downloaded archives.
  -?|-h|--help:           print this usage message and exit
  --version:              print this usage message and exit
 
@@ -408,16 +435,22 @@ Example configuration:
 [General]
 arch=armel
 directory=/opt/multistrap/
+# same as --tidy-up option if set to true
+cleanup=true
+# same as --no-auth option if set to true
+noauth=false
 # extract all downloaded archives (default is true)
 unpack=true
 # the order of sections is not important.
+# the debootstrap option determines which repository
+# is used to calculate the list of Priority: required packages.
 debootstrap=Debian
 
 [Debian]
 packages=
 source=http://ftp.uk.debian.org/debian
+keyring=debian-archive-keyring
 suite=lenny
-script=
 
 This will result in a completely normal debootstrap of Debian lenny from
 the specified mirror, for armel in /opt/multistrap/.
@@ -425,7 +458,8 @@ the specified mirror, for armel in /opt/multistrap/.
 'Architecture' and 'directory' can be overridden on the command line.
 
 Specify a package to extend the debootstap to include that package and
-all dependencies.
+all dependencies. Dependencies will be calculated by apt so as to use
+only the most recent suitable version from all configured repositories.
 
 General settings:
 
@@ -447,56 +481,128 @@ em_multistrap - extends debootstrap for multiple repository support
  em_multistrap [-a ARCH] [-d DIR] -f CONFIG_FILE
  em_multistrap -?|-h|--help|--version
 
+=head1 Options
+
+(These options can also be set in the configuration file.)
+
+--tidy-up - remove apt cache data, downloaded Packages files and
+the apt package cache. Same as cleanup=true.
+
+--no-auth - allow the use of unauthenticated repositories. Same
+as noauth=true
+
 =head1 Description
 
-em_multistrap extends debootstrap to provide support for multiple
+em_multistrap provides a debootstrap-like method based on apt and
-repositories, using a configuration file to specify the relevant suites,
+extended to provide support for multiple repositories, using a
-debootstrap options, architecture, extra packages and the mirror to use
+configuration file to specify the relevant suites, architecture,
-for each repository.
+extra packages and the mirror to use for each debootstrap.
+
+The aim is to create a complete debootstrap with all packages
+installed and configured, instead of just the base system.
 
 Example configuration:
 
  [General]
  arch=armel
  directory=/opt/multistrap/
+ # same as --tidy-up option if set to true
+ cleanup=true
+ # same as --no-auth option if set to true
+ noauth=false
  # extract all downloaded archives (default is true)
  unpack=true
  # the order of sections is not important.
+ # the debootstrap option determines which repository
+ # is used to calculate the list of Priority: required packages.
  debootstrap=Debian
  
  [Debian]
  packages=
  source=http://ftp.uk.debian.org/debian
+ keyring=debian-archive-keyring
  suite=lenny
- script=
- options=
 
 This will result in a completely normal debootstrap of Debian lenny from
-the specified mirror, for armel in /opt/multistrap/.
+the specified mirror, for armel in '/opt/multistrap/'.
-
-'Architecture' and 'directory' can be overridden on the command line.
-
-Section names are case-insensitive.
 
 Specify a package to extend the multistrap to include that package and
 all dependencies.
 
+Specify more debootstraps by adding new sections. Section names are used
+in the debootstrap general option.
+
+Section names are case-insensitive.
+
+e.g. change
+
+ debootstrap=Debian
+
+to
+
+ debootstrap=Grip
+ 
+then add the new section for Grip:
+ 
+ [Grip]
+ packages=locales
+ keyring=emdebian-archive-keyring
+ source=http://www.emdebian.org/grip
+ suite=lenny
+
+Setting Grip instead of Debian in the debootstrap option, as above,
+will provide a base system from Emdebian Grip 1.0 and locate any
+missing dependencies in Debian 5.0 Lenny, allowing you to add any
+package(s) you need from Debian that are not yet in Emdebian Grip.
+
+All dependencies are resolved only by apt, using all configured
+repositories, to use only the most recent and most suitable
+dependencies. Note that multistrap turns off Install-Recommends
+so if the multistrap needs a package that is only a Recommended
+dependency, the recommended package needs to be specified in the
+packages line explicitly.
+
+'Architecture' and 'directory' can be overridden on the command line.
+Other general options have command line options, except debootstrap
+itself.
 
 =head1 General settings:
 
 'directory' specifies the top level directory where the debootstrap
 will be created - it is not packed into a .tgz once complete.
 
-Note that em_multistrap deliberately turns off Install-Recommends.
+As with debootstrap, em_multistrap will continue after errors.
 
-As with debootstrap, em_multistrap will continue after errors although
+em_multistrap does not currently implement the machine:variant support
-you may want to purge $directory/var/cache/apt/archives from time to
-time.
-
-em_multistrap does not currently implement machine:variant support
 used in Emdebian but the build directory is not packed up at the
 end of the run so other scripts can be used to implement customisations.
 
+=head1 Secure Apt
+
+To use authenticated apt repositories, multistrap either needs to be
+able to install an appropriate keyring package from the existing apt
+sources *outside the multistrap environment* or have the relevant keys
+already configured using apt-key *on the host system*.
+
+If relevant packages exist, specify them in the 'keyring' option for
+each repository. em_multistrap will then check that apt has already
+installed this package so that the repository can be authenticated
+before any packages are downloaded from it.
+
+Note that *all* repositories to be used with multistrap must be
+authenticated or apt will fail. Similarly, secure apt can only be
+disabled for all repositories (by using the --no-auth command line
+option or setting the general noauth option in the configuration
+file), even if only one repository does not have a suitable keyring
+available. Not all packages need keyring packages, if you configure
+apt-key appropriately.
+
+The keyring package(s) will also be installed inside the multistrap
+environment to match the installed apt sources for the multistrap.
+
+All configuration of apt-key needs to be done for the machine
+running multistrap itself.
+
 =head1 State
 
 multistrap is stateless - if the directory exists, it will simply
@@ -505,11 +611,8 @@ proceed as normal and apt will try to pick up where it left off.
 =head1 Native mode - multistrap
 
 em_multistrap was not intended for native support, it was developed for
-cross architeeture support. In order for multiple repositories to be
+cross architecture support. In order for multiple repositories to be
-used, em_multistrap sets the --foreign option with debootstrap even if
+used, em_multistrap only unpacks the packages selected by apt.
-debootstrap would be able to complete the installation of each single
-repository. The reason for this is so that only the packages selected
-by apt are actually unpacked.
 
 In native mode, various post-multistrap operations are likely to be
 needed that debootstrap would do for you:
@@ -522,4 +625,7 @@ needed that debootstrap would do for you:
 locales to your multistrap configuration file in the 'packages'
 option.
 
+Eventually, multistrap will either replace em_multistrap or call
+em_multistrap with --arch and take over native duties.
+
 =cut