pbuilder/multistrap : remove debootstrap code and do all the work in apt+dpkg. Improve native support and re-enable.

git-svn-id: http://emdebian.org/svn/current@5723 563faec7-e20c-0410-992a-a66f704d0ccd

bash/multistrap

@@ -0,0 +1,44 @@
+#  bash completion support
+#
+#  Copyright (C) 2009 Neil Williams <codehelp@debian.org>
+#
+#  This package is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+_get_dpkg_cross_list()
+{
+	grep Choices: /var/lib/dpkg/info/dpkg-cross.templates \
+		| cut -d':' -f2 | sed -e 's/None, //' | sed -e 's/,//g'
+}
+
+_em_multistrap()
+{
+	local cur prev opts cmds help machine arch quiet
+	COMPREPLY=()
+	cur="${COMP_WORDS[COMP_CWORD]}"
+	prev="${COMP_WORDS[COMP_CWORD-1]}"
+	help="-h -? --help --version "
+	arch="-a --arch "
+	opts="-f --file"
+	machine=""
+	case "$prev" in
+			-@(a|-arch))
+			COMPREPLY=( $( _get_dpkg_cross_list $cur ) )
+			;;
+			*)
+		COMPREPLY=( $(compgen -W "${arch}${help}${opts}${machine}${cmds}" -- ${cur}) )
+		;;
+	esac
+}
+complete -F _em_multistrap -o default em_multistrap

em_multistrap

@@ -22,12 +22,13 @@ use Parse::Debian::Packages;
 use strict;
 use vars qw/ $progname $ourversion %scripts $dstrap $script $extra 
  @archives $deb $cachedir $config_str %packages $retval $str $retries
- $dir $include $arch $foreign $suite $url $forceunpack $option %options
+ $dir $include $arch $foreign $suite $url $unpack $option %options
  @e $sourcesname $libdir $dpkgdir @debootstrap %suites $mirror $etcdir
  $repo @dirs @touch %sources $section %keys $host $key $value $type
  $file $config $tidy /;
 $progname = basename($0);
-$ourversion = "0.0.3";
+$ourversion = "0.0.4";
+$unpack = "true";
 
 while( @ARGV ) {
 	$_= shift( @ARGV );
@@ -77,9 +78,13 @@ foreach $section (sort keys %keys)
 	if ($section eq "general")
 	{
 		$arch = $keys{$section}{'arch'} if (not defined $arch);
-		$retries = $keys{$section}{'retries'};
 		$dir = $keys{$section}{'directory'} if (not defined $dir);
-		$forceunpack = lc($keys{$section}{'forceunpack'});
+		# support the original value but replace by new value.
+		$unpack = lc($keys{$section}{'unpack'})
+			if (defined $keys{$section}{'forceunpack'});
+		$unpack = lc($keys{$section}{'unpack'})
+			if (defined $keys{$section}{'unpack'});
+		$tidy++ if (defined $keys{$section}{'cleanup'});
 		@debootstrap = split(' ', lc($keys{$section}{'debootstrap'}));
 	}
 	else
@@ -88,16 +93,12 @@ foreach $section (sort keys %keys)
 		$packages{$section}=$keys{$section}{'packages'};
 		$suites{$section}=$keys{$section}{'suite'};
 		$scripts{$section}=$keys{$section}{'script'};
-		$options{$section}=$keys{$section}{'options'};
 	}
 }
 print "$progname $ourversion using $file\n";
 $host = `dpkg-architecture -qDEB_BUILD_ARCH`;
 chomp ($host);
-die ("$progname is not currently able to support native operation.\n")
+$foreign++ if ($host ne $arch);
-	if ($host eq $arch);
-# don't let debootstrap do second-stage
-$foreign = "--foreign";       # always set
 $cachedir = "var/cache/apt/"; # archives
 $libdir = "var/lib/apt/";     # lists
 $etcdir = "etc/apt/";         # sources
@@ -179,38 +180,11 @@ $retval = system ("apt-get -y $config_str install $str");
 die ("apt download failed. Exit value: ".($retval/256)."\n")
 	if ($retval != 0);
 
-foreach $dstrap (@debootstrap)
+&force_unpack if ($unpack eq "true");
-{
-	$url = $sources{$dstrap};
-	$suite = $suites{$dstrap};
-	$script = $scripts{$dstrap};
-	$option = $options{$dstrap};
-	$extra = $packages{$dstrap};
-	$include = '';
-	if (($extra ne '') and ($option !~ /no-resolve-deps/))
-	{
-		@e = split(' ', $extra);
-		$include = "--include ";
-		$include .= join (',', @e);
-	}
-	&write_script ($dstrap);
-	$str = "/usr/sbin/debootstrap $option $include --arch $arch $foreign";
-	$str .= " --keep-debootstrap-dir $suite $dir $url $script";
-	print "$str\n";
-	$retval = system ($str);
-	while ($retval != 0 and $retries > 0)
-	{
-		print "Problem - trying again, $retries left. ".($retval/250)."\n$!\n";
-		sleep 1;
-		$retval = system ("$str");
-		$retries--;
-	}
-}
-
-&force_unpack if ($forceunpack eq "true");
 system ("touch ${dir}${libdir}lists/lock");
 (not defined $tidy) ? system ("apt-get $config_str update") : &tidy_apt;
 system ("rm -rf ${dir}debootstrap");
+&native if (not defined ($foreign));
 print "\nMultistrap system installed successfully in $dir.\n\n";
 exit 0;
 
@@ -353,7 +327,7 @@ sub tidy_apt
 		next unless ($file =~ /\.bin$/);
 		unlink ("${dir}${cachedir}$file");
 	}
-	if ($forceunpack eq "true")
+	if ($unpack eq "true")
 	{
 		opendir (DEBS, "${dir}${cachedir}/archives/")
 			or die ("Cannot read apt archives directory/.\n");
@@ -368,6 +342,13 @@ sub tidy_apt
 	}
 }
 
+# if native arch, do a few tasks just because we can and probably should.
+sub native
+{
+	print "I: Native mode - configuring unpacked packages . . .\n";
+	system ("sudo chroot $dir dpkg --configure -a");
+}
+
 sub get_required_debs
 {
 	# emulate required="$(get_debs Priority: required)"
@@ -380,11 +361,8 @@ sub get_required_debs
 		or die ("Cannot open ${dir}${libdir}lists/ directory. $!\n");
 	my @lists=grep(/_Packages$/, readdir (PKGS));
 	closedir (PKGS);
-	# only read Packages files from the debootstrap entries where
-	# options do not include --no-resolve-deps
 	foreach my $strap (@debootstrap)
 	{
-		next if ($options{$strap} =~ /no-resolve-deps/);
 		my $s = lc($strap);
 		foreach my $l (@lists)
 		{
@@ -405,34 +383,6 @@ sub get_required_debs
 	return \@debs;
 }
 
-sub write_script
-{
-	($dstrap) = @_;
-	$extra = $packages{$dstrap};
-	$script = $scripts{$dstrap};
-	return if ($script eq '');
-	open (SCRIPT, ">$script") or die ("Cannot open $script. $!\n");
-	print SCRIPT<<END;
-mirror_style release
-download_style apt
-finddebs_style hardcoded
-variants - buildd fakechroot minbase
-
-work_out_debs () {
-    required="$extra"
-}
-
-first_stage_install () {
-    extract \$required
-}
-
-second_stage_install () {
-    info BASESUCCESS "Base system installed successfully."
-}
-END
-	close (SCRIPT);
-}
-
 sub usageversion {
 	print(STDERR <<END)
 $progname version $ourversion
@@ -452,19 +402,15 @@ Options:
 
 $progname extends debootstrap to provide support for multiple
 repositories, using a configuration file to specify the relevant suites,
-debootstrap options, architecture, extra packages and the mirror to use
+architecture, extra packages and the mirror to use for each repository.
-for each repository.
 
 Example configuration:
 [General]
-arch=arm
+arch=armel
 directory=/opt/multistrap/
-retries=5
+# extract all downloaded archives (default is true)
-# extract all downloaded archives as well as those
+unpack=true
-# unpacked by debootstrap.
+# the order of sections is not important.
-forceunpack=false
-# the order of sections is important.
-# debootstraps are unpacked in this sequence.
 debootstrap=Debian
 
 [Debian]
@@ -472,41 +418,20 @@ packages=
 source=http://ftp.uk.debian.org/debian
 suite=lenny
 script=
-options=
 
 This will result in a completely normal debootstrap of Debian lenny from
-the specified mirror, for ARM in /opt/multistrap/.
+the specified mirror, for armel in /opt/multistrap/.
 
 'Architecture' and 'directory' can be overridden on the command line.
 
 Specify a package to extend the debootstap to include that package and
 all dependencies.
 
-Specify a script only if the suite is not currently supported by
-debootstrap (which in turn are based on Debian and Ubuntu suite names).
-
-Useful options can include --no-resolve-deps if one of the repositories
-contains packages that are also in any of the other repositories but at
-a lower version. This prevents debootstrap unpacking the older version
-of the duplicated package.
-
 General settings:
 
-The order of repositories specified in the debootstrap (under General),
-determines which repository is unpacked first - this is important if one
-repository is incomplete or contains older versions.
-
 'directory' specifies the top level directory where the debootstrap
 will be created - it is not packed into a .tgz once complete.
 
-'retries' determines how many times debootstrap should be allowed to
-restart (the same value is used for all repositories) to allow for
-temporary network issues.
-
-If 'arch' does not match the build architecture (determined using
-`dpkg-architecture -qDEB_BUILD_ARCH`, --foreign is passed to
-each debootstrap.
-
 END
         || die "$progname: failed to write usage: $!\n";
 }
@@ -532,14 +457,11 @@ for each repository.
 Example configuration:
 
  [General]
- arch=arm
+ arch=armel
  directory=/opt/multistrap/
- retries=5
+ # extract all downloaded archives (default is true)
- # extract all downloaded archives as well as those
+ unpack=true
- # unpacked by debootstrap.
+ # the order of sections is not important.
- forceunpack=false
- # the order of sections is important.
- # debootstraps are unpacked in this sequence.
  debootstrap=Debian
  
  [Debian]
@@ -550,40 +472,21 @@ Example configuration:
  options=
 
 This will result in a completely normal debootstrap of Debian lenny from
-the specified mirror, for ARM in /opt/multistrap/.
+the specified mirror, for armel in /opt/multistrap/.
 
 'Architecture' and 'directory' can be overridden on the command line.
 
 Section names are case-insensitive.
 
-Specify a package to extend the debootstap to include that package and
+Specify a package to extend the multistrap to include that package and
 all dependencies.
 
-Specify a script only if the suite is not currently supported by
-debootstrap (which in turn are based on Debian and Ubuntu suite names).
-
-Useful options can include --no-resolve-deps if one of the repositories
-contains packages that are also in any of the other repositories but at
-a lower version. This prevents debootstrap unpacking the older version
-of the duplicated package.
 
 =head1 General settings:
 
-The order of repositories specified in the debootstrap (under General),
-determines which repository is unpacked first - this is important if one
-repository is incomplete or contains older versions.
-
 'directory' specifies the top level directory where the debootstrap
 will be created - it is not packed into a .tgz once complete.
 
-'retries' determines how many times debootstrap should be allowed to
-restart (the same value is used for all repositories) to allow for
-temporary network issues.
-
-If 'arch' does not match the build architecture (determined using
-`dpkg-architecture -qDEB_BUILD_ARCH`, --foreign is passed to
-each debootstrap.
-
 Note that em_multistrap deliberately turns off Install-Recommends.
 
 As with debootstrap, em_multistrap will continue after errors although
@@ -591,18 +494,32 @@ you may want to purge $directory/var/cache/apt/archives from time to
 time.
 
 em_multistrap does not currently implement machine:variant support
-but the build directory is not packed up at the end of the run so
+used in Emdebian but the build directory is not packed up at the
-other scripts can be used to implement customisations.
+end of the run so other scripts can be used to implement customisations.
 
-=head1 Native mode disabled
+=head1 State
 
-em_multistrap is not intended for native support, it was developed for
+multistrap is stateless - if the directory exists, it will simply
+proceed as normal and apt will try to pick up where it left off.
+
+=head1 Native mode - multistrap
+
+em_multistrap was not intended for native support, it was developed for
 cross architeeture support. In order for multiple repositories to be
 used, em_multistrap sets the --foreign option with debootstrap even if
 debootstrap would be able to complete the installation of each single
 repository. The reason for this is so that only the packages selected
 by apt are actually unpacked.
 
-Currently, em_multistrap disables native support.
+In native mode, various post-multistrap operations are likely to be
+needed that debootstrap would do for you:
+
+ 1. copy /etc/hosts into the chroot
+ 2. clean the environment to unset LANGUAGE, LC_ALL and LANG
+    to silence nuisance perl warnings that obscure other errors
+
+(An alternative to unset the localisation variables is to add
+locales to your multistrap configuration file in the 'packages'
+option.
 
 =cut

multistrap

@@ -0,0 +1,10 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+# Eventually, this will replace the em_ one or call em_ with
+# --arch and take over native duties.
+#debug
+my $str = join(" ", @ARGV);
+system ("sudo /opt/working/emdebian/svn/current/em_multistrap $str");