multistrap/em_multistrap

#!/usr/bin/perl

# Copyright (C) 2009  Neil Williams <codehelp@debian.org>
#
# This package is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

use Config::Auto;
use File::Basename;
use strict;
use vars qw/ $progname $ourversion %scripts $dstrap $script $extra 
 @archives $deb $cachedir $config_str %packages $retval $str $retries
 $dir $include $arch $foreign $suite $url $forceunpack $option %options
 @e $sourcesname $libdir $dpkgdir @debootstrap %suites $mirror $etcdir
 $repo @dirs @touch %sources $section %keys $host $key $value $type
 $file $config /;
$progname = basename($0);

while( @ARGV ) {
	$_= shift( @ARGV );
	last if m/^--$/;
	if (!/^-/) {
		unshift(@ARGV,$_);
		last;
	}
	elsif (/^(-\?|-h|--help|--version)$/) {
	&usageversion();
		exit( 0 );
	}
	elsif (/^(-f|--file)$/) {
		$file = shift(@ARGV);
	}
	elsif (/^(-a|--arch)$/) {
		$arch = shift(@ARGV);
	}
	else {
		die "$progname: Unknown option $_.\n";
	}
}
die ("Need a configuration file - use $progname -f\n")
	if (not defined $file);
$config = Config::Auto::parse("$file");
%keys=();
foreach $key (%$config)
{
	$type = lc($key) if (ref $key ne "HASH");
	$value = $key if (ref $key eq "HASH");
	$keys{$type} = $value;
}
%sources=();
%packages=();
%suites=();
%options=();
%scripts=();
foreach $section (sort keys %keys)
{
	if ($section eq "general")
	{
		$arch = $keys{$section}{'arch'} if (not defined $arch);
		$retries = $keys{$section}{'retries'};
		$dir = $keys{$section}{'directory'};
		$forceunpack = lc($keys{$section}{'forceunpack'});
		@debootstrap = split(' ', lc($keys{$section}{'debootstrap'}));
	}
	else
	{
		$sources{$section}=$keys{$section}{'source'};
		$packages{$section}=$keys{$section}{'packages'};
		$suites{$section}=$keys{$section}{'suite'};
		$scripts{$section}=$keys{$section}{'script'};
		$options{$section}=$keys{$section}{'options'};
	}
}

$host = `dpkg-architecture -qDEB_BUILD_ARCH`;
chomp ($host);
$foreign = ($host ne $arch) ? "--foreign" : '';
$cachedir = "var/cache/apt/"; # archives
$libdir = "var/lib/apt/";     # lists
$etcdir = "etc/apt/";         # sources
$dpkgdir = "var/lib/dpkg/";   # state

mkdir ("$dir") if (not -d "$dir");
system ("mkdir -p ${dir}${cachedir}");
system ("mkdir -p ${dir}${libdir}");
system ("mkdir -p ${dir}${dpkgdir}");
system ("mkdir -p ${dir}etc/apt/sources.list.d/");
@dirs = qw/ alternatives info parts updates/;
@touch = qw/ diversion statoverride status lock/;
foreach my $dpkgd (@dirs) {
	if (not -d "${dir}${dpkgdir}$dpkgd") {
		mkdir "${dir}${dpkgdir}$dpkgd";
	}
}
foreach my $file (@touch) {
	utime(time, time, "${dir}${dpkgdir}/$file") or (
		open(F, ">${dir}${dpkgdir}/$file") && close F )
}
unlink ("${dir}etc/apt/sources.list.d/sources.list")
	if (-f "${dir}etc/apt/sources.list.d/sources.list");
unlink ("${dir}etc/apt/sources.list")
	if (-f "${dir}etc/apt/sources.list");

foreach $repo (sort keys %suites)
{
	if (not -e "${dir}${cachedir}") {
		mkdir "${dir}${cachedir}";
	}
	if (not -e "$dir/${libdir}lists") {
		mkdir "$dir/${libdir}lists";
	}
	if (not -e "$dir/${libdir}lists/partial") {
		mkdir "$dir/${libdir}lists/partial";
	}
	if (not -e "$dir/${cachedir}archives") {
		mkdir "$dir/${cachedir}archives";
	}
	if (not -e "$dir/${cachedir}archives/partial") {
		mkdir "$dir/${cachedir}archives/partial";
	}
	if (-d "${dir}etc/apt/")
	{
		open (SOURCES, ">>${dir}etc/apt/sources.list.d/sources.list") 
			or die "Cannot open sources list $!";
		$mirror = $sources{$repo};
		$suite = $suites{$repo};
		print SOURCES<<END;
deb $mirror $suite main
deb-src $mirror $suite main
END
		close SOURCES;
	}
}

$config_str = '';
$config_str .= " -o Apt::Architecture=$arch";
$config_str .= " -o Apt::Get::AllowUnauthenticated=true";
$config_str .= " -o Apt::Get::Download-Only=true";
$config_str .= " -o Apt::Install-Recommends=false";
$config_str .= " -o Dir=$dir";
$config_str .= " -o Dir::Etc=${dir}${etcdir}";
$sourcesname = "sources.list.d/$repo.sources.list";
$config_str .= " -o Dir::Etc::SourceList=${dir}${etcdir}$sourcesname";
$config_str .= " -o Dir::State=${dir}${libdir}";
$config_str .= " -o Dir::State::Status=${dir}${dpkgdir}/status";
$config_str .= " -o Dir::Cache=${dir}${cachedir}";
system ("apt-get $config_str update");
$str = join (' ', values %packages);
chomp($str);
print "apt-get -y $config_str install $str\n";
$retval = system ("apt-get -y $config_str install $str");
die ("apt download failed. Exit value: ".($retval/256)."\n")
	if ($retval != 0);

foreach $dstrap (@debootstrap)
{
	$url = $sources{$dstrap};
	$suite = $suites{$dstrap};
	$script = $scripts{$dstrap};
	$option = $options{$dstrap};
	$extra = $packages{$dstrap};
	$include = '';
	if (($extra ne '') and ($option !~ /no-resolve-deps/))
	{
		@e = split(' ', $extra);
		$include = "--include ";
		$include .= join (',', @e);
	}
	&write_script ($dstrap);
	$str = "debootstrap $option $include --arch $arch $foreign $suite $dir ";
	$str .= "$url $script";
	print "$str\n";
	$retval = system ($str);
	while ($retval != 0 and $retries > 0)
	{
		print "Problem - trying again, $retries left.\n";
		sleep 1;
		$retval = system ("$str");
		$retries--;
	}
}
if ($forceunpack eq "true")
{
	opendir (DEBS, "${dir}${cachedir}archives/")
		or die ("Cannot read apt archives directory.\n");
	@archives=grep(/.*\.deb$/, readdir DEBS);
	closedir (DEBS);
	chdir ("${dir}");
	foreach $deb (@archives)
	{
		print "Extracting $deb...\n";
		system ("ar -p \"./${cachedir}archives/$deb\" data.tar.gz | zcat | tar -xf -");
	}
}
system ("apt-get $config_str update");

sub write_script
{
	($dstrap) = @_;
	$extra = $packages{$dstrap};
	$script = $scripts{$dstrap};
	return if ($script eq '');
	open (SCRIPT, ">$script") or die ("Cannot open $script. $!\n");
	print SCRIPT<<END;
mirror_style release
download_style apt
finddebs_style hardcoded
variants - buildd fakechroot minbase

work_out_debs () {
    required="$extra"
}

first_stage_install () {
    extract \$required
}

second_stage_install () {
    info BASESUCCESS "Base system installed successfully."
}
END
	close (SCRIPT);
}

sub usageversion {
	print(STDERR <<END)
$progname version $ourversion

Usage:
 $progname [-a ARCH] [-d DIR] -f CONFIG_FILE
 $progname -?|-h|--help|--version

Command:
 -f|--file CONFIG_FILE:  path the the multistrap configuration file.

Options:
 -a|--arch ARCHITECTURE: override the configuration file architecture.
 -d|--dir PATH:          override the configuration file directory.
 -?|-h|--help:           print this usage message and exit
 --version:              print this usage message and exit

$progname extends debootstrap to provide support for multiple
repositories, using a configuration file to specify the relevant suites,
debootstrap options, architecture, extra packages and the mirror to use
for each repository.

Example configuration:
[General]
arch=arm
directory=/opt/multistrap/
retries=5
# extract all downloaded archives as well as those
# unpacked by debootstrap.
forceunpack=false
# the order of sections is important.
# debootstraps are unpacked in this sequence.
debootstrap=Debian

[Debian]
packages=
source=http://ftp.uk.debian.org/debian
suite=lenny
script=
options=

This will result in a completely normal debootstrap of Debian lenny from
the specified mirror, for ARM in /opt/multistrap/.

'Architecture' and 'directory' can be overridden on the command line.

Specify a package to extend the debootstap to include that package and
all dependencies.

Specify a script only if the suite is not currently supported by
debootstrap (which in turn are based on Debian and Ubuntu suite names).

Useful options can include --no-resolve-deps if one of the repositories
contains packages that are also in any of the other repositories but at
a lower version. This prevents debootstrap unpacking the older version
of the duplicated package.

General settings:

The order of repositories specified in the debootstrap (under General),
determines which repository is unpacked first - this is important if one
repository is incomplete or contains older versions.

'directory' specifies the top level directory where the debootstrap
will be created - it is not packed into a .tgz once complete.

'retries' determines how many times debootstrap should be allowed to
restart (the same value is used for all repositories) to allow for
temporary network issues.

If 'arch' does not match the build architecture (determined using
`dpkg-architecture -qDEB_BUILD_ARCH`, --foreign is passed to
each debootstrap.

END
        || die "$progname: failed to write usage: $!\n";
}

=pod

=head1 Name

em_multistrap - extends debootstrap for multiple repository support

=head1 Description

em_multistrap extends debootstrap to provide support for multiple
repositories, using a configuration file to specify the relevant suites,
debootstrap options, architecture, extra packages and the mirror to use
for each repository.

Example configuration:

 [General]
 arch=arm
 directory=/opt/emdebian/debootstrap/multistrap/
 retries=5
 # the order of sections is important.
 # debootstraps are unpacked in this sequence.
 debootstrap=Debian
 
 [Debian]
 packages=
 source=http://ftp.uk.debian.org/debian
 suite=lenny
 script=
 options=

This will result in a completely normal debootstrap of Debian lenny from
the specified mirror using packages for ARM.

Section names are case-insensitive.

Specify a package to extend the debootstap to include that package and
all dependencies.

Specify a script only if the suite is not currently supported by
debootstrap (which in turn are based on Debian and Ubuntu suite names).

Useful options can include --no-resolve-deps if one of the repositories
contains packages that are also in any of the other repositories but at
a lower version. This prevents debootstrap unpacking the older version
of the duplicated package.

General settings:

The order of repositories specified in the debootstrap (under General),
determines which repository is unpacked first - this is important if one
repository is incomplete or contains older versions.

'directory' specifies the top level directory where the debootstrap
will be created - it is not packed into a .tgz once complete.

'retries' determines how many times debootstrap should be allowed to
restart (the same value is used for all repositories) to allow for
temporary network issues.

If 'arch' does not match the build architecture (determined using
`dpkg-architecture -qDEB_BUILD_ARCH`, --foreign is passed to
each debootstrap.

Note that em_multistrap deliberately turns off Install-Recommends.

As with debootstrap, em_multistrap will continue after errors although
you may want to purge $directory/var/cache/apt/archives from time to
time.

em_multistrap does not currently implement machine:variant support
but the build directory is not packed up at the end of the run so
other scripts can be used to implement customisations.

=cut