josch/multistrap
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

pbuilder/em_multistrap : add support for complete unpacking, ported from emsandbox, with options to remove the .debs and cache data. Disable broken native support.

Browse source 
git-svn-id: http://emdebian.org/svn/current@5691 563faec7-e20c-0410-992a-a66f704d0ccd
This commit is contained in:
codehelp 2009-03-08 18:10:24 +00:00
parent adcf3a18a3
commit 05bad91461
1 changed files with 237 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

252
em_multistrap
View file

@ -15,16 +15,19 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
use IO::File;
use Config::Auto; use Config::Auto;
use File::Basename; use File::Basename;
use Parse::Debian::Packages;
use strict; use strict;
use vars qw/ $progname $ourversion %scripts $dstrap $script $extra use vars qw/ $progname $ourversion %scripts $dstrap $script $extra
@archives $deb $cachedir $config_str %packages $retval $str $retries @archives $deb $cachedir $config_str %packages $retval $str $retries
$dir $include $arch $foreign $suite $url $forceunpack $option %options $dir $include $arch $foreign $suite $url $forceunpack $option %options
@e $sourcesname $libdir $dpkgdir @debootstrap %suites $mirror $etcdir @e $sourcesname $libdir $dpkgdir @debootstrap %suites $mirror $etcdir
$repo @dirs @touch %sources $section %keys $host $key $value $type $repo @dirs @touch %sources $section %keys $host $key $value $type
$file $config /; $file $config $tidy /;
$progname = basename($0); $progname = basename($0);
$ourversion = "0.0.3";
while( @ARGV ) { while( @ARGV ) {
$_= shift( @ARGV ); $_= shift( @ARGV );
@ -43,6 +46,13 @@ while( @ARGV ) {
elsif (/^(-a|--arch)$/) { elsif (/^(-a|--arch)$/) {
$arch = shift(@ARGV); $arch = shift(@ARGV);
} }
elsif (/^(-d|--dir)$/) {
$dir = shift(@ARGV);
$dir .= ($dir =~ m:/$:) ? '' : "/";
}
elsif (/^(--tidy-up)$/) {
$tidy++;
}
else { else {
die "$progname: Unknown option $_.\n"; die "$progname: Unknown option $_.\n";
} }
@ -68,7 +78,7 @@ foreach $section (sort keys %keys)
{ {
$arch = $keys{$section}{'arch'} if (not defined $arch); $arch = $keys{$section}{'arch'} if (not defined $arch);
$retries = $keys{$section}{'retries'}; $retries = $keys{$section}{'retries'};
$dir = $keys{$section}{'directory'}; $dir = $keys{$section}{'directory'} if (not defined $dir);
$forceunpack = lc($keys{$section}{'forceunpack'}); $forceunpack = lc($keys{$section}{'forceunpack'});
@debootstrap = split(' ', lc($keys{$section}{'debootstrap'})); @debootstrap = split(' ', lc($keys{$section}{'debootstrap'}));
} }
@ -81,10 +91,13 @@ foreach $section (sort keys %keys)
$options{$section}=$keys{$section}{'options'}; $options{$section}=$keys{$section}{'options'};
} }
} }
print "$progname $ourversion using $file\n";
$host = `dpkg-architecture -qDEB_BUILD_ARCH`; $host = `dpkg-architecture -qDEB_BUILD_ARCH`;
chomp ($host); chomp ($host);
$foreign = ($host ne $arch) ? "--foreign" : ''; die ("$progname is not currently able to support native operation.\n")
if ($host eq $arch);
# don't let debootstrap do second-stage
$foreign = "--foreign"; # always set
$cachedir = "var/cache/apt/"; # archives $cachedir = "var/cache/apt/"; # archives
$libdir = "var/lib/apt/"; # lists $libdir = "var/lib/apt/"; # lists
$etcdir = "etc/apt/"; # sources $etcdir = "etc/apt/"; # sources
@ -157,6 +170,10 @@ $config_str .= " -o Dir::Cache=${dir}${cachedir}";
system ("apt-get $config_str update"); system ("apt-get $config_str update");
$str = join (' ', values %packages); $str = join (' ', values %packages);
chomp($str); chomp($str);
$str .= " ";
my $required = &get_required_debs;
$str .= join (' ', @$required);
chomp($str);
print "apt-get -y $config_str install $str\n"; print "apt-get -y $config_str install $str\n";
$retval = system ("apt-get -y $config_str install $str"); $retval = system ("apt-get -y $config_str install $str");
die ("apt download failed. Exit value: ".($retval/256)."\n") die ("apt download failed. Exit value: ".($retval/256)."\n")
@ -177,32 +194,216 @@ foreach $dstrap (@debootstrap)
$include .= join (',', @e); $include .= join (',', @e);
} }
&write_script ($dstrap); &write_script ($dstrap);
$str = "debootstrap $option $include --arch $arch $foreign $suite $dir "; $str = "/usr/sbin/debootstrap $option $include --arch $arch $foreign";
$str .= "$url $script"; $str .= " --keep-debootstrap-dir $suite $dir $url $script";
print "$str\n"; print "$str\n";
$retval = system ($str); $retval = system ($str);
while ($retval != 0 and $retries > 0) while ($retval != 0 and $retries > 0)
{ {
print "Problem - trying again, $retries left.\n"; print "Problem - trying again, $retries left. ".($retval/250)."\n$!\n";
sleep 1; sleep 1;
$retval = system ("$str"); $retval = system ("$str");
$retries--; $retries--;
} }
} }
if ($forceunpack eq "true")
&force_unpack if ($forceunpack eq "true");
system ("touch ${dir}${libdir}lists/lock");
(not defined $tidy) ? system ("apt-get $config_str update") : &tidy_apt;
system ("rm -rf ${dir}debootstrap");
print "\nMultistrap system installed successfully in $dir.\n\n";
exit 0;
sub force_unpack
{ {
my %unpack=();
opendir (DEBS, "${dir}${cachedir}archives/") opendir (DEBS, "${dir}${cachedir}archives/")
or die ("Cannot read apt archives directory.\n"); or die ("Cannot read apt archives directory.\n");
@archives=grep(/.*\.deb$/, readdir DEBS); @archives=grep(/.*\.deb$/, readdir DEBS);
closedir (DEBS); closedir (DEBS);
chdir ("${dir}"); print "I: Calculating obsolete packages\n";
foreach $deb (@archives) foreach $deb (sort @archives)
{ {
print "Extracting $deb...\n"; my $version = `dpkg -f ${dir}${cachedir}archives/$deb Version`;
my $package = `dpkg -f ${dir}${cachedir}archives/$deb Package`;
chomp ($version);
chomp ($package);
if (exists $unpack{$package})
{
my $test=system("dpkg --compare-versions $unpack{$package} '<<' $version");
$test /= 256;
# unlink version in $unpack if 0
# unlink $deb (current one) if 1
if ($test == 0)
{
my $old = $deb;
$old =~ s/$version/$unpack{$package}/;
print "I: Removing $old\n";
unlink "${dir}${cachedir}archives/$old";
next;
}
else
{
print "I: Removing $deb\n";
unlink "${dir}${cachedir}archives/$deb";
}
}
$unpack{$package}=$version;
}
open (LOCK, ">${dir}${libdir}lists/lock");
close (LOCK);
opendir (DEBS, "${dir}${cachedir}archives/")
or die ("Cannot read apt archives directory.\n");
@archives=grep(/.*\.deb$/, readdir DEBS);
closedir (DEBS);
my $old = `pwd`;
chomp ($old);
chdir ("${dir}");
foreach $deb (sort @archives)
{
print "I: Extracting $deb...\n";
system ("ar -p \"./${cachedir}archives/$deb\" data.tar.gz | zcat | tar -xf -"); system ("ar -p \"./${cachedir}archives/$deb\" data.tar.gz | zcat | tar -xf -");
my $ver=`dpkg -f ./${cachedir}archives/$deb Version`;
my $pkg=`dpkg -f ./${cachedir}archives/$deb Package`;
chomp ($ver);
chomp ($pkg);
mkdir ("./tmp");
my $tmpdir = `mktemp -p ./tmp -d -t multistrap.XXXXXX`;
chomp ($tmpdir);
mkdir ("./${tmpdir}/listing");
system ("ar -p \"./${cachedir}archives/$deb\" data.tar.gz > ./${tmpdir}/listing/data.tar.gz");
my $datatar = `tar -tzf ./${tmpdir}/listing/data.tar.gz`;
my @lines = split("\n", $datatar);
open (LIST, ">>./${dpkgdir}info/${pkg}.list");
foreach my $l (@lines)
{
chomp ($l);
$l =~ s:^\.::;
$l =~ s:^/$:/\.:;
$l =~ s:/$::;
print LIST "$l\n";
}
close (LIST);
system ("rm -rf ./${tmpdir}/listing");
system ("dpkg -e ./${cachedir}archives/$deb ${tmpdir}/");
opendir (MAINT, "./${tmpdir}");
my @maint=grep(!m:\.\.?:, readdir (MAINT));
closedir (MAINT);
open (AVAIL, ">>./${dpkgdir}available");
open (STATUS, ">>./${dpkgdir}status");
foreach my $mscript (@maint)
{
rename "./${tmpdir}/$mscript", "./${dpkgdir}info/$pkg.$mscript";
if ( $mscript eq "control" )
{
open (MSCRIPT, "./${dpkgdir}info/$pkg.$mscript");
my @scr=<MSCRIPT>;
close (MSCRIPT);
my @avail = grep(!/^$/, @scr);
print AVAIL @avail;
print STATUS @avail;
print AVAIL "\n";
print STATUS "Status: install ok unpacked\n";
unlink ("./${dpkgdir}info/$mscript");
} }
} }
system ("apt-get $config_str update"); close (AVAIL);
if ( -f "./${dpkgdir}info/$pkg.conffiles")
{
print STATUS "Conffiles:\n";
print " -> Processing conffiles for $pkg\n";
open (CONF, "./${dpkgdir}info/$pkg.conffiles");
my @lines=<CONF>;
close (CONF);
foreach my $line (@lines)
{
chomp ($line);
my $md5=`md5sum ./$line | cut -d" " -f1`;
chomp ($md5);
print STATUS " $line $md5\n";
}
}
print STATUS "\n";
close (STATUS);
system ("rm -rf ./${tmpdir}");
}
chdir ("$old");
print "I: Unpacking complete.\n";
}
sub tidy_apt
{
print "I: Tidying up apt cache and list data.\n";
opendir (DEBS, "${dir}${libdir}lists/")
or die ("Cannot read apt lists directory.\n");
my @lists=grep(!m:\.\.?$:, readdir DEBS);
closedir (DEBS);
foreach my $file (@lists)
{
next if (-d $file);
unlink ("${dir}${libdir}lists/$file");
}
opendir (DEBS, "${dir}${cachedir}/")
or die ("Cannot read apt cache directory/.\n");
my @files=grep(!m:\.\.?$:, readdir DEBS);
closedir (DEBS);
foreach my $file (@files)
{
next if (-d $file);
next unless ($file =~ /\.bin$/);
unlink ("${dir}${cachedir}$file");
}
if ($forceunpack eq "true")
{
opendir (DEBS, "${dir}${cachedir}/archives/")
or die ("Cannot read apt archives directory/.\n");
my @files=grep(!m:\.\.?$:, readdir DEBS);
closedir (DEBS);
foreach my $file (@files)
{
next if (-d $file);
next unless ($file =~ /\.deb$/);
unlink ("${dir}${cachedir}archives/$file");
}
}
}
sub get_required_debs
{
# emulate required="$(get_debs Priority: required)"
# from debootstrap/functions
# needs to be run after the first apt-get install so that
# Packages files exist
my @required=();
my @debs=();
opendir (PKGS, "${dir}${libdir}lists/")
or die ("Cannot open ${dir}${libdir}lists/ directory. $!\n");
my @lists=grep(/_Packages$/, readdir (PKGS));
closedir (PKGS);
# only read Packages files from the debootstrap entries where
# options do not include --no-resolve-deps
foreach my $strap (@debootstrap)
{
next if ($options{$strap} =~ /no-resolve-deps/);
my $s = lc($strap);
foreach my $l (@lists)
{
next unless ($l =~ /$s/);
push (@required, $l);
}
}
foreach my $file (@required)
{
my $fh = IO::File->new("${dir}${libdir}lists/$file");
my $parser = Parse::Debian::Packages->new( $fh );
while (my %package = $parser->next)
{
next unless $package{'Priority'} eq "required";
push @debs, $package{'Package'};
}
}
return \@debs;
}
sub write_script sub write_script
{ {
@ -316,6 +517,11 @@ END
em_multistrap - extends debootstrap for multiple repository support em_multistrap - extends debootstrap for multiple repository support
=head1 Synopsis
em_multistrap [-a ARCH] [-d DIR] -f CONFIG_FILE
em_multistrap -?|-h|--help|--version
=head1 Description =head1 Description
em_multistrap extends debootstrap to provide support for multiple em_multistrap extends debootstrap to provide support for multiple
@ -327,8 +533,11 @@ Example configuration:
[General] [General]
arch=arm arch=arm
directory=/opt/emdebian/debootstrap/multistrap/ directory=/opt/multistrap/
retries=5 retries=5
# extract all downloaded archives as well as those
# unpacked by debootstrap.
forceunpack=false
# the order of sections is important. # the order of sections is important.
# debootstraps are unpacked in this sequence. # debootstraps are unpacked in this sequence.
debootstrap=Debian debootstrap=Debian
@ -341,7 +550,9 @@ Example configuration:
options= options=
This will result in a completely normal debootstrap of Debian lenny from This will result in a completely normal debootstrap of Debian lenny from
the specified mirror using packages for ARM. the specified mirror, for ARM in /opt/multistrap/.
'Architecture' and 'directory' can be overridden on the command line.
Section names are case-insensitive. Section names are case-insensitive.
@ -356,7 +567,7 @@ contains packages that are also in any of the other repositories but at
a lower version. This prevents debootstrap unpacking the older version a lower version. This prevents debootstrap unpacking the older version
of the duplicated package. of the duplicated package.
General settings: =head1 General settings:
The order of repositories specified in the debootstrap (under General), The order of repositories specified in the debootstrap (under General),
determines which repository is unpacked first - this is important if one determines which repository is unpacked first - this is important if one
@ -383,4 +594,15 @@ em_multistrap does not currently implement machine:variant support
but the build directory is not packed up at the end of the run so but the build directory is not packed up at the end of the run so
other scripts can be used to implement customisations. other scripts can be used to implement customisations.
=head1 Native mode disabled
em_multistrap is not intended for native support, it was developed for
cross architeeture support. In order for multiple repositories to be
used, em_multistrap sets the --foreign option with debootstrap even if
debootstrap would be able to complete the installation of each single
repository. The reason for this is so that only the packages selected
by apt are actually unpacked.
Currently, em_multistrap disables native support.
=cut =cut