From 05bad914614b5a6395303bef5f1128bc407c2035 Mon Sep 17 00:00:00 2001 From: codehelp Date: Sun, 8 Mar 2009 18:10:24 +0000 Subject: [PATCH] pbuilder/em_multistrap : add support for complete unpacking, ported from emsandbox, with options to remove the .debs and cache data. Disable broken native support. git-svn-id: http://emdebian.org/svn/current@5691 563faec7-e20c-0410-992a-a66f704d0ccd --- em_multistrap | 250 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 236 insertions(+), 14 deletions(-) diff --git a/em_multistrap b/em_multistrap index 3830cf5..49cae98 100755 --- a/em_multistrap +++ b/em_multistrap @@ -15,16 +15,19 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +use IO::File; use Config::Auto; use File::Basename; +use Parse::Debian::Packages; use strict; use vars qw/ $progname $ourversion %scripts $dstrap $script $extra @archives $deb $cachedir $config_str %packages $retval $str $retries $dir $include $arch $foreign $suite $url $forceunpack $option %options @e $sourcesname $libdir $dpkgdir @debootstrap %suites $mirror $etcdir $repo @dirs @touch %sources $section %keys $host $key $value $type - $file $config /; + $file $config $tidy /; $progname = basename($0); +$ourversion = "0.0.3"; while( @ARGV ) { $_= shift( @ARGV ); @@ -43,6 +46,13 @@ while( @ARGV ) { elsif (/^(-a|--arch)$/) { $arch = shift(@ARGV); } + elsif (/^(-d|--dir)$/) { + $dir = shift(@ARGV); + $dir .= ($dir =~ m:/$:) ? '' : "/"; + } + elsif (/^(--tidy-up)$/) { + $tidy++; + } else { die "$progname: Unknown option $_.\n"; } @@ -68,7 +78,7 @@ foreach $section (sort keys %keys) { $arch = $keys{$section}{'arch'} if (not defined $arch); $retries = $keys{$section}{'retries'}; - $dir = $keys{$section}{'directory'}; + $dir = $keys{$section}{'directory'} if (not defined $dir); $forceunpack = lc($keys{$section}{'forceunpack'}); @debootstrap = split(' ', lc($keys{$section}{'debootstrap'})); } @@ -81,10 +91,13 @@ foreach $section (sort keys %keys) $options{$section}=$keys{$section}{'options'}; } } - +print "$progname $ourversion using $file\n"; $host = `dpkg-architecture -qDEB_BUILD_ARCH`; chomp ($host); -$foreign = ($host ne $arch) ? "--foreign" : ''; +die ("$progname is not currently able to support native operation.\n") + if ($host eq $arch); +# don't let debootstrap do second-stage +$foreign = "--foreign"; # always set $cachedir = "var/cache/apt/"; # archives $libdir = "var/lib/apt/"; # lists $etcdir = "etc/apt/"; # sources @@ -157,6 +170,10 @@ $config_str .= " -o Dir::Cache=${dir}${cachedir}"; system ("apt-get $config_str update"); $str = join (' ', values %packages); chomp($str); +$str .= " "; +my $required = &get_required_debs; +$str .= join (' ', @$required); +chomp($str); print "apt-get -y $config_str install $str\n"; $retval = system ("apt-get -y $config_str install $str"); die ("apt download failed. Exit value: ".($retval/256)."\n") @@ -177,32 +194,216 @@ foreach $dstrap (@debootstrap) $include .= join (',', @e); } &write_script ($dstrap); - $str = "debootstrap $option $include --arch $arch $foreign $suite $dir "; - $str .= "$url $script"; + $str = "/usr/sbin/debootstrap $option $include --arch $arch $foreign"; + $str .= " --keep-debootstrap-dir $suite $dir $url $script"; print "$str\n"; $retval = system ($str); while ($retval != 0 and $retries > 0) { - print "Problem - trying again, $retries left.\n"; + print "Problem - trying again, $retries left. ".($retval/250)."\n$!\n"; sleep 1; $retval = system ("$str"); $retries--; } } -if ($forceunpack eq "true") + +&force_unpack if ($forceunpack eq "true"); +system ("touch ${dir}${libdir}lists/lock"); +(not defined $tidy) ? system ("apt-get $config_str update") : &tidy_apt; +system ("rm -rf ${dir}debootstrap"); +print "\nMultistrap system installed successfully in $dir.\n\n"; +exit 0; + +sub force_unpack { + my %unpack=(); + opendir (DEBS, "${dir}${cachedir}archives/") + or die ("Cannot read apt archives directory.\n"); + @archives=grep(/.*\.deb$/, readdir DEBS); + closedir (DEBS); + print "I: Calculating obsolete packages\n"; + foreach $deb (sort @archives) + { + my $version = `dpkg -f ${dir}${cachedir}archives/$deb Version`; + my $package = `dpkg -f ${dir}${cachedir}archives/$deb Package`; + chomp ($version); + chomp ($package); + if (exists $unpack{$package}) + { + my $test=system("dpkg --compare-versions $unpack{$package} '<<' $version"); + $test /= 256; + # unlink version in $unpack if 0 + # unlink $deb (current one) if 1 + if ($test == 0) + { + my $old = $deb; + $old =~ s/$version/$unpack{$package}/; + print "I: Removing $old\n"; + unlink "${dir}${cachedir}archives/$old"; + next; + } + else + { + print "I: Removing $deb\n"; + unlink "${dir}${cachedir}archives/$deb"; + } + } + $unpack{$package}=$version; + } + open (LOCK, ">${dir}${libdir}lists/lock"); + close (LOCK); opendir (DEBS, "${dir}${cachedir}archives/") or die ("Cannot read apt archives directory.\n"); @archives=grep(/.*\.deb$/, readdir DEBS); closedir (DEBS); + my $old = `pwd`; + chomp ($old); chdir ("${dir}"); - foreach $deb (@archives) + foreach $deb (sort @archives) { - print "Extracting $deb...\n"; + print "I: Extracting $deb...\n"; system ("ar -p \"./${cachedir}archives/$deb\" data.tar.gz | zcat | tar -xf -"); + my $ver=`dpkg -f ./${cachedir}archives/$deb Version`; + my $pkg=`dpkg -f ./${cachedir}archives/$deb Package`; + chomp ($ver); + chomp ($pkg); + mkdir ("./tmp"); + my $tmpdir = `mktemp -p ./tmp -d -t multistrap.XXXXXX`; + chomp ($tmpdir); + mkdir ("./${tmpdir}/listing"); + system ("ar -p \"./${cachedir}archives/$deb\" data.tar.gz > ./${tmpdir}/listing/data.tar.gz"); + my $datatar = `tar -tzf ./${tmpdir}/listing/data.tar.gz`; + my @lines = split("\n", $datatar); + open (LIST, ">>./${dpkgdir}info/${pkg}.list"); + foreach my $l (@lines) + { + chomp ($l); + $l =~ s:^\.::; + $l =~ s:^/$:/\.:; + $l =~ s:/$::; + print LIST "$l\n"; + } + close (LIST); + system ("rm -rf ./${tmpdir}/listing"); + system ("dpkg -e ./${cachedir}archives/$deb ${tmpdir}/"); + opendir (MAINT, "./${tmpdir}"); + my @maint=grep(!m:\.\.?:, readdir (MAINT)); + closedir (MAINT); + open (AVAIL, ">>./${dpkgdir}available"); + open (STATUS, ">>./${dpkgdir}status"); + foreach my $mscript (@maint) + { + rename "./${tmpdir}/$mscript", "./${dpkgdir}info/$pkg.$mscript"; + if ( $mscript eq "control" ) + { + open (MSCRIPT, "./${dpkgdir}info/$pkg.$mscript"); + my @scr=; + close (MSCRIPT); + my @avail = grep(!/^$/, @scr); + print AVAIL @avail; + print STATUS @avail; + print AVAIL "\n"; + print STATUS "Status: install ok unpacked\n"; + unlink ("./${dpkgdir}info/$mscript"); + } + } + close (AVAIL); + if ( -f "./${dpkgdir}info/$pkg.conffiles") + { + print STATUS "Conffiles:\n"; + print " -> Processing conffiles for $pkg\n"; + open (CONF, "./${dpkgdir}info/$pkg.conffiles"); + my @lines=; + close (CONF); + foreach my $line (@lines) + { + chomp ($line); + my $md5=`md5sum ./$line | cut -d" " -f1`; + chomp ($md5); + print STATUS " $line $md5\n"; + } + } + print STATUS "\n"; + close (STATUS); + system ("rm -rf ./${tmpdir}"); } + chdir ("$old"); + print "I: Unpacking complete.\n"; +} + +sub tidy_apt +{ + print "I: Tidying up apt cache and list data.\n"; + opendir (DEBS, "${dir}${libdir}lists/") + or die ("Cannot read apt lists directory.\n"); + my @lists=grep(!m:\.\.?$:, readdir DEBS); + closedir (DEBS); + foreach my $file (@lists) + { + next if (-d $file); + unlink ("${dir}${libdir}lists/$file"); + } + opendir (DEBS, "${dir}${cachedir}/") + or die ("Cannot read apt cache directory/.\n"); + my @files=grep(!m:\.\.?$:, readdir DEBS); + closedir (DEBS); + foreach my $file (@files) + { + next if (-d $file); + next unless ($file =~ /\.bin$/); + unlink ("${dir}${cachedir}$file"); + } + if ($forceunpack eq "true") + { + opendir (DEBS, "${dir}${cachedir}/archives/") + or die ("Cannot read apt archives directory/.\n"); + my @files=grep(!m:\.\.?$:, readdir DEBS); + closedir (DEBS); + foreach my $file (@files) + { + next if (-d $file); + next unless ($file =~ /\.deb$/); + unlink ("${dir}${cachedir}archives/$file"); + } + } +} + +sub get_required_debs +{ + # emulate required="$(get_debs Priority: required)" + # from debootstrap/functions + # needs to be run after the first apt-get install so that + # Packages files exist + my @required=(); + my @debs=(); + opendir (PKGS, "${dir}${libdir}lists/") + or die ("Cannot open ${dir}${libdir}lists/ directory. $!\n"); + my @lists=grep(/_Packages$/, readdir (PKGS)); + closedir (PKGS); + # only read Packages files from the debootstrap entries where + # options do not include --no-resolve-deps + foreach my $strap (@debootstrap) + { + next if ($options{$strap} =~ /no-resolve-deps/); + my $s = lc($strap); + foreach my $l (@lists) + { + next unless ($l =~ /$s/); + push (@required, $l); + } + } + foreach my $file (@required) + { + my $fh = IO::File->new("${dir}${libdir}lists/$file"); + my $parser = Parse::Debian::Packages->new( $fh ); + while (my %package = $parser->next) + { + next unless $package{'Priority'} eq "required"; + push @debs, $package{'Package'}; + } + } + return \@debs; } -system ("apt-get $config_str update"); sub write_script { @@ -316,6 +517,11 @@ END em_multistrap - extends debootstrap for multiple repository support +=head1 Synopsis + + em_multistrap [-a ARCH] [-d DIR] -f CONFIG_FILE + em_multistrap -?|-h|--help|--version + =head1 Description em_multistrap extends debootstrap to provide support for multiple @@ -327,8 +533,11 @@ Example configuration: [General] arch=arm - directory=/opt/emdebian/debootstrap/multistrap/ + directory=/opt/multistrap/ retries=5 + # extract all downloaded archives as well as those + # unpacked by debootstrap. + forceunpack=false # the order of sections is important. # debootstraps are unpacked in this sequence. debootstrap=Debian @@ -341,7 +550,9 @@ Example configuration: options= This will result in a completely normal debootstrap of Debian lenny from -the specified mirror using packages for ARM. +the specified mirror, for ARM in /opt/multistrap/. + +'Architecture' and 'directory' can be overridden on the command line. Section names are case-insensitive. @@ -356,7 +567,7 @@ contains packages that are also in any of the other repositories but at a lower version. This prevents debootstrap unpacking the older version of the duplicated package. -General settings: +=head1 General settings: The order of repositories specified in the debootstrap (under General), determines which repository is unpacked first - this is important if one @@ -383,4 +594,15 @@ em_multistrap does not currently implement machine:variant support but the build directory is not packed up at the end of the run so other scripts can be used to implement customisations. +=head1 Native mode disabled + +em_multistrap is not intended for native support, it was developed for +cross architeeture support. In order for multiple repositories to be +used, em_multistrap sets the --foreign option with debootstrap even if +debootstrap would be able to complete the installation of each single +repository. The reason for this is so that only the packages selected +by apt are actually unpacked. + +Currently, em_multistrap disables native support. + =cut