josch/mmdebstrap
2
3
Fork 6
Code Issues 8 Pull requests 1 Projects Releases Wiki Activity

add support for generating squashfs images using tar2sqfs

Browse source
This commit is contained in:
Johannes 'josch' Schauer 2020-01-07 17:40:13 +01:00
parent 51fab612ed
commit dbdf3f34c6
Signed by: josch
GPG key ID: F2CBA5C78FBD83E1
3 changed files with 96 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
coverage.sh
View file

@ -49,7 +49,7 @@ if [ ! -e shared/mmdebstrap ] || [ mmdebstrap -nt shared/mmdebstrap ]; then
fi fi
starttime= starttime=
total=122 total=123
skipped=0 skipped=0
runtests=0 runtests=0
i=1 i=1
@ -547,6 +547,29 @@ else
runtests=$((runtests+1)) runtests=$((runtests+1))
fi fi
print_header "mode=$defaultmode,variant=apt: test squashfs image"
cat << END > shared/test.sh
#!/bin/sh
set -eu
export LC_ALL=C.UTF-8
$CMD --mode=$defaultmode --variant=apt $DEFAULT_DIST /tmp/debian-chroot.squashfs $mirror
printf 'hsqs' | cmp --bytes=4 /tmp/debian-chroot.squashfs -
# workaround for https://github.com/AgentD/squashfs-tools-ng/issues/37
sed 's#\\([^.]\\)/\$#\\1#' tar1.txt | sort > /tmp/tar1noslash.txt
sqfs2tar --no-skip --root-becomes . /tmp/debian-chroot.squashfs | tar -t | sort | diff -u /tmp/tar1noslash.txt -
rm /tmp/debian-chroot.squashfs /tmp/tar1noslash.txt
END
if [ "$HAVE_QEMU" = "yes" ]; then
./run_qemu.sh
runtests=$((runtests+1))
elif [ "$defaultmode" = "root" ]; then
./run_null.sh SUDO
runtests=$((runtests+1))
else
./run_null.sh
runtests=$((runtests+1))
fi
print_header "mode=auto,variant=apt: test auto-mode without unshare capabilities" print_header "mode=auto,variant=apt: test auto-mode without unshare capabilities"
cat << END > shared/test.sh cat << END > shared/test.sh
#!/bin/sh #!/bin/sh

2
make_mirror.sh
View file

@ -407,7 +407,7 @@ if [ "$HAVE_QEMU" = "yes" ]; then
tmpdir="$(mktemp -d)" tmpdir="$(mktemp -d)"
trap "cleanuptmpdir; cleanup_newcachedir" EXIT INT TERM trap "cleanuptmpdir; cleanup_newcachedir" EXIT INT TERM
pkgs=perl-doc,linux-image-amd64,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,proot,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,debootstrap,procps,apt-cudf,aspcud pkgs=perl-doc,linux-image-amd64,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,proot,qemu-user-static,binfmt-support,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,debootstrap,procps,apt-cudf,aspcud,squashfs-tools-ng
if [ "$HOSTARCH" = amd64 ] && [ "$RUN_MA_SAME_TESTS" = "yes" ]; then if [ "$HOSTARCH" = amd64 ] && [ "$RUN_MA_SAME_TESTS" = "yes" ]; then
arches=amd64,armhf arches=amd64,armhf
pkgs="$pkgs,libfakechroot:armhf,libfakeroot:armhf" pkgs="$pkgs,libfakechroot:armhf,libfakeroot:armhf"

104
mmdebstrap
View file

@ -2707,16 +2707,9 @@ sub main() {
# figure out whether a tarball has to be created in the end # figure out whether a tarball has to be created in the end
$options->{maketar} = 0; $options->{maketar} = 0;
$options->{makesqfs} = 0;
if (defined $tar_compressor or $options->{target} =~ /\.tar$/ or $options->{target} eq '-') { if (defined $tar_compressor or $options->{target} =~ /\.tar$/ or $options->{target} eq '-') {
$options->{maketar} = 1; $options->{maketar} = 1;
if (any { $_ eq $options->{variant} } ('extract', 'custom') and any { $_ eq $options->{mode} } ('fakechroot', 'proot')) {
info "creating a tarball in fakechroot mode or proot mode might fail in extract and custom variants because there might be no tar inside the chroot";
}
# try to fail early if target tarball cannot be opened for writing
if ($options->{target} ne '-') {
open my $fh, '>', $options->{target} or error "cannot open $options->{target} for writing: $!";
close $fh;
}
# check if the compressor is installed # check if the compressor is installed
if (defined $tar_compressor) { if (defined $tar_compressor) {
my $pid = fork() // error "fork() failed: $!"; my $pid = fork() // error "fork() failed: $!";
@ -2730,9 +2723,31 @@ sub main() {
error ("failed to start " . (join " ", @{$tar_compressor})); error ("failed to start " . (join " ", @{$tar_compressor}));
} }
} }
} elsif ($options->{target} =~ /\.(squashfs|sqfs)$/) {
$options->{makesqfs} = 1;
# check if tar2sqfs is installed
my $pid = fork() // error "fork() failed: $!";
if ($pid == 0) {
open(STDOUT, '>', '/dev/null') or error "cannot open /dev/null for writing: $!";
open(STDIN, '<', '/dev/null') or error "cannot open /dev/null for reading: $!";
exec ('tar2sqfs', '--version') or error ("cannot exec tar2sqfs --version: $!");
}
waitpid $pid, 0;
if ($? != 0) {
error ("failed to start tar2sqfs --version");
}
} }
if ($options->{maketar}) { if ($options->{maketar} or $options->{makesqfs}) {
if (any { $_ eq $options->{variant} } ('extract', 'custom') and any { $_ eq $options->{mode} } ('fakechroot', 'proot')) {
info "creating a tarball or squashfs image in fakechroot mode or proot mode might fail in extract and custom variants because there might be no tar inside the chroot";
}
# try to fail early if target tarball or squashfs image cannot be
# opened for writing
if ($options->{target} ne '-') {
open my $fh, '>', $options->{target} or error "cannot open $options->{target} for writing: $!";
close $fh;
}
# since the output is a tarball, we create the rootfs in a temporary # since the output is a tarball, we create the rootfs in a temporary
# directory # directory
$options->{root} = tempdir( $options->{root} = tempdir(
@ -2837,7 +2852,7 @@ sub main() {
my $devtar = ''; my $devtar = '';
# We always craft the /dev entries ourselves if a tarball is to be created # We always craft the /dev entries ourselves if a tarball is to be created
if ($options->{maketar}) { if ($options->{maketar} or $options->{makesqfs}) {
foreach my $file (@devfiles) { foreach my $file (@devfiles) {
my ($fname, $mode, $type, $linkname, $devmajor, $devminor) = @{$file}; my ($fname, $mode, $type, $linkname, $devmajor, $devminor) = @{$file};
my $entry = pack('a100 a8 a8 a8 a12 a12 A8 a1 a100 a8 a32 a32 a8 a8 a155 x12', my $entry = pack('a100 a8 a8 a8 a12 a12 A8 a1 a100 a8 a32 a32 a8 a8 a155 x12',
@ -2901,7 +2916,7 @@ sub main() {
close $childsock; close $childsock;
if ($options->{maketar}) { if ($options->{maketar} or $options->{makesqfs}) {
info "creating tarball..."; info "creating tarball...";
# redirect tar output to the writing end of the pipe so that the # redirect tar output to the writing end of the pipe so that the
@ -2943,7 +2958,7 @@ sub main() {
close $childsock; close $childsock;
if ($options->{maketar}) { if ($options->{maketar} or $options->{makesqfs}) {
info "creating tarball..."; info "creating tarball...";
# redirect tar output to the writing end of the pipe so that the # redirect tar output to the writing end of the pipe so that the
@ -3306,7 +3321,7 @@ sub main() {
close $parentsock; close $parentsock;
if ($options->{maketar}) { if ($options->{maketar} or $options->{makesqfs}) {
# we use eval() so that error() doesn't take this process down and # we use eval() so that error() doesn't take this process down and
# thus leaves the setup() process without a parent # thus leaves the setup() process without a parent
eval { eval {
@ -3315,7 +3330,17 @@ sub main() {
error "cannot copy to standard output: $!"; error "cannot copy to standard output: $!";
} }
} else { } else {
if (defined $tar_compressor) { if ($options->{makesqfs} or defined $tar_compressor) {
my @argv = ();
if ($options->{makesqfs}) {
push @argv, 'tar2sqfs',
'--quiet', '--no-skip', '--force', '--exportable',
'--compressor', 'xz',
'--block-size', '1048576',
$options->{target};
} else {
push @argv, @{$tar_compressor};
}
POSIX::sigprocmask(SIG_BLOCK, $sigset) or error "Can't block signals: $!"; POSIX::sigprocmask(SIG_BLOCK, $sigset) or error "Can't block signals: $!";
my $cpid = fork() // error "fork() failed: $!"; my $cpid = fork() // error "fork() failed: $!";
if ($cpid == 0) { if ($cpid == 0) {
@ -3327,14 +3352,19 @@ sub main() {
# unblock all delayed signals (and possibly handle them) # unblock all delayed signals (and possibly handle them)
POSIX::sigprocmask(SIG_UNBLOCK, $sigset) or error "Can't unblock signals: $!"; POSIX::sigprocmask(SIG_UNBLOCK, $sigset) or error "Can't unblock signals: $!";
open(STDOUT, '>', $options->{target}) or error "cannot open $options->{target} for writing: $!";
if ($options->{makesqfs}) {
open(STDOUT, '>', '/dev/null') or error "cannot open /dev/null for writing: $!";
} else {
open(STDOUT, '>', $options->{target}) or error "cannot open $options->{target} for writing: $!";
}
open(STDIN, '<&', $rfh) or error "cannot open file handle for reading: $!"; open(STDIN, '<&', $rfh) or error "cannot open file handle for reading: $!";
exec { $tar_compressor->[0] } @{$tar_compressor} or error ("cannot exec " . (join " ", @{$tar_compressor}) . ": $!"); exec { $argv[0] } @argv or error ("cannot exec " . (join " ", @argv) . ": $!");
} }
POSIX::sigprocmask(SIG_UNBLOCK, $sigset) or error "Can't unblock signals: $!"; POSIX::sigprocmask(SIG_UNBLOCK, $sigset) or error "Can't unblock signals: $!";
waitpid $cpid, 0; waitpid $cpid, 0;
if ($? != 0) { if ($? != 0) {
error ("failed to start " . (join " ", @{$tar_compressor})); error ("failed to start " . (join " ", @argv));
} }
} else { } else {
if(!copy($rfh, $options->{target})) { if(!copy($rfh, $options->{target})) {
@ -3359,7 +3389,7 @@ sub main() {
# change signal handler message # change signal handler message
$waiting_for = "cleanup"; $waiting_for = "cleanup";
if ($options->{maketar} and -e $options->{root}) { if (($options->{maketar} or $options->{makesqfs}) and -e $options->{root}) {
info "removing tempdir $options->{root}..."; info "removing tempdir $options->{root}...";
if ($options->{mode} eq 'unshare') { if ($options->{mode} eq 'unshare') {
# We don't have permissions to remove the directory outside # We don't have permissions to remove the directory outside
@ -3447,25 +3477,29 @@ installed inside the chroot. If any mirror contains a tor+xxx URI, then the
apt-transport-tor package will be installed inside the chroot. apt-transport-tor package will be installed inside the chroot.
The optional I<TARGET> argument can either be the path to a directory, the The optional I<TARGET> argument can either be the path to a directory, the
path to a tarball filename or C<->. If I<TARGET> ends with C<.tar>, or with path to a tarball filename, the path to a squashfs image or C<->. If I<TARGET>
any of the filename extensions listed in the section B<COMPRESSION>, then ends with C<.tar>, or with any of the filename extensions listed in the
I<TARGET> will be interpreted as a path to a tarball filename. If I<TARGET> is section B<COMPRESSION>, then I<TARGET> will be interpreted as a path to a
the path to a tarball filename or if I<TARGET> is C<-> or if no I<TARGET> was tarball filename. If I<TARGET> ends with C<.squashfs> or C<.sqfs>, then
specified, B<mmdebstrap> will create a temporary chroot directory in I<TARGET> will be interpreted as a path to a squashfs image. If I<TARGET> is
C<$TMPDIR> or F</tmp>. If I<TARGET> is the path to a tarball filename, the path to a tarball filename or a squashfs image or if I<TARGET> is C<-> or
B<mmdebstrap> will create a tarball of that directory and store it as if no I<TARGET> was specified, B<mmdebstrap> will create a temporary chroot
I<TARGET>, optionally applying a compression algorithm as indicated by its directory in C<$TMPDIR> or F</tmp>. If I<TARGET> is the path to a tarball
filename, B<mmdebstrap> will create a tarball of that directory and store it
as I<TARGET>, optionally applying a compression algorithm as indicated by its
filename extension. If I<TARGET> is C<-> or if no I<TARGET> was specified, filename extension. If I<TARGET> is C<-> or if no I<TARGET> was specified,
then an uncompressed tarball of that directory will be sent to standard then an uncompressed tarball of that directory will be sent to standard
output. When B<mmdebstrap> creates a tarball it also stores extended output. When B<mmdebstrap> creates a tarball it also stores extended
attributes. To preserve the extended attributes, you have to pass B<--xattrs attributes. To preserve the extended attributes, you have to pass B<--xattrs
--xattrs-include='*'> to tar when extracting the tarball. If I<TARGET> does --xattrs-include='*'> to tar when extracting the tarball. If I<TARGET> is the
not end in C<.tar> or with any of the filename extensions listed in the path to a squashfs image, B<mmdebstrap> will create an xz compressed image
section B<COMPRESSION>, then I<TARGET> will be interpreted as the path to a with a blocksize of 1048576 bytes. If I<TARGET> does neither end with C<.tar>
directory. If the directory already exists, it must either be empty or only nor with any of the filename extensions listed in the section B<COMPRESSION>,
contain an empty C<lost+found> directory. If a directory is chosen as output nor with C<.squashfs> or C<.sqfs>, then I<TARGET> will be interpreted as the
in any other mode than B<sudo>, then its contents will have wrong ownership path to a directory. If the directory already exists, it must either be empty
information and special device files will be missing. or only contain an empty C<lost+found> directory. If a directory is chosen as
output in any other mode than B<sudo>, then its contents will have wrong
ownership information and special device files will be missing.
The I<SUITE> may be a valid release code name (eg, sid, stretch, jessie) or a The I<SUITE> may be a valid release code name (eg, sid, stretch, jessie) or a
symbolic name (eg, unstable, testing, stable, oldstable). Any suite name that symbolic name (eg, unstable, testing, stable, oldstable). Any suite name that
@ -3938,6 +3972,10 @@ create device nodes:
$ mmdebstrap unstable | tar --delete ./dev > unstable-chroot.tar $ mmdebstrap unstable | tar --delete ./dev > unstable-chroot.tar
Instead of a tarball, a squashfs image can be created:
$ mmdebstrap unstable unstable-chroot.squashfs
By default, debootstrapping a stable distribution will add mirrors for security By default, debootstrapping a stable distribution will add mirrors for security
and updates to the sources.list. and updates to the sources.list.