From 409686048b677be8ebc562e92c7db773986edab8 Mon Sep 17 00:00:00 2001 From: Johannes Schauer Marin Rodrigues Date: Sun, 12 May 2024 17:16:51 +0200 Subject: [PATCH] add --format=ext4 --- coverage.py | 2 +- coverage.txt | 4 +- make_mirror.sh | 4 +- mmdebstrap | 183 +++++++++++++++++++++++++++++++++++++---------- tests/mmdebstrap | 15 +++- 5 files changed, 162 insertions(+), 46 deletions(-) diff --git a/coverage.py b/coverage.py index 7e911cf..01ee7b5 100755 --- a/coverage.py +++ b/coverage.py @@ -34,7 +34,7 @@ all_variants = [ "standard", ] default_format = "auto" -all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "null"] +all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "ext4", "null"] mirror = os.getenv("mirror", "http://127.0.0.1/debian") hostarch = subprocess.check_output(["dpkg", "--print-architecture"]).decode().strip() diff --git a/coverage.txt b/coverage.txt index 2836908..dde5b33 100644 --- a/coverage.txt +++ b/coverage.txt @@ -59,7 +59,7 @@ Needs-QEMU: true Test: mmdebstrap Needs-Root: true Modes: root -Formats: tar squashfs ext2 +Formats: tar squashfs ext2 ext4 Variants: essential apt minbase buildd - standard Skip-If: variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558 @@ -68,7 +68,7 @@ Skip-If: Test: check-for-bit-by-bit-identical-format-output Modes: unshare fakechroot -Formats: tar squashfs ext2 +Formats: tar squashfs ext2 ext4 Variants: essential apt minbase buildd - standard Skip-If: variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558 diff --git a/make_mirror.sh b/make_mirror.sh index cb4dca1..3f8aae4 100755 --- a/make_mirror.sh +++ b/make_mirror.sh @@ -33,7 +33,7 @@ deletecache() { done # deleting artifacts from test "mmdebstrap" for variant in essential apt minbase buildd - standard; do - for format in tar ext2 squashfs; do + for format in tar ext2 ext4 squashfs; do if [ -e "$dir/mmdebstrap-$dist-$variant.$format" ]; then # attempt to delete for all dists because DEFAULT_DIST might've been different the last time rm "$dir/mmdebstrap-$dist-$variant.$format" @@ -453,7 +453,7 @@ if [ "$HAVE_QEMU" = "yes" ]; then tmpdir="$(mktemp -d)" trap 'kill "$PROXYPID" || :;cleanuptmpdir; cleanup_newcachedir' EXIT INT TERM - pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,squashfs-tools-ng,genext2fs,linux-image-generic,passwd + pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,squashfs-tools-ng,genext2fs,linux-image-generic,passwd,e2fsprogs,uuid-runtime if [ ! -e ./mmdebstrap ]; then pkgs="$pkgs,mmdebstrap" fi diff --git a/mmdebstrap b/mmdebstrap index 3217201..6049cd4 100755 --- a/mmdebstrap +++ b/mmdebstrap @@ -46,6 +46,7 @@ use Socket; use Time::HiRes; use Math::BigInt; use Text::ParseWords; +use Digest::SHA; use version; ## no critic (InputOutput::RequireBriefOpen) @@ -66,13 +67,16 @@ use version; *MS_BIND = \0x1000; *MS_REC = \0x4000; *MNT_DETACH = \2; +# uuid_t NameSpace_DNS in rfc4122 +*UUID_NS_DNS = \'6ba7b810-9dad-11d1-80b4-00c04fd430c8'; our ( $CLONE_NEWNS, $CLONE_NEWUTS, $CLONE_NEWIPC, $CLONE_NEWUSER, $CLONE_NEWPID, $CLONE_NEWNET, $_LINUX_CAPABILITY_VERSION_3, $CAP_SYS_ADMIN, $PR_CAPBSET_READ, $MS_BIND, - $MS_REC, $MNT_DETACH + $MS_REC, $MNT_DETACH, + $UUID_NS_DNS ); #<<< @@ -215,11 +219,12 @@ sub minor { } sub can_execute { - my $tool = shift; - my $pid = open my $fh, '-|' // return 0; + my $tool = shift; + my $verbose = shift // '--version'; + my $pid = open my $fh, '-|' // return 0; if ($pid == 0) { open(STDERR, '>&', STDOUT) or die; - exec {$tool} $tool, '--version' or die; + exec {$tool} $tool, $verbose or die; } chomp( my $content = do { local $/; <$fh> } @@ -303,6 +308,28 @@ sub shellescape { return "'$string'"; } +sub create_v5_uuid { + use bytes; + my $ns_uuid = shift; + my $name = shift; + my $version = 0x50; + # convert the namespace uuid to binary + $ns_uuid =~ tr/-//d; + $ns_uuid = pack 'H*', $ns_uuid; + # concatenate namespace and name and take sha1 + my $digest = Digest::SHA->new(1); + $digest->add($ns_uuid); + $digest->add($name); + # only the first 16 bytes matter + my $uuid = substr($digest->digest(), 0, 16); + # set the version + substr $uuid, 6, 1, chr(ord(substr($uuid, 6, 1)) & 0x0f | $version); + substr $uuid, 8, 1, chr(ord(substr $uuid, 8, 1) & 0x3f | 0x80); + # convert binary back to uuid formatting + return join '-', map { unpack 'H*', $_ } + map { substr $uuid, 0, $_, '' } (4, 2, 2, 2, 6); +} + sub test_unshare_userns { my $verbose = shift; @@ -4380,15 +4407,16 @@ sub guess_sources_format { } sub approx_disk_usage { - my $directory = shift; + my $directory = shift; + my $block_size = shift; info "approximating disk usage..."; # the "du" utility reports different results depending on the underlying # filesystem, see https://bugs.debian.org/650077 for a discussion # # we use code similar to the one used by dpkg-gencontrol instead # - # Regular files are measured in number of 1024 byte blocks. All other - # entries are assumed to take one block of space. + # Regular files are measured in number of $block_size byte blocks. All + # other entries are assumed to take one block of space. # # We ignore /dev because depending on the mode, the directory might be # populated or not and we want consistent disk usage results independent @@ -4412,8 +4440,8 @@ sub approx_disk_usage { return if exists $hardlink{"$dev:$ino"}; # Track hardlinks to avoid repeated additions. $hardlink{"$dev:$ino"} = 1 if $nlink > 1; - # add file size in 1024 byte blocks, rounded up - $installed_size += int(((-s _) + 1024) / 1024); + # add file size in $block_size byte blocks, rounded up + $installed_size += int(((-s _) + $block_size) / $block_size); } else { # all other entries are assumed to only take up one block $installed_size += 1; @@ -4805,7 +4833,7 @@ sub main() { $options->{format} = 'directory'; } my @valid_formats - = ('auto', 'directory', 'tar', 'squashfs', 'ext2', 'null'); + = ('auto', 'directory', 'tar', 'squashfs', 'ext2', 'ext4', 'null'); if (none { $_ eq $options->{format} } @valid_formats) { error "invalid format. Choose from " . (join ', ', @valid_formats); } @@ -5670,6 +5698,30 @@ sub main() { if ($exitstatus != 0) { error "genext2fs failed with exit status: $exitstatus"; } + } elsif ($options->{target} =~ /\.ext4$/) { + $options->{format} = 'ext4'; + # check if the installed version of e2fsprogs supports tarballs on + # stdin + (undef, my $filename) = tempfile( + "mmdebstrap.ext4.XXXXXXXXXXXX", + OPEN => 0, + TMPDIR => 1 + ); + # creating file to suppress message "Creating regular file ..." + { open my $fh, '>', $filename; } + open my $fh, '|-', 'mke2fs', '-q', '-F', '-o', 'Linux', '-T', + 'ext4', '-b', '4096', '-d', '-', $filename, + '16384' // error "failed to fork(): $!"; + # write 10240 null-bytes to mke2fs -- this represents an empty + # tar archive + print $fh ("\0" x 10240) + or error "cannot write to mke2fs process"; + close $fh; + my $exitstatus = $?; + unlink $filename // die "cannot unlink $filename"; + if ($exitstatus != 0) { + error "mke2fs failed with exit status: $exitstatus"; + } } else { $options->{format} = 'directory'; } @@ -5687,21 +5739,30 @@ sub main() { info "ignoring target $options->{target} with null format"; } + my $blocksize = -1; if ($options->{format} eq 'ext2') { if (!can_execute 'genext2fs') { error "need genext2fs for ext2 format"; } + $blocksize = 1024; + } elsif ($options->{format} eq 'ext4') { + if (!can_execute 'mke2fs', '-V') { + error "need mke2fs for ext4 format"; + } + $blocksize = 4096; } elsif ($options->{format} eq 'squashfs') { if (!can_execute 'tar2sqfs') { error "need tar2sqfs binary from the squashfs-tools-ng package"; } + $blocksize = 1048576; } - if (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2', 'null')) { + if (any { $_ eq $options->{format} } + ('tar', 'squashfs', 'ext2', 'ext4', 'null')) { if ($options->{format} ne 'null') { if (any { $_ eq $options->{variant} } ('extract', 'custom') and $options->{mode} eq 'fakechroot') { - info "creating a tarball or squashfs image or ext2 image in" + info "creating a tarball, squashfs, ext2 or ext4 image in" . " fakechroot mode might fail in extract and" . " custom variants because there might be no tar inside the" . " chroot"; @@ -5885,7 +5946,7 @@ sub main() { # If both the above assertion change, we can stop creating /dev entries as # well. my $devtar = ''; - if (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2')) { + if (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2', 'ext4')) { foreach my $file (@devfiles) { my ($fname, $mode, $type, $linkname, $devmajor, $devminor) = @{$file}; @@ -5954,6 +6015,9 @@ sub main() { push @taropts, '--xattrs', '--xattrs-exclude=system.*'; } elsif ($options->{format} eq "ext2") { warning "genext2fs does not support extended attributes"; + warning "ext2 does not support sub-second precision timestamps"; + warning "ext2 does not support timestamps beyond 2038 January 18"; + warning "ext2 inode size of 128 prevents removing these limitations"; } else { push @taropts, '--xattrs'; } @@ -6001,8 +6065,9 @@ sub main() { close $childsock; close $nblkreader; - if (!$options->{dryrun} && $options->{format} eq 'ext2') { - my $numblocks = approx_disk_usage($options->{root}); + if (!$options->{dryrun} && any { $_ eq $options->{format} } + ('ext2', 'ext4')) { + my $numblocks = approx_disk_usage($options->{root}, $blocksize); print $nblkwriter "$numblocks\n"; $nblkwriter->flush(); } @@ -6010,8 +6075,8 @@ sub main() { if ($options->{dryrun}) { info "simulate creating tarball..."; - } elsif (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2')) - { + } elsif (any { $_ eq $options->{format} } + ('tar', 'squashfs', 'ext2', 'ext4')) { info "creating tarball..."; # redirect tar output to the writing end of the pipe so @@ -6132,7 +6197,8 @@ sub main() { my $numblocks = 0; close $nblkwriter; - if (!$options->{dryrun} && $options->{format} eq 'ext2') { + if (!$options->{dryrun} && any { $_ eq $options->{format} } + ('ext2', 'ext4')) { $numblocks = <$nblkreader>; if (defined $numblocks) { chomp $numblocks; @@ -6151,9 +6217,11 @@ sub main() { # nothing to do } elsif (any { $_ eq $options->{format} } ('directory', 'null')) { # nothing to do - } elsif ($options->{format} eq 'ext2' && $numblocks <= 0) { + } elsif ((any { $_ eq $options->{format} } ('ext2', 'ext4')) + && $numblocks <= 0) { # nothing to do because of invalid $numblocks - } elsif (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2')) { + } elsif (any { $_ eq $options->{format} } + ('tar', 'squashfs', 'ext2', 'ext4')) { # we use eval() so that error() doesn't take this process down and # thus leaves the setup() process without a parent eval { @@ -6162,16 +6230,16 @@ sub main() { error "cannot copy to standard output: $!"; } } else { - if ( $options->{format} eq 'squashfs' - or $options->{format} eq 'ext2' - or defined $tar_compressor) { + if (any { $_ eq $options->{format} } + ('squashfs', 'ext2', 'ext4') + or defined $tar_compressor) { my @argv = (); if ($options->{format} eq 'squashfs') { push @argv, 'tar2sqfs', '--quiet', '--no-skip', '--force', '--exportable', '--compressor', 'xz', - '--block-size', '1048576', + '--block-size', $blocksize, $options->{target}; } elsif ($options->{format} eq 'ext2') { if ($numblocks <= 0) { @@ -6179,6 +6247,26 @@ sub main() { } push @argv, 'genext2fs', '-B', 1024, '-b', $numblocks, '-i', '16384', '-a', '-', $options->{target}; + } elsif ($options->{format} eq 'ext4') { + if ($numblocks <= 0) { + error "invalid number of blocks: $numblocks"; + } + push @argv, 'mke2fs', '-q', '-F', '-o', 'Linux', '-T', + 'ext4'; + if (exists $ENV{SOURCE_DATE_EPOCH}) { + # if SOURCE_DATE_EPOCH was set, make the image + # reproducible by setting a fixed uuid and + # hash_seed + my $uuid = create_v5_uuid( + create_v5_uuid( + $UUID_NS_DNS, "mister-muffin.de" + ), + $mtime + ); + push @argv, '-U', $uuid, '-E', "hash_seed=$uuid"; + } + push @argv, '-b', $blocksize, '-d', '-', + $options->{target}, $numblocks; } elsif ($options->{format} eq 'tar') { push @argv, @{$tar_compressor}; } else { @@ -6200,8 +6288,8 @@ sub main() { or error "Can't unblock signals: $!"; # redirect stdout to file or /dev/null - if ( $options->{format} eq 'squashfs' - or $options->{format} eq 'ext2') { + if (any { $_ eq $options->{format} } + ('squashfs', 'ext2', 'ext4')) { open(STDOUT, '>', '/dev/null') or error "cannot open /dev/null for writing: $!"; } elsif ($options->{format} eq 'tar') { @@ -6282,7 +6370,7 @@ sub main() { if (any { $_ eq $options->{format} } ('directory')) { # nothing to do } elsif (any { $_ eq $options->{format} } - ('tar', 'squashfs', 'ext2', 'null')) { + ('tar', 'squashfs', 'ext2', 'ext4', 'null')) { if (!-e $options->{root}) { error "$options->{root} does not exist"; } @@ -6381,12 +6469,12 @@ can be disabled by choosing the empty string for I. See the section B for more information. The I option may either be the path to a directory, the path to a -tarball filename, the path to a squashfs image, the path to an ext2 image, a -FIFO, a character special device, or C<->. The I option is optional if -no I option is provided. If I is missing or if I is -C<->, an uncompressed tarball will be sent to standard output. Without the -B<--format> option, I will be used to choose the format. See the -section B for more information. +tarball filename, the path to a squashfs image, the path to an ext2 or ext4 +image, a FIFO, a character special device, or C<->. The I option is +optional if no I option is provided. If I is missing or if +I is C<->, an uncompressed tarball will be sent to standard output. +Without the B<--format> option, I will be used to choose the format. +See the section B for more information. The I option may either be provided as a URI, in apt one-line format, as a path to a file in apt's one-line or deb822-format, or C<->. If no @@ -6463,8 +6551,8 @@ information. =item B<--format>=I Choose the output format. Valid format Is are B, B, -B, B, B and B. The default format is B. See -the section B for more information. +B, B, B, B and B. The default format is +B. See the section B for more information. =item B<--aptopt>=I