josch/mmdebstrap
2
3
Fork 6
Code Issues 8 Pull requests 1 Projects Releases Wiki Activity

add --format=ext4

Browse source
This commit is contained in:
Johannes Schauer Marin Rodrigues 2024-05-12 17:16:51 +02:00
parent 4a294f05bd
commit 409686048b
Signed by: josch
GPG key ID: F2CBA5C78FBD83E1
5 changed files with 162 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
coverage.py
View file

@ -34,7 +34,7 @@ all_variants = [
"standard",
]
default_format = "auto"
all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "null"]
all_formats = ["auto", "directory", "tar", "squashfs", "ext2", "ext4", "null"]
mirror = os.getenv("mirror", "http://127.0.0.1/debian")
hostarch = subprocess.check_output(["dpkg", "--print-architecture"]).decode().strip()

4
coverage.txt
View file

@ -59,7 +59,7 @@ Needs-QEMU: true
Test: mmdebstrap
Needs-Root: true
Modes: root
Formats: tar squashfs ext2
Formats: tar squashfs ext2 ext4
Variants: essential apt minbase buildd - standard
Skip-If:
variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558
@ -68,7 +68,7 @@ Skip-If:
Test: check-for-bit-by-bit-identical-format-output
Modes: unshare fakechroot
Formats: tar squashfs ext2
Formats: tar squashfs ext2 ext4
Variants: essential apt minbase buildd - standard
Skip-If:
variant == "standard" and dist == "oldstable" # #864082, #1004557, #1004558

4
make_mirror.sh
View file

@ -33,7 +33,7 @@ deletecache() {
done
# deleting artifacts from test "mmdebstrap"
for variant in essential apt minbase buildd - standard; do
for format in tar ext2 squashfs; do
for format in tar ext2 ext4 squashfs; do
if [ -e "$dir/mmdebstrap-$dist-$variant.$format" ]; then
# attempt to delete for all dists because DEFAULT_DIST might've been different the last time
rm "$dir/mmdebstrap-$dist-$variant.$format"
@ -453,7 +453,7 @@ if [ "$HAVE_QEMU" = "yes" ]; then
tmpdir="$(mktemp -d)"
trap 'kill "$PROXYPID" || :;cleanuptmpdir; cleanup_newcachedir' EXIT INT TERM
pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,squashfs-tools-ng,genext2fs,linux-image-generic,passwd
pkgs=perl-doc,systemd-sysv,perl,arch-test,fakechroot,fakeroot,mount,uidmap,qemu-user-static,qemu-user,dpkg-dev,mini-httpd,libdevel-cover-perl,libtemplate-perl,debootstrap,procps,apt-cudf,aspcud,python3,libcap2-bin,gpg,debootstrap,distro-info-data,iproute2,ubuntu-keyring,apt-utils,squashfs-tools-ng,genext2fs,linux-image-generic,passwd,e2fsprogs,uuid-runtime
if [ ! -e ./mmdebstrap ]; then
pkgs="$pkgs,mmdebstrap"
fi

183
mmdebstrap
View file

@ -46,6 +46,7 @@ use Socket;
use Time::HiRes;
use Math::BigInt;
use Text::ParseWords;
use Digest::SHA;
use version;
## no critic (InputOutput::RequireBriefOpen)
@ -66,13 +67,16 @@ use version;
*MS_BIND = \0x1000;
*MS_REC = \0x4000;
*MNT_DETACH = \2;
# uuid_t NameSpace_DNS in rfc4122
*UUID_NS_DNS = \'6ba7b810-9dad-11d1-80b4-00c04fd430c8';
our (
$CLONE_NEWNS, $CLONE_NEWUTS,
$CLONE_NEWIPC, $CLONE_NEWUSER,
$CLONE_NEWPID, $CLONE_NEWNET,
$_LINUX_CAPABILITY_VERSION_3, $CAP_SYS_ADMIN,
$PR_CAPBSET_READ, $MS_BIND,
$MS_REC, $MNT_DETACH
$MS_REC, $MNT_DETACH,
$UUID_NS_DNS
);
#<<<
@ -215,11 +219,12 @@ sub minor {
}
sub can_execute {
my $tool = shift;
my $pid = open my $fh, '-|' // return 0;
my $tool = shift;
my $verbose = shift // '--version';
my $pid = open my $fh, '-|' // return 0;
if ($pid == 0) {
open(STDERR, '>&', STDOUT) or die;
exec {$tool} $tool, '--version' or die;
exec {$tool} $tool, $verbose or die;
}
chomp(
my $content = do { local $/; <$fh> }
@ -303,6 +308,28 @@ sub shellescape {
return "'$string'";
}
sub create_v5_uuid {
use bytes;
my $ns_uuid = shift;
my $name = shift;
my $version = 0x50;
# convert the namespace uuid to binary
$ns_uuid =~ tr/-//d;
$ns_uuid = pack 'H*', $ns_uuid;
# concatenate namespace and name and take sha1
my $digest = Digest::SHA->new(1);
$digest->add($ns_uuid);
$digest->add($name);
# only the first 16 bytes matter
my $uuid = substr($digest->digest(), 0, 16);
# set the version
substr $uuid, 6, 1, chr(ord(substr($uuid, 6, 1)) & 0x0f | $version);
substr $uuid, 8, 1, chr(ord(substr $uuid, 8, 1) & 0x3f | 0x80);
# convert binary back to uuid formatting
return join '-', map { unpack 'H*', $_ }
map { substr $uuid, 0, $_, '' } (4, 2, 2, 2, 6);
}
sub test_unshare_userns {
my $verbose = shift;
@ -4380,15 +4407,16 @@ sub guess_sources_format {
}
sub approx_disk_usage {
my $directory = shift;
my $directory = shift;
my $block_size = shift;
info "approximating disk usage...";
# the "du" utility reports different results depending on the underlying
# filesystem, see https://bugs.debian.org/650077 for a discussion
#
# we use code similar to the one used by dpkg-gencontrol instead
#
# Regular files are measured in number of 1024 byte blocks. All other
# entries are assumed to take one block of space.
# Regular files are measured in number of $block_size byte blocks. All
# other entries are assumed to take one block of space.
#
# We ignore /dev because depending on the mode, the directory might be
# populated or not and we want consistent disk usage results independent
@ -4412,8 +4440,8 @@ sub approx_disk_usage {
return if exists $hardlink{"$dev:$ino"};
# Track hardlinks to avoid repeated additions.
$hardlink{"$dev:$ino"} = 1 if $nlink > 1;
# add file size in 1024 byte blocks, rounded up
$installed_size += int(((-s _) + 1024) / 1024);
# add file size in $block_size byte blocks, rounded up
$installed_size += int(((-s _) + $block_size) / $block_size);
} else {
# all other entries are assumed to only take up one block
$installed_size += 1;
@ -4805,7 +4833,7 @@ sub main() {
$options->{format} = 'directory';
}
my @valid_formats
= ('auto', 'directory', 'tar', 'squashfs', 'ext2', 'null');
= ('auto', 'directory', 'tar', 'squashfs', 'ext2', 'ext4', 'null');
if (none { $_ eq $options->{format} } @valid_formats) {
error "invalid format. Choose from " . (join ', ', @valid_formats);
}
@ -5670,6 +5698,30 @@ sub main() {
if ($exitstatus != 0) {
error "genext2fs failed with exit status: $exitstatus";
}
} elsif ($options->{target} =~ /\.ext4$/) {
$options->{format} = 'ext4';
# check if the installed version of e2fsprogs supports tarballs on
# stdin
(undef, my $filename) = tempfile(
"mmdebstrap.ext4.XXXXXXXXXXXX",
OPEN => 0,
TMPDIR => 1
);
# creating file to suppress message "Creating regular file ..."
{ open my $fh, '>', $filename; }
open my $fh, '|-', 'mke2fs', '-q', '-F', '-o', 'Linux', '-T',
'ext4', '-b', '4096', '-d', '-', $filename,
'16384' // error "failed to fork(): $!";
# write 10240 null-bytes to mke2fs -- this represents an empty
# tar archive
print $fh ("\0" x 10240)
or error "cannot write to mke2fs process";
close $fh;
my $exitstatus = $?;
unlink $filename // die "cannot unlink $filename";
if ($exitstatus != 0) {
error "mke2fs failed with exit status: $exitstatus";
}
} else {
$options->{format} = 'directory';
}
@ -5687,21 +5739,30 @@ sub main() {
info "ignoring target $options->{target} with null format";
}
my $blocksize = -1;
if ($options->{format} eq 'ext2') {
if (!can_execute 'genext2fs') {
error "need genext2fs for ext2 format";
}
$blocksize = 1024;
} elsif ($options->{format} eq 'ext4') {
if (!can_execute 'mke2fs', '-V') {
error "need mke2fs for ext4 format";
}
$blocksize = 4096;
} elsif ($options->{format} eq 'squashfs') {
if (!can_execute 'tar2sqfs') {
error "need tar2sqfs binary from the squashfs-tools-ng package";
}
$blocksize = 1048576;
}
if (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2', 'null')) {
if (any { $_ eq $options->{format} }
('tar', 'squashfs', 'ext2', 'ext4', 'null')) {
if ($options->{format} ne 'null') {
if (any { $_ eq $options->{variant} } ('extract', 'custom')
and $options->{mode} eq 'fakechroot') {
info "creating a tarball or squashfs image or ext2 image in"
info "creating a tarball, squashfs, ext2 or ext4 image in"
. " fakechroot mode might fail in extract and"
. " custom variants because there might be no tar inside the"
. " chroot";
@ -5885,7 +5946,7 @@ sub main() {
# If both the above assertion change, we can stop creating /dev entries as
# well.
my $devtar = '';
if (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2')) {
if (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2', 'ext4')) {
foreach my $file (@devfiles) {
my ($fname, $mode, $type, $linkname, $devmajor, $devminor)
= @{$file};
@ -5954,6 +6015,9 @@ sub main() {
push @taropts, '--xattrs', '--xattrs-exclude=system.*';
} elsif ($options->{format} eq "ext2") {
warning "genext2fs does not support extended attributes";
warning "ext2 does not support sub-second precision timestamps";
warning "ext2 does not support timestamps beyond 2038 January 18";
warning "ext2 inode size of 128 prevents removing these limitations";
} else {
push @taropts, '--xattrs';
}
@ -6001,8 +6065,9 @@ sub main() {
close $childsock;
close $nblkreader;
if (!$options->{dryrun} && $options->{format} eq 'ext2') {
my $numblocks = approx_disk_usage($options->{root});
if (!$options->{dryrun} && any { $_ eq $options->{format} }
('ext2', 'ext4')) {
my $numblocks = approx_disk_usage($options->{root}, $blocksize);
print $nblkwriter "$numblocks\n";
$nblkwriter->flush();
}
@ -6010,8 +6075,8 @@ sub main() {
if ($options->{dryrun}) {
info "simulate creating tarball...";
} elsif (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2'))
{
} elsif (any { $_ eq $options->{format} }
('tar', 'squashfs', 'ext2', 'ext4')) {
info "creating tarball...";
# redirect tar output to the writing end of the pipe so
@ -6132,7 +6197,8 @@ sub main() {
my $numblocks = 0;
close $nblkwriter;
if (!$options->{dryrun} && $options->{format} eq 'ext2') {
if (!$options->{dryrun} && any { $_ eq $options->{format} }
('ext2', 'ext4')) {
$numblocks = <$nblkreader>;
if (defined $numblocks) {
chomp $numblocks;
@ -6151,9 +6217,11 @@ sub main() {
# nothing to do
} elsif (any { $_ eq $options->{format} } ('directory', 'null')) {
# nothing to do
} elsif ($options->{format} eq 'ext2' && $numblocks <= 0) {
} elsif ((any { $_ eq $options->{format} } ('ext2', 'ext4'))
&& $numblocks <= 0) {
# nothing to do because of invalid $numblocks
} elsif (any { $_ eq $options->{format} } ('tar', 'squashfs', 'ext2')) {
} elsif (any { $_ eq $options->{format} }
('tar', 'squashfs', 'ext2', 'ext4')) {
# we use eval() so that error() doesn't take this process down and
# thus leaves the setup() process without a parent
eval {
@ -6162,16 +6230,16 @@ sub main() {
error "cannot copy to standard output: $!";
}
} else {
if ( $options->{format} eq 'squashfs'
or $options->{format} eq 'ext2'
or defined $tar_compressor) {
if (any { $_ eq $options->{format} }
('squashfs', 'ext2', 'ext4')
or defined $tar_compressor) {
my @argv = ();
if ($options->{format} eq 'squashfs') {
push @argv, 'tar2sqfs',
'--quiet', '--no-skip', '--force',
'--exportable',
'--compressor', 'xz',
'--block-size', '1048576',
'--block-size', $blocksize,
$options->{target};
} elsif ($options->{format} eq 'ext2') {
if ($numblocks <= 0) {
@ -6179,6 +6247,26 @@ sub main() {
}
push @argv, 'genext2fs', '-B', 1024, '-b', $numblocks,
'-i', '16384', '-a', '-', $options->{target};
} elsif ($options->{format} eq 'ext4') {
if ($numblocks <= 0) {
error "invalid number of blocks: $numblocks";
}
push @argv, 'mke2fs', '-q', '-F', '-o', 'Linux', '-T',
'ext4';
if (exists $ENV{SOURCE_DATE_EPOCH}) {
# if SOURCE_DATE_EPOCH was set, make the image
# reproducible by setting a fixed uuid and
# hash_seed
my $uuid = create_v5_uuid(
create_v5_uuid(
$UUID_NS_DNS, "mister-muffin.de"
),
$mtime
);
push @argv, '-U', $uuid, '-E', "hash_seed=$uuid";
}
push @argv, '-b', $blocksize, '-d', '-',
$options->{target}, $numblocks;
} elsif ($options->{format} eq 'tar') {
push @argv, @{$tar_compressor};
} else {
@ -6200,8 +6288,8 @@ sub main() {
or error "Can't unblock signals: $!";
# redirect stdout to file or /dev/null
if ( $options->{format} eq 'squashfs'
or $options->{format} eq 'ext2') {
if (any { $_ eq $options->{format} }
('squashfs', 'ext2', 'ext4')) {
open(STDOUT, '>', '/dev/null')
or error "cannot open /dev/null for writing: $!";
} elsif ($options->{format} eq 'tar') {
@ -6282,7 +6370,7 @@ sub main() {
if (any { $_ eq $options->{format} } ('directory')) {
# nothing to do
} elsif (any { $_ eq $options->{format} }
('tar', 'squashfs', 'ext2', 'null')) {
('tar', 'squashfs', 'ext2', 'ext4', 'null')) {
if (!-e $options->{root}) {
error "$options->{root} does not exist";
}
@ -6381,12 +6469,12 @@ can be disabled by choosing the empty string for I<SUITE>. See the section
B<VARIANTS> for more information.
The I<TARGET> option may either be the path to a directory, the path to a
tarball filename, the path to a squashfs image, the path to an ext2 image, a
FIFO, a character special device, or C<->. The I<TARGET> option is optional if
no I<MIRROR> option is provided. If I<TARGET> is missing or if I<TARGET> is
C<->, an uncompressed tarball will be sent to standard output. Without the
B<--format> option, I<TARGET> will be used to choose the format. See the
section B<FORMATS> for more information.
tarball filename, the path to a squashfs image, the path to an ext2 or ext4
image, a FIFO, a character special device, or C<->. The I<TARGET> option is
optional if no I<MIRROR> option is provided. If I<TARGET> is missing or if
I<TARGET> is C<->, an uncompressed tarball will be sent to standard output.
Without the B<--format> option, I<TARGET> will be used to choose the format.
See the section B<FORMATS> for more information.
The I<MIRROR> option may either be provided as a URI, in apt one-line format,
as a path to a file in apt's one-line or deb822-format, or C<->. If no
@ -6463,8 +6551,8 @@ information.
=item B<--format>=I<name>
Choose the output format. Valid format I<name>s are B<auto>, B<directory>,
B<tar>, B<squashfs>, B<ext2> and B<null>. The default format is B<auto>. See
the section B<FORMATS> for more information.
B<tar>, B<squashfs>, B<ext2>, B<ext4> and B<null>. The default format is
B<auto>. See the section B<FORMATS> for more information.
=item B<--aptopt>=I<option>|I<file>
@ -7043,6 +7131,7 @@ I<TARGET> equals C<->, or if I<TARGET> is a named pipe (fifo) or if I<TARGET>
is a character special file, then the B<tar> format will be chosen. If
I<TARGET> ends with C<.squashfs> or C<.sqfs>, then the B<squashfs> format will
be chosen. If I<TARGET> ends with C<.ext2> then the B<ext2> format will be
chosen. If I<TARGET> ends with C<.ext4> then the B<ext4> format will be
chosen. If none of these conditions apply, the B<directory> format will be
chosen.
@ -7092,8 +7181,24 @@ with this format because C<genext2fs> can only write to a regular file. If you
need your ext2 image be named C<->, then just explicitly pass the relative path
to it like F<./->. To convert the result to an ext3 image, use C<tune2fs -O
has_journal TARGET> and to convert it to ext4, use C<tune2fs -O
extents,uninit_bg,dir_index,has_journal TARGET>. Since C<genext2fs> does not
support extended attributes, the resulting image will not contain them.
extents,uninit_bg,dir_index,has_journal TARGET>.
B<CAUTION>: the ext2 format does not support timestamps beyond 2038 January 19,
does not support sub-second precision timestamps and does not support extended
attributes. Its inode size of 128 prevents adding these features with tune2fs
later on.
=item B<ext4>
A temporary chroot directory will be created in C<$TMPDIR> or F</tmp> if
C<$TMPDIR> is not set. A tarball of that directory will be piped to the
C<mke2fs> utility, which will create an ext4 image that will be approximately
90% full in I<TARGET>. The special I<TARGET> C<-> does not work with this
format because C<mke2fs> can only write to a regular file. If you need your
ext4 image be named C<->, then just explicitly pass the relative path to it
like F<./->. If C<SOURCE_DATE_EPOCH> is set, the filesystem UUID and hash_seed
will be set to a UUID derived from SOURCE_DATE_EPOCH to create reproducible
images.
=item B<null>
@ -7402,8 +7507,8 @@ Performs cleanup tasks, unless B<--skip=cleanup> is used:
=item B<output>
For formats other than B<directory>, pack up the temporary chroot directory
into a tarball, ext2 image or squashfs image and delete the temporary chroot
directory.
into a tarball, ext2 image, ext4 image or squashfs image and delete the
temporary chroot directory.
If B<--skip=output/dev> is added, the resulting chroot will not contain the
device nodes, directories and symlinks that B<debootstrap> creates but just

15
tests/mmdebstrap
View file

@ -5,15 +5,26 @@ export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
[ "$(id -u)" -eq 0 ]
[ {{ MODE }} = "root" ]
case {{ FORMAT }} in tar|squashfs|ext2) : ;; *) exit 1;; esac
case {{ FORMAT }} in tar|squashfs|ext2|ext4) : ;; *) exit 1;; esac
{{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} {{ MIRROR }}
{{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} /tmp/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} {{ MIRROR }}
# creating an ext4 image on a 9p filesystem produces different results compared
# to creating it on a tmpfs or ext4 fs because 9p does not support discards and
# even when running with -E nodiscard, the number of written bytes will differ
# https://lore.kernel.org/linux-ext4/171484520952.2626447.2160419274451668597@localhost/T/#t
mv /tmp/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }}
if [ "{{ FORMAT }}" = tar ]; then
printf 'ustar ' | cmp --bytes=6 --ignore-initial=257:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar -
elif [ "{{ FORMAT }}" = squashfs ]; then
printf 'hsqs' | cmp --bytes=4 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.squashfs -
elif [ "{{ FORMAT }}" = ext2 ]; then
printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext2 -
printf '\000\000\000\000\000\000\000\000\000\000\000\000' | cmp --bytes=12 --ignore-initial=1116:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext2 -
elif [ "{{ FORMAT }}" = ext4 ]; then
printf '\123\357' | cmp --bytes=2 --ignore-initial=1080:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext4 -
printf '\074\020\000\000\302\042\000\000\153\004\000\000' | cmp --bytes=12 --ignore-initial=1116:0 ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext4 -
[ "$(/sbin/blkid --match-tag UUID --output value ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.ext4)" = "$(uuidgen --sha1 --namespace="$(uuidgen --sha1 --namespace='@dns' --name mister-muffin.de)" --name $SOURCE_DATE_EPOCH)" ]
else
echo "unknown format: {{ FORMAT }}" >&2
exit 1