README.md: add Gioele Barabucci

The generated image will not be bootable if `mbr.bin` is not installed
into the MBR.

These lines are copied from the "Debian desktop on USB stick" example.

CHANGELOG.md

@@ -1,3 +1,10 @@
+0.8.4 (2022-02-11)
+------------------
+
+ - tarfilter: add --strip-components option
+ - don't install essential packages in run_install()
+ - remove /var/lib/dbus/machine-id
+
 0.8.3 (2022-01-08)
 ------------------
 

README.md

@@ -154,3 +154,4 @@ Contributors
  - Konstantin Demin
  - Trent W. Buck
  - Vagrant Cascadian
+ - Gioele Barabucci

coverage.sh

@@ -180,6 +180,7 @@ export SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH
 # compared to the one chosen in debootstrap because of different installation
 # order in comparison to the systemd users
 # https://bugs.debian.org/969631
+# we cannot use useradd because passwd is not Essential:yes
 $CMD --variant=$variant --mode=$defaultmode \
 	--essential-hook='if [ $variant = - ]; then echo _apt:*:100:65534::/nonexistent:/usr/sbin/nologin >> "\$1"/etc/passwd; fi' \
 	$dist /tmp/debian-$dist-mm.tar $mirror
@@ -321,6 +322,18 @@ else
 	echo no difference for /etc/shadow- on $dist $variant >&2
 fi
 
+# Because of unreproducible uids (#969631) we created the _apt user ourselves
+# and because passwd is not Essential:yes we didn't use useradd. But passwd
+# since 1:4.11.1+dfsg1-1 will create empty mail files, so we create it too.
+# https://bugs.debian.org/1004710
+if [ $variant = - ]; then
+	if [ -e /tmp/debian-$dist-debootstrap/var/mail/_apt ]; then
+		touch /tmp/debian-$dist-mm/var/mail/_apt
+		chmod 660 /tmp/debian-$dist-mm/var/mail/_apt
+		chown 100:8 /tmp/debian-$dist-mm/var/mail/_apt
+	fi
+fi
+
 # check if the file content differs
 diff --unified --no-dereference --recursive /tmp/debian-$dist-debootstrap /tmp/debian-$dist-mm
 
@@ -742,9 +755,8 @@ fi
 for variant in essential apt minbase buildd important standard; do
 	for format in tar squashfs ext2; do
 		print_header "mode=root/unshare/fakechroot,variant=$variant: check for bit-by-bit identical $format output"
-		# fontconfig doesn't install reproducibly because differences
-		# in /var/cache/fontconfig/. See
-		# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=864082
+		# pyc files and man index.db are not reproducible
+		# See #1004557 and #1004558
 		if [ "$variant" = "standard" ]; then
 			echo "skipping test because of #864082" >&2
 			skipped=$((skipped+1))
@@ -846,6 +858,7 @@ cmp /tmp/debian-chroot.tar /tmp/debian-chroot-shiftedback.tar
 # manually adjust uid/gid and compare "tar -t" output
 tar --numeric-owner -tvf /tmp/debian-chroot.tar \
 	| sed 's# 100/0 # 100100/100000 #' \
+	| sed 's# 100/8 # 100100/100008 #' \
 	| sed 's# 0/0 # 100000/100000 #' \
 	| sed 's# 0/5 # 100000/100005 #' \
 	| sed 's# 0/8 # 100000/100008 #' \
@@ -2973,9 +2986,8 @@ fi
 # into /var/cache/apt/archives/partial
 for variant in extract custom essential apt minbase buildd important standard; do
 	print_header "mode=$defaultmode,variant=$variant: compare output with pre-seeded /var/cache/apt/archives"
-	# fontconfig doesn't install reproducibly because differences
-	# in /var/cache/fontconfig/. See
-	# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=864082
+	# pyc files and man index.db are not reproducible
+	# See #1004557 and #1004558
 	if [ "$variant" = "standard" ]; then
 		echo "skipping test because of #864082" >&2
 		skipped=$((skipped+1))

mmdebstrap

@@ -23,7 +23,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '0.8.3';
+our $VERSION = '0.8.4';
 
 use English;
 use Getopt::Long;
@@ -2722,13 +2722,18 @@ sub run_install() {
         any { $_ eq $options->{variant} }
         ('required', 'important', 'standard', 'buildd')
     ) {
+        # Many of the priority:required packages are also essential:yes. We
+        # make sure not to select those here to avoid useless "xxx is already
+        # the newest version" messages.
         my $priority;
         if (any { $_ eq $options->{variant} } ('required', 'buildd')) {
-            $priority = '?priority(required)';
+            $priority = '?and(?priority(required),?not(?essential))';
         } elsif ($options->{variant} eq 'important') {
-            $priority = '?or(?priority(required),?priority(important))';
+            $priority = '?and(?or(?priority(required),?priority(important)),'
+              . '?not(?essential))';
         } elsif ($options->{variant} eq 'standard') {
-            $priority = '?or(~prequired,~pimportant,~pstandard)';
+            $priority = '?and(?or(~prequired,~pimportant,~pstandard),'
+              . '?not(?essential))';
         }
         $pkgs_to_install{
             "?narrow("
@@ -2780,6 +2785,9 @@ sub run_install() {
             #
             #  - we can make use of file:// and copy://
             #
+            #  - we can use EDSP solvers without installing apt-utils or other
+            #    solvers inside the chroot
+            #
             # The DPkg::Install::Recursive::force=true workaround can be
             # dropped after this issue is fixed:
             # https://salsa.debian.org/apt-team/apt/-/merge_requests/189
@@ -2915,7 +2923,8 @@ sub run_cleanup() {
         foreach my $fname (
             '/var/log/dpkg.log',             '/var/log/apt/history.log',
             '/var/log/apt/term.log',         '/var/log/alternatives.log',
-            '/var/cache/ldconfig/aux-cache', '/var/log/apt/eipp.log.xz'
+            '/var/cache/ldconfig/aux-cache', '/var/log/apt/eipp.log.xz',
+            '/var/lib/dbus/machine-id'
         ) {
             my $path = "$options->{root}$fname";
             if (!-e $path) {
@@ -6288,11 +6297,7 @@ needs to be able to mount and thus requires C<SYS_CAP_ADMIN>.
 This mode uses Linux user namespaces to allow unprivileged use of chroot and
 creation of files that appear to be owned by the superuser inside the unshared
 namespace. A tarball created in this mode should be bit-by-bit identical to a
-tarball created with the B<root> mode. In Debian, this mode requires the sysctl
-C<kernel.unprivileged_userns_clone> being set to C<1>. The default used to be
-C<0> but was changed to C<1> with linux 5.10.1 or Debian 11 (Bullseye).
-B<SETTING THIS OPTION TO 1 HAS SECURITY IMPLICATIONS>. Refer to
-L<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=898446>
+tarball created with the B<root> mode.
 
 A directory chroot created with this mode will end up with wrong ownership
 information. For correct ownership information, the directory must be accessed
@@ -6716,7 +6721,7 @@ Performs cleanup tasks, unless B<--skip=cleanup> is used:
 
 =item * Remove all files that were put into the chroot for setup purposes, like F</etc/apt/apt.conf.d/00mmdebstrap>, the temporary apt config and the qemu-user-static binary. This can be disabled using B<--skip=cleanup/mmdebstrap>.
 
-=item * Remove all files that make the result unreproducible, like apt and dpkg logs and caches or F</etc/machine-id>. This can be disabled using B<--skip=cleanup/reproducible>
+=item * Remove all files that make the result unreproducible, like apt and dpkg logs and caches or F</etc/machine-id> and F</var/lib/dbus/machine-id>. This can be disabled using B<--skip=cleanup/reproducible>
 
 =item * Remove everything in F</tmp> inside the chroot. This can be disabled using B<--skip=cleanup/tmp>.
 
@@ -6812,12 +6817,14 @@ Create a bootable USB Stick that boots into a full Debian desktop:
     END
     # You can use $(sudo blockdev --getsize64 /dev/sdXXX) to get the right
     # image size for the target medium in bytes
-    $ guestfish -N debian-unstable.img=disk:8G -- part-disk /dev/sda mbr : \
-        part-set-bootable /dev/sda 1 true : mkfs ext2 /dev/sda1 : \
-        set-label /dev/sda1 rootfs : mount /dev/sda1 / : \
+    $ guestfish -N debian-unstable.img=disk:8G -- \
+        part-disk /dev/sda mbr : \
+        part-set-bootable /dev/sda 1 true : \
+        set-label /dev/sda1 rootfs : \
+        mkfs ext4 /dev/sda1 : mount /dev/sda1 / : \
         tar-in debian-unstable.tar / xattrs:true : \
-        upload /usr/lib/SYSLINUX/mbr.bin /mbr.bin : \
-        copy-file-to-device /mbr.bin /dev/sda size:440 : rm /mbr.bin : \
+        upload /usr/lib/EXTLINUX/mbr.bin /boot/mbr.bin : \
+        copy-file-to-device /boot/mbr.bin /dev/sda size:440 : \
         extlinux / : copy-in extlinux.conf / : sync : umount / : shutdown
     $ qemu-system-x86_64 -m 1G -enable-kvm debian-unstable.img
     $ sudo dd if=debian-unstable.img of=/dev/sdXXX status=progress
@@ -6852,10 +6859,11 @@ Use as replacement for autopkgtest-build-qemu and vmdb2:
     $ guestfish -N debian-unstable.img=disk:8G -- \
         part-disk /dev/sda mbr : \
         part-set-bootable /dev/sda 1 true : \
-        mkfs ext2 /dev/sda1 : mount /dev/sda1 / : \
+        mkfs ext4 /dev/sda1 : mount /dev/sda1 / : \
         tar-in debian-unstable.tar / xattrs:true : \
-        extlinux / : copy-in extlinux.conf / : \
-        sync : umount / : shutdown
+        upload /usr/lib/EXTLINUX/mbr.bin /boot/mbr.bin : \
+        copy-file-to-device /boot/mbr.bin /dev/sda size:440 : \
+        extlinux / : copy-in extlinux.conf / : sync : umount / : shutdown
     $ qemu-img convert -O qcow2 debian-unstable.img debian-unstable.qcow2
 
 As a debootstrap wrapper to run it without superuser privileges but using Linux

tarfilter

@@ -64,6 +64,10 @@ Both types of options use Unix shell-style wildcards:
        ? matches any single character
    [seq] matches any character in seq
   [!seq] matches any character not in seq
+
+Thirdly, strip leading directory components off of tar members. Just as with
+GNU tar --strip-components, tar members that have less or equal components in
+their path are not passed through.
 """
     )
     parser.add_argument(
@@ -90,8 +94,18 @@ Both types of options use Unix shell-style wildcards:
         action=PaxFilterAction,
         help="Re-include a pax header after a previous exclusion.",
     )
+    parser.add_argument(
+        "--strip-components",
+        metavar="number",
+        type=int,
+        help="Strip NUMBER leading components from file names",
+    )
     args = parser.parse_args()
-    if not hasattr(args, "pathfilter") and not hasattr(args, "paxfilter"):
+    if (
+        not hasattr(args, "pathfilter")
+        and not hasattr(args, "paxfilter")
+        and not hasattr(args, "strip_components")
+    ):
         from shutil import copyfileobj
 
         copyfileobj(sys.stdin.buffer, sys.stdout.buffer)
@@ -141,6 +155,11 @@ Both types of options use Unix shell-style wildcards:
         for member in in_tar:
             if path_filter_should_skip(member):
                 continue
+            if args.strip_components:
+                comps = member.name.split("/")
+                if len(comps) <= args.strip_components:
+                    continue
+                member.name = "/".join(comps[args.strip_components :])
             member.pax_headers = {
                 k: v
                 for k, v in member.pax_headers.items()