try unsharing before automatically choosing unshare mode

coverage.sh

@@ -69,7 +69,7 @@ mirror="http://127.0.0.1/debian"
 
 export HAVE_QEMU HAVE_BINFMT RUN_MA_SAME_TESTS DEFAULT_DIST SOURCE_DATE_EPOCH CMD mirror
 
-./coverage.py
+./coverage.py "$@"
 
 if [ -e shared/cover_db.img ]; then
 	# produce report inside the VM to make sure that the versions match or

coverage.txt

@@ -283,10 +283,16 @@ Test: debootstrap-no-op-options
 Needs-Root: true
 
 Test: verbose
-Needs-Root: true
+Variants: - standard
+Skip-If:
+ variant == "-" and hostarch not in ["armel", "armhf", "mipsel"] # #1031276
+ variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
 Test: debug
-Needs-Root: true
+Variants: - standard
+Skip-If:
+ variant == "-" and hostarch not in ["armel", "armhf", "mipsel"] # #1031276
+ variant == "standard" and hostarch in ["armel", "armhf", "mipsel"] # #1031276
 
 Test: quiet
 Needs-Root: true

make_mirror.sh

@@ -20,7 +20,10 @@ deletecache() {
 		return 1
 	fi
 	# be very careful with removing the old directory
-	for dist in oldstable stable testing unstable; do
+	# experimental is pulled in with USE_HOST_APT_CONFIG=yes on debci
+	# when testing a package from experimental
+	for dist in oldstable stable testing unstable experimental; do
+		# deleting artifacts from test "debootstrap"
 		for variant in minbase buildd -; do
 			if [ -e "$dir/debian-$dist-$variant.tar" ]; then
 				rm "$dir/debian-$dist-$variant.tar"
@@ -28,6 +31,18 @@ deletecache() {
 				echo "does not exist: $dir/debian-$dist-$variant.tar" >&2
 			fi
 		done
+		# deleting artifacts from test "mmdebstrap"
+		for variant in essential apt minbase buildd - standard; do
+			for format in tar ext2 squashfs; do
+				if [ -e "$dir/mmdebstrap-$dist-$variant.$format" ]; then
+					# attempt to delete for all dists because DEFAULT_DIST might've been different the last time
+					rm "$dir/mmdebstrap-$dist-$variant.$format"
+				elif [ "$dist" = "$DEFAULT_DIST" ]; then
+					# only warn about non-existance when it's expected to exist
+					echo "does not exist: $dir/mmdebstrap-$dist-$variant.$format" >&2
+				fi
+			done
+		done
 		if [ -e "$dir/debian/dists/$dist" ]; then
 			rm --one-file-system --recursive "$dir/debian/dists/$dist"
 		else
@@ -63,11 +78,16 @@ deletecache() {
 			rm --one-file-system "$f"
 		fi
 	done
-	if [ -e "$dir/debian/pool/main" ]; then
+	# on i386 and amd64, the intel-microcode and amd64-microcode packages
-		rm --one-file-system --recursive "$dir/debian/pool/main"
+	# from non-free-firwame get pulled in because they are
-	else
+	# priority:standard with USE_HOST_APT_CONFIG=yes
-		echo "does not exist: $dir/debian/pool/main" >&2
+	for c in main non-free-firmware; do
-	fi
+		if [ -e "$dir/debian/pool/$c" ]; then
+			rm --one-file-system --recursive "$dir/debian/pool/$c"
+		else
+			echo "does not exist: $dir/debian/pool/$c" >&2
+		fi
+	done
 	if [ -e "$dir/debian-security/pool/updates/main" ]; then
 		rm --one-file-system --recursive "$dir/debian-security/pool/updates/main"
 	else

mmdebstrap

@@ -304,15 +304,24 @@ sub shellescape {
 }
 
 sub test_unshare_userns {
-    my $verbose      = shift;
+    my $verbose = shift;
-    my $unshare_fail = shift;
+    my $fail    = shift;
-    if ($EFFECTIVE_USER_ID == 0) {
+
-        my $msg = "cannot unshare user namespace when executing as root";
+    local *maybe_warn = sub {
+        my $msg = shift;
         if ($verbose) {
-            warning $msg;
+            if ($fail) {
+                error $msg;
+            } else {
+                warning $msg;
+            }
         } else {
             debug $msg;
         }
+    };
+
+    if ($EFFECTIVE_USER_ID == 0) {
+        maybe_warn("cannot unshare user namespace when executing as root");
         return 0;
     }
     # arguments to syscalls have to be stored in their own variable or
@@ -326,12 +335,7 @@ sub test_unshare_userns {
         if ($ret == 0) {
             exit 0;
         } else {
-            my $msg = "unshare syscall failed: $!";
+            maybe_warn("unshare syscall failed: $!");
-            if ($verbose) {
-                warning $msg;
-            } else {
-                debug $msg;
-            }
             exit 1;
         }
     }
@@ -344,120 +348,140 @@ sub test_unshare_userns {
     system "newuidmap 2>/dev/null";
     if (($? >> 8) != 1) {
         if (($? >> 8) == 127) {
-            my $msg = "cannot find newuidmap";
+            maybe_warn("cannot find newuidmap");
-            if ($verbose) {
-                if ($unshare_fail) {
-                    error $msg;
-                } else {
-                    warning $msg;
-                }
-            } else {
-                debug $msg;
-            }
         } else {
-            my $msg = "newuidmap returned unknown exit status: $?";
+            maybe_warn("newuidmap returned unknown exit status: $?");
-            if ($verbose) {
-                warning $msg;
-            } else {
-                debug $msg;
-            }
         }
         return 0;
     }
     system "newgidmap 2>/dev/null";
     if (($? >> 8) != 1) {
         if (($? >> 8) == 127) {
-            my $msg = "cannot find newgidmap";
+            maybe_warn("cannot find newgidmap");
-            if ($verbose) {
-                warning $msg;
-            } else {
-                debug $msg;
-            }
         } else {
-            my $msg = "newgidmap returned unknown exit status: $?";
+            maybe_warn("newgidmap returned unknown exit status: $?");
-            if ($verbose) {
-                warning $msg;
-            } else {
-                debug $msg;
-            }
         }
         return 0;
     }
+    my @idmap = read_subuid_subgid($verbose);
+    if (scalar @idmap == 0) {
+        maybe_warn("failed to parse /etc/subuid and /etc/subgid");
+        return 0;
+    }
+    # too much can go wrong when doing the dance required to unsharing the user
+    # namespace, so instead of adding more complexity to support maybe_warn()
+    # to a function that is already too complex, we use eval()
+    eval {
+        $pid = get_unshare_cmd(
+            sub {
+                if ($EFFECTIVE_USER_ID == 0) {
+                    exit 0;
+                } else {
+                    exit 1;
+                }
+            },
+            \@idmap
+        );
+        waitpid $pid, 0;
+        if ($? != 0) {
+            maybe_warn("failed to unshare the user namespace");
+            return 0;
+        }
+    };
+    if ($@) {
+        maybe_warn($@);
+        return 0;
+    }
     return 1;
 }
 
-sub read_subuid_subgid() {
+sub read_subuid_subgid {
+    my $verbose  = shift;
+    my @result   = ();
     my $username = getpwuid $REAL_USER_ID;
     my ($subid, $num_subid, $fh, $n);
-    my @result = ();
 
+    local *maybe_warn = sub {
+        my $msg = shift;
+        if ($verbose) {
+            warning $msg;
+        } else {
+            debug $msg;
+        }
+    };
     if (!-e "/etc/subuid") {
-        warning "/etc/subuid doesn't exist";
+        maybe_warn("/etc/subuid doesn't exist");
         return;
     }
     if (!-r "/etc/subuid") {
-        warning "/etc/subuid is not readable";
+        maybe_warn("/etc/subuid is not readable");
         return;
     }
 
     open $fh, "<", "/etc/subuid"
-      or error "cannot open /etc/subuid for reading: $!";
+      or maybe_warn("cannot open /etc/subuid for reading: $!");
+    if (!$fh) {
+        return;
+    }
     while (my $line = <$fh>) {
         ($n, $subid, $num_subid) = split(/:/, $line, 3);
         last if ($n eq $username);
     }
     close $fh;
     if (!length $subid) {
-        warning "/etc/subuid is empty";
+        maybe_warn("/etc/subuid is empty");
         return;
     }
     if ($n ne $username) {
-        warning "no entry in /etc/subuid for $username";
+        maybe_warn("no entry in /etc/subuid for $username");
         return;
     }
     push @result, ["u", 0, $subid, $num_subid];
 
     if (scalar(@result) < 1) {
-        warning "/etc/subuid does not contain an entry for $username";
+        maybe_warn("/etc/subuid does not contain an entry for $username");
         return;
     }
     if (scalar(@result) > 1) {
-        warning "/etc/subuid contains multiple entries for $username";
+        maybe_warn("/etc/subuid contains multiple entries for $username");
         return;
     }
 
     if (!-e "/etc/subgid") {
-        warning "/etc/subgid doesn't exist";
+        maybe_warn("/etc/subgid doesn't exist");
         return;
     }
     if (!-r "/etc/subgid") {
-        warning "/etc/subgid is not readable";
+        maybe_warn("/etc/subgid is not readable");
         return;
     }
 
     open $fh, "<", "/etc/subgid"
-      or error "cannot open /etc/subgid for reading: $!";
+      or maybe_warn("cannot open /etc/subgid for reading: $!");
+    if (!$fh) {
+        return;
+    }
     while (my $line = <$fh>) {
         ($n, $subid, $num_subid) = split(/:/, $line, 3);
         last if ($n eq $username);
     }
     close $fh;
     if (!length $subid) {
-        warning "/etc/subgid is empty";
+        maybe_warn("/etc/subgid is empty");
         return;
     }
     if ($n ne $username) {
-        warning "no entry in /etc/subgid for $username";
+        maybe_warn("no entry in /etc/subgid for $username");
         return;
     }
     push @result, ["g", 0, $subid, $num_subid];
 
     if (scalar(@result) < 2) {
-        warning "/etc/subgid does not contain an entry for $username";
+        maybe_warn("/etc/subgid does not contain an entry for $username");
         return;
     }
     if (scalar(@result) > 2) {
-        warning "/etc/subgid contains multiple entries for $username";
+        maybe_warn("/etc/subgid contains multiple entries for $username");
         return;
     }
 
@@ -4353,7 +4377,7 @@ sub main() {
         }
         my @idmap = ();
         if ($EFFECTIVE_USER_ID != 0) {
-            @idmap = read_subuid_subgid;
+            @idmap = read_subuid_subgid 1;
         }
         my $pid = get_unshare_cmd(
             sub {
@@ -5665,7 +5689,7 @@ sub main() {
     # for unshare mode the rootfs directory has to have appropriate
     # permissions
     if ($EFFECTIVE_USER_ID != 0 and $options->{mode} eq 'unshare') {
-        @idmap = read_subuid_subgid;
+        @idmap = read_subuid_subgid 1;
         # sanity check
         if (   scalar(@idmap) != 2
             || $idmap[0][0] ne 'u'

tests/check-for-bit-by-bit-identical-format-output

@@ -5,14 +5,21 @@ export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
 
 trap "rm -f /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }}" EXIT INT TERM
 
-if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+case {{ MODE }} in unshare|fakechroot) : ;; *) exit 1;; esac
-	if [ ! -e /mmdebstrap-testenv ]; then
+
-		echo "this test modifies the system and should only be run inside a container" >&2
+prefix=
-		exit 1
+if [ "$(id -u)" -eq 0 ] && [ "{{ MODE }}" != "root" ] && [ "{{ MODE }}" != "auto" ]; then
+	if ! id "${SUDO_USER:-user}" >/dev/null 2>&1; then
+		if [ ! -e /mmdebstrap-testenv ]; then
+			echo "this test modifies the system and should only be run inside a container" >&2
+			exit 1
+		fi
+		useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
 	fi
-	useradd --home-dir "/home/${SUDO_USER:-user}" --create-home "${SUDO_USER:-user}"
+	prefix="runuser -u ${SUDO_USER:-user} --"
 fi
-runuser -u "${SUDO_USER:-user}" -- {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} {{ MIRROR }}
+
+$prefix {{ CMD }} --mode={{ MODE }} --variant={{ VARIANT }} {{ DIST }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} {{ MIRROR }}
 cmp ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }} \
 	|| diffoscope ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.{{ FORMAT }} /tmp/debian-chroot-{{ MODE }}.{{ FORMAT }}
 

tests/debug

@@ -1,6 +1,17 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-{{ CMD }} --mode=root --variant=apt --debug {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+
-rm -r /tmp/debian-chroot
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+
+# we use variant standard in verbose mode to see the maximum number of packages
+# that was chosen in case of USE_HOST_APT_CONFIG=yes
+# we use variant important on arches where variant standard is not bit-by-bit
+# reproducible due to #1031276
+case {{ VARIANT }} in standard|-) : ;; *) exit 1;; esac
+
+{{ CMD }} --variant={{ VARIANT }} --debug {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+
+cmp ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar /tmp/debian-chroot.tar \
+	|| diffoscope ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar /tmp/debian-chroot.tar

tests/fail-without-etc-subuid

@@ -13,4 +13,4 @@ if [ "$ret" = 0 ]; then
 	echo expected failure but got exit $ret >&2
 	exit 1
 fi
-rm -r /tmp/debian-chroot
+[ ! -e /tmp/debian-chroot ]

tests/fail-without-username-in-etc-subuid

@@ -14,4 +14,4 @@ if [ "$ret" = 0 ]; then
 	echo expected failure but got exit $ret >&2
 	exit 1
 fi
-rm -r /tmp/debian-chroot
+[ ! -e /tmp/debian-chroot ]

tests/verbose

@@ -1,6 +1,17 @@
 #!/bin/sh
 set -eu
 export LC_ALL=C.UTF-8
-{{ CMD }} --mode=root --variant=apt --verbose {{ DIST }} /tmp/debian-chroot {{ MIRROR }}
+export SOURCE_DATE_EPOCH={{ SOURCE_DATE_EPOCH }}
-tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort | diff -u tar1.txt -
+
-rm -r /tmp/debian-chroot
+trap "rm -f /tmp/debian-chroot.tar" EXIT INT TERM
+
+# we use variant standard in verbose mode to see the maximum number of packages
+# that was chosen in case of USE_HOST_APT_CONFIG=yes
+# we use variant important on arches where variant standard is not bit-by-bit
+# reproducible due to #1031276
+case {{ VARIANT }} in standard|-) : ;; *) exit 1;; esac
+
+{{ CMD }} --variant={{ VARIANT }} --verbose {{ DIST }} /tmp/debian-chroot.tar {{ MIRROR }}
+
+cmp ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar /tmp/debian-chroot.tar \
+	|| diffoscope ./cache/mmdebstrap-{{ DIST }}-{{ VARIANT }}.tar /tmp/debian-chroot.tar