do not run xz and zstd with --threads=0

There are now systems with 160 cores (debci runs on two Ampere Altra
ARMv8 Neoverse-N1), which makes xz fail with: "xz: (stdin): Cannot
allocate memory"

coverage.sh

@@ -103,12 +103,12 @@ if [ ! -e shared/ldconfig.fakechroot ] || [ ldconfig.fakechroot -nt shared/ldcon
 		cp -a /usr/libexec/mmdebstrap/ldconfig.fakechroot shared/ldconfig.fakechroot
 	fi
 fi
-mkdir -p shared/hooks
+mkdir -p shared/hooks/merged-usr
-if [ ! -e shared/hooks/setup00-merged-usr.sh ] || [ hooks/setup00-merged-usr.sh -nt shared/hooks/setup00-merged-usr.sh ]; then
+if [ ! -e shared/hooks/merged-usr/setup00.sh ] || [ hooks/merged-usr/setup00.sh -nt shared/hooks/merged-usr/setup00.sh ]; then
-	if [ -e hooks/setup00-merged-usr.sh ]; then
+	if [ -e hooks/merged-usr/setup00.sh ]; then
-		cp -a hooks/setup00-merged-usr.sh shared/hooks/
+		cp -a hooks/merged-usr/setup00.sh shared/hooks/merged-usr/
 	else
-		cp -a /usr/share/mmdebstrap/hooks/setup00-merged-usr.sh shared/hooks/
+		cp -a /usr/share/mmdebstrap/hooks/merged-usr/setup00.sh shared/hooks/merged-usr/
 	fi
 fi
 mkdir -p shared/hooks/eatmydata
@@ -240,7 +240,7 @@ rm /tmp/debian-$dist-debootstrap/var/lib/dpkg/status-old \
 	/tmp/debian-$dist-mm/var/lib/dpkg/status-old
 # remove dpkg files
 rm /tmp/debian-$dist-debootstrap/var/lib/dpkg/available
-touch /tmp/debian-$dist-debootstrap/var/lib/dpkg/available
+rm /tmp/debian-$dist-debootstrap/var/lib/dpkg/cmethopt
 # since we installed packages directly from the .deb files, Priorities differ
 # thus we first check for equality and then remove the files
 chroot /tmp/debian-$dist-debootstrap dpkg --list > dpkg1
@@ -2076,7 +2076,7 @@ cat << END > shared/test.sh
 set -eu
 export LC_ALL=C.UTF-8
 $CMD --mode=root --variant=apt \
-	--setup-hook=./hooks/setup00-merged-usr.sh \
+	--setup-hook=./hooks/merged-usr/setup00.sh \
 	--customize-hook='[ -L "\$1"/bin -a -L "\$1"/sbin -a -L "\$1"/lib ]' \
 	$DEFAULT_DIST /tmp/debian-chroot $mirror
 tar -C /tmp/debian-chroot --one-file-system -c . | tar -t | sort > tar2.txt
@@ -2763,8 +2763,6 @@ rm /tmp/debian-chroot/etc/fstab
 rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
-rm /tmp/debian-chroot/var/lib/dpkg/available
-rm /tmp/debian-chroot/var/lib/dpkg/cmethopt
 rm /tmp/debian-chroot/var/lib/dpkg/status
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir
@@ -2793,8 +2791,6 @@ rm /tmp/debian-chroot/etc/fstab
 rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
-rm /tmp/debian-chroot/var/lib/dpkg/available
-rm /tmp/debian-chroot/var/lib/dpkg/cmethopt
 rm /tmp/debian-chroot/var/lib/dpkg/status
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir
@@ -2824,8 +2820,6 @@ rm /tmp/debian-chroot/etc/fstab
 rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
-rm /tmp/debian-chroot/var/lib/dpkg/available
-rm /tmp/debian-chroot/var/lib/dpkg/cmethopt
 rm /tmp/debian-chroot/var/lib/dpkg/status
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir
@@ -2844,8 +2838,8 @@ cat << END > shared/test.sh
 set -eu
 export LC_ALL=C.UTF-8
 $CMD --mode=$defaultmode --variant=essential --include=apt \
-	--essential-hook='apt-get -o Dir="\$1" -oDir::Etc::TrustedParts=/etc/apt/trusted.gpg.d -oAcquire::Languages=none update' \
+	--essential-hook='APT_CONFIG=\$MMDEBSTRAP_APT_CONFIG apt-get update' \
-	--essential-hook='apt-get --yes install --no-install-recommends -o Dir="\$1" -oDPkg::Chroot-Directory="\$1" apt' \
+	--essential-hook='APT_CONFIG=\$MMDEBSTRAP_APT_CONFIG apt-get --yes install -oDPkg::Chroot-Directory="\$1" apt' \
 	$DEFAULT_DIST /tmp/debian-chroot.tar $mirror
 tar -tf /tmp/debian-chroot.tar | sort | grep -v ./var/lib/apt/extended_states | diff -u tar1.txt -
 rm /tmp/debian-chroot.tar
@@ -2971,8 +2965,6 @@ rm /tmp/debian-chroot/etc/fstab
 rm /tmp/debian-chroot/etc/hostname
 rm /tmp/debian-chroot/etc/resolv.conf
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
-rm /tmp/debian-chroot/var/lib/dpkg/available
-rm /tmp/debian-chroot/var/lib/dpkg/cmethopt
 rm /tmp/debian-chroot/var/lib/dpkg/status
 # the rest should be empty directories that we can rmdir recursively
 find /tmp/debian-chroot -depth -print0 | xargs -0 rmdir
@@ -3216,10 +3208,6 @@ rm /tmp/debian-chroot/var/lib/dpkg/status
 rm /tmp/debian-chroot/var/cache/apt/archives/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock-frontend
-if [ "$mode" != "chrootless" ] || dpkg --compare-versions "\$(dpkg --robot --version)" lt 1.20.0; then
-rm /tmp/debian-chroot/var/lib/dpkg/available
-rm /tmp/debian-chroot/var/lib/dpkg/cmethopt
-fi
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
 ## delete merged usr symlinks
 #rm /tmp/debian-chroot/libx32
@@ -3285,10 +3273,6 @@ rm /tmp/debian-chroot/var/cache/apt/archives/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock-frontend
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
-if [ "$mode" != "chrootless" ] || dpkg --compare-versions "\$(dpkg --robot --version)" lt 1.20.0; then
-rm /tmp/debian-chroot/var/lib/dpkg/available
-rm /tmp/debian-chroot/var/lib/dpkg/cmethopt
-fi
 ## delete merged usr symlinks
 #rm /tmp/debian-chroot/libx32
 #rm /tmp/debian-chroot/lib64
@@ -3435,10 +3419,6 @@ rm /tmp/debian-chroot/var/cache/apt/archives/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock-frontend
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
-if dpkg --compare-versions "\$(dpkg --robot --version)" lt 1.20.0; then
-rm /tmp/debian-chroot/var/lib/dpkg/available
-rm /tmp/debian-chroot/var/lib/dpkg/cmethopt
-fi
 ## delete merged usr symlinks
 #rm /tmp/debian-chroot/libx32
 #rm /tmp/debian-chroot/lib64
@@ -3497,10 +3477,6 @@ rm /tmp/debian-chroot/var/cache/apt/archives/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock
 rm /tmp/debian-chroot/var/lib/dpkg/lock-frontend
 rm /tmp/debian-chroot/var/lib/apt/lists/lock
-if dpkg --compare-versions "\$(dpkg --robot --version)" lt 1.20.0; then
-rm /tmp/debian-chroot/var/lib/dpkg/available
-rm /tmp/debian-chroot/var/lib/dpkg/cmethopt
-fi
 ## delete merged usr symlinks
 #rm /tmp/debian-chroot/libx32
 #rm /tmp/debian-chroot/lib64

ldconfig.fakechroot

@@ -90,6 +90,12 @@ def main():
 
     chroot = Path(os.environ["FAKECHROOT_BASE_ORIG"])
 
+    # if chrootless mode is used from within a fakechroot chroot, then
+    # FAKECHROOT_BASE_ORIG will point at the outer chroot. We want to use
+    # the path from DPKG_ROOT inside of that instead
+    if os.environ.get("DPKG_ROOT", "") not in ["", "/"]:
+        chroot /= os.environ["DPKG_ROOT"].lstrip("/")
+
     if not (chroot / "sbin" / "ldconfig").exists():
         sys.exit(0)
 

make_mirror.sh

@@ -97,7 +97,7 @@ get_oldaptnames() {
 	if [ ! -e "$1/$2" ]; then
 		return
 	fi
-	gzip -dc "$1/$2" \
+	xz -dc "$1/$2" \
 		| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename '' \
 		| paste -sd "    \n" \
 		| while read name ver arch fname; do
@@ -122,10 +122,10 @@ get_newaptnames() {
 		return
 	fi
 	# skip empty files by trying to uncompress the first byte of the payload
-	if [ "$(gzip -dc "$1/$2" | head -c1 | wc -c)" -eq 0 ]; then
+	if [ "$(xz -dc "$1/$2" | head -c1 | wc -c)" -eq 0 ]; then
 		return
 	fi
-	gzip -dc "$1/$2" \
+	xz -dc "$1/$2" \
 		| grep-dctrl --no-field-names --show-field=Package,Version,Architecture,Filename,SHA256 '' \
 		| paste -sd "     \n" \
 		| while read name ver arch fname hash; do
@@ -235,17 +235,17 @@ END
 	# /var/cache/apt/archives so that apt will not re-download *.deb
 	# packages that we already have
 	{
-		get_oldaptnames "$oldmirrordir" "dists/$dist/main/binary-$nativearch/Packages.gz"
+		get_oldaptnames "$oldmirrordir" "dists/$dist/main/binary-$nativearch/Packages.xz"
 		case "$dist" in oldstable|stable)
-			get_oldaptnames "$oldmirrordir" "dists/$dist-updates/main/binary-$nativearch/Packages.gz"
+			get_oldaptnames "$oldmirrordir" "dists/$dist-updates/main/binary-$nativearch/Packages.xz"
 			;;
 		esac
 		case "$dist" in
 			oldstable)
-				get_oldaptnames "$oldcachedir/debian-security" "dists/$dist/updates/main/binary-$nativearch/Packages.gz"
+				get_oldaptnames "$oldcachedir/debian-security" "dists/$dist/updates/main/binary-$nativearch/Packages.xz"
 				;;
 			stable)
-				get_oldaptnames "$oldcachedir/debian-security" "dists/$dist-security/main/binary-$nativearch/Packages.gz"
+				get_oldaptnames "$oldcachedir/debian-security" "dists/$dist-security/main/binary-$nativearch/Packages.xz"
 				;;
 		esac
 	} | sort -u > "$rootdir/oldaptnames"
@@ -266,12 +266,12 @@ END
 	mkdir -p "$newmirrordir/dists/$dist/main/binary-$nativearch/"
 	curl --location "$mirror/dists/$dist/Release" > "$newmirrordir/dists/$dist/Release"
 	curl --location "$mirror/dists/$dist/Release.gpg" > "$newmirrordir/dists/$dist/Release.gpg"
-	curl --location "$mirror/dists/$dist/main/binary-$nativearch/Packages.gz" > "$newmirrordir/dists/$dist/main/binary-$nativearch/Packages.gz"
+	curl --location "$mirror/dists/$dist/main/binary-$nativearch/Packages.xz" > "$newmirrordir/dists/$dist/main/binary-$nativearch/Packages.xz"
 	case "$dist" in oldstable|stable)
 		mkdir -p "$newmirrordir/dists/$dist-updates/main/binary-$nativearch/"
 		curl --location "$mirror/dists/$dist-updates/Release" > "$newmirrordir/dists/$dist-updates/Release"
 		curl --location "$mirror/dists/$dist-updates/Release.gpg" > "$newmirrordir/dists/$dist-updates/Release.gpg"
-		curl --location "$mirror/dists/$dist-updates/main/binary-$nativearch/Packages.gz" > "$newmirrordir/dists/$dist-updates/main/binary-$nativearch/Packages.gz"
+		curl --location "$mirror/dists/$dist-updates/main/binary-$nativearch/Packages.xz" > "$newmirrordir/dists/$dist-updates/main/binary-$nativearch/Packages.xz"
 		;;
 	esac
 	case "$dist" in
@@ -279,13 +279,13 @@ END
 			mkdir -p "$newcachedir/debian-security/dists/$dist/updates/main/binary-$nativearch/"
 			curl --location "$security_mirror/dists/$dist/updates/Release" > "$newcachedir/debian-security/dists/$dist/updates/Release"
 			curl --location "$security_mirror/dists/$dist/updates/Release.gpg" > "$newcachedir/debian-security/dists/$dist/updates/Release.gpg"
-			curl --location "$security_mirror/dists/$dist/updates/main/binary-$nativearch/Packages.gz" > "$newcachedir/debian-security/dists/$dist/updates/main/binary-$nativearch/Packages.gz"
+			curl --location "$security_mirror/dists/$dist/updates/main/binary-$nativearch/Packages.xz" > "$newcachedir/debian-security/dists/$dist/updates/main/binary-$nativearch/Packages.xz"
 			;;
 		stable)
 			mkdir -p "$newcachedir/debian-security/dists/$dist-security/main/binary-$nativearch/"
 			curl --location "$security_mirror/dists/$dist-security/Release" > "$newcachedir/debian-security/dists/$dist-security/Release"
 			curl --location "$security_mirror/dists/$dist-security/Release.gpg" > "$newcachedir/debian-security/dists/$dist-security/Release.gpg"
-			curl --location "$security_mirror/dists/$dist-security/main/binary-$nativearch/Packages.gz" > "$newcachedir/debian-security/dists/$dist-security/main/binary-$nativearch/Packages.gz"
+			curl --location "$security_mirror/dists/$dist-security/main/binary-$nativearch/Packages.xz" > "$newcachedir/debian-security/dists/$dist-security/main/binary-$nativearch/Packages.xz"
 			;;
 	esac
 
@@ -298,17 +298,17 @@ END
 	# stripping the epoch from the filename and will break once mirrors change.
 	# This way, it doesn't matter where the mirror ends up storing the package.
 	{
-		get_newaptnames "$newmirrordir" "dists/$dist/main/binary-$nativearch/Packages.gz";
+		get_newaptnames "$newmirrordir" "dists/$dist/main/binary-$nativearch/Packages.xz";
 		case "$dist" in oldstable|stable)
-			get_newaptnames "$newmirrordir" "dists/$dist-updates/main/binary-$nativearch/Packages.gz"
+			get_newaptnames "$newmirrordir" "dists/$dist-updates/main/binary-$nativearch/Packages.xz"
 			;;
 		esac
 		case "$dist" in
 			oldstable)
-				get_newaptnames "$newcachedir/debian-security" "dists/$dist/updates/main/binary-$nativearch/Packages.gz"
+				get_newaptnames "$newcachedir/debian-security" "dists/$dist/updates/main/binary-$nativearch/Packages.xz"
 				;;
 			stable)
-				get_newaptnames "$newcachedir/debian-security" "dists/$dist-security/main/binary-$nativearch/Packages.gz"
+				get_newaptnames "$newcachedir/debian-security" "dists/$dist-security/main/binary-$nativearch/Packages.xz"
 				;;
 		esac
 	} | sort -u > "$rootdir/newaptnames"

mmdebstrap

@@ -244,9 +244,9 @@ sub get_tar_compressor {
     } elsif ($filename =~ /\.lz4$/) {
         return ['lz4'];
     } elsif ($filename =~ /\.(xz|txz)$/) {
-        return ['xz', '--threads=0'];
+        return ['xz'];
     } elsif ($filename =~ /\.zst$/) {
-        return ['zstd', '--threads=0'];
+        return ['zstd'];
     }
     return;
 }
@@ -1732,8 +1732,9 @@ sub run_setup() {
     # removals will fail
     # since we do not know the dpkg version inside the chroot at this point, we
     # can only omit it in chrootless mode
-    if (   $options->{mode} ne 'chrootless'
+    # since we want to produce equivalent output independent on the mode, we
-        or $dpkgversion < "1.20.0") {
+    # only check for the dpkg version
+    if ($dpkgversion < "1.20.0") {
         open my $fh, '>', "$options->{root}/var/lib/dpkg/available"
           or error "failed to open(): $!";
         close $fh;
@@ -1743,8 +1744,9 @@ sub run_setup() {
     # see #930788
     # since we do not know the dpkg version inside the chroot at this point, we
     # can only omit it in chrootless mode
-    if (   $options->{mode} ne 'chrootless'
+    # since we want to produce equivalent output independent on the mode, we
-        or $dpkgversion < "1.20.0") {
+    # only check for the dpkg version
+    if ($dpkgversion < "1.20.0") {
         open my $fh, '>', "$options->{root}/var/lib/dpkg/cmethopt"
           or error "failed to open(): $!";
         print $fh "apt apt\n";
@@ -7133,9 +7135,9 @@ extension to compressor applies:
  .tlz      lzma
  .lzo      lzop
  .lz4      lz4
- .xz       xz --threads=0
+ .xz       xz
- .txz      xz --threads=0
+ .txz      xz
- .zst      zstd --threads=0
+ .zst      zstd
 
 To change compression specific options, either use the respecitve environment
 variables like B<XZ_OPT> or send B<mmdebstrap> output to your compressor of