From b3e08897c35a15c6ee994ab6d116b82803b1dc25 Mon Sep 17 00:00:00 2001 From: Johannes Schauer Marin Rodrigues Date: Thu, 16 Sep 2021 16:20:15 +0200 Subject: [PATCH] examples/twb: format with black --- examples/twb/debian-11-minimal.py | 128 ++++++----- examples/twb/debian-sid-zfs.py | 371 +++++++++++++++--------------- 2 files changed, 254 insertions(+), 245 deletions(-) diff --git a/examples/twb/debian-11-minimal.py b/examples/twb/debian-11-minimal.py index 7f287a0..fac7b9b 100644 --- a/examples/twb/debian-11-minimal.py +++ b/examples/twb/debian-11-minimal.py @@ -21,68 +21,80 @@ NOTE: this is the simplest config possible. """ parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument('output_file', nargs='?', default=pathlib.Path('filesystem.img'), type=pathlib.Path) +parser.add_argument( + "output_file", nargs="?", default=pathlib.Path("filesystem.img"), type=pathlib.Path +) args = parser.parse_args() -filesystem_img_size = '256M' # big enough to include filesystem.squashfs + about 64M of bootloader, kernel, and ramdisk. -esp_offset = 1024 * 1024 # 1MiB -esp_label = 'UEFI-ESP' # max 8 bytes for FAT32 -live_media_path = 'debian-live' +filesystem_img_size = "256M" # big enough to include filesystem.squashfs + about 64M of bootloader, kernel, and ramdisk. +esp_offset = 1024 * 1024 # 1MiB +esp_label = "UEFI-ESP" # max 8 bytes for FAT32 +live_media_path = "debian-live" -with tempfile.TemporaryDirectory(prefix='debian-live-bullseye-amd64-minimal.') as td: +with tempfile.TemporaryDirectory(prefix="debian-live-bullseye-amd64-minimal.") as td: td = pathlib.Path(td) subprocess.check_call( - ['mmdebstrap', - '--mode=unshare', - '--variant=apt', - '--aptopt=Acquire::http::Proxy "http://apt-cacher-ng.cyber.com.au:3142"', - '--aptopt=Acquire::https::Proxy "DIRECT"', - '--dpkgopt=force-unsafe-io', - '--include=linux-image-amd64 init initramfs-tools live-boot netbase', - '--include=dbus', # https://bugs.debian.org/814758 - '--include=live-config iproute2 keyboard-configuration locales sudo user-setup', - '--include=ifupdown isc-dhcp-client', # live-config doesn't support systemd-networkd yet. + [ + "mmdebstrap", + "--mode=unshare", + "--variant=apt", + '--aptopt=Acquire::http::Proxy "http://apt-cacher-ng.cyber.com.au:3142"', + '--aptopt=Acquire::https::Proxy "DIRECT"', + "--dpkgopt=force-unsafe-io", + "--include=linux-image-amd64 init initramfs-tools live-boot netbase", + "--include=dbus", # https://bugs.debian.org/814758 + "--include=live-config iproute2 keyboard-configuration locales sudo user-setup", + "--include=ifupdown isc-dhcp-client", # live-config doesn't support systemd-networkd yet. + # Do the **BARE MINIMUM** to make a USB key that can boot on X86_64 UEFI. + # We use mtools so we do not ever need root privileges. + # We can't use mkfs.vfat, as that needs kpartx or losetup (i.e. root). + # We can't use mkfs.udf, as that needs mount (i.e. root). + # We can't use "refind-install --usedefault" as that runs mount(8) (i.e. root). + # We don't use genisoimage because + # 1) ISO9660 must die; + # 2) incomplete UDF 1.5+ support; + # 3) resulting filesystem can't be tweaked after flashing (e.g. debian-live/site.dir/etc/systemd/network/up.network). + # + # We use refind because 1) I hate grub; and 2) I like refind. + # If you want aarch64 or ia32 you need to install their BOOTxxx.EFI files. + # If you want kernel+initrd on something other than FAT, you need refind/drivers_xxx/xxx_xxx.EFI. + # + # FIXME: with qemu in UEFI mode (OVMF), I get dumped into startup.nsh (UEFI REPL). + # From there, I can manually type in "FS0:\EFI\BOOT\BOOTX64.EFI" to start refind, tho. + # So WTF is its problem? Does it not support fallback bootloader? + "--include=refind parted mtools", + "--essential-hook=echo refind refind/install_to_esp boolean false | chroot $1 debconf-set-selections", + "--customize-hook=echo refind refind/install_to_esp boolean true | chroot $1 debconf-set-selections", + "--customize-hook=chroot $1 mkdir -p /boot/USB /boot/EFI/BOOT", + "--customize-hook=chroot $1 cp /usr/share/refind/refind/refind_x64.efi /boot/EFI/BOOT/BOOTX64.EFI", + f"--customize-hook=chroot $1 truncate --size={filesystem_img_size} /boot/USB/filesystem.img", + f"--customize-hook=chroot $1 parted --script --align=optimal /boot/USB/filesystem.img mklabel gpt mkpart {esp_label} {esp_offset}b 100% set 1 esp on", + f"--customize-hook=chroot $1 mformat -i /boot/USB/filesystem.img@@{esp_offset} -F -v {esp_label}", + f"--customize-hook=chroot $1 mmd -i /boot/USB/filesystem.img@@{esp_offset} ::{live_media_path}", + f"""--customize-hook=echo '"Boot with default options" "boot=live live-media-path={live_media_path}"' >$1/boot/refind_linux.conf""", + # NOTE: find sidesteps the "glob expands before chroot applies" problem. + f"""--customize-hook=chroot $1 find -O3 /boot/ -xdev -mindepth 1 -maxdepth 1 -regextype posix-egrep -iregex '.*/(EFI|refind_linux.conf|vmlinuz.*|initrd.img.*)' -exec mcopy -vsbpm -i /boot/USB/filesystem.img@@{esp_offset} {{}} :: ';'""", + # FIXME: copy-out doesn't handle sparseness, so is REALLY slow (about 50 seconds). + # Therefore instead leave it in the squashfs, and extract it later. + # f'--customize-hook=copy-out /boot/USB/filesystem.img /tmp/', + # f'--customize-hook=chroot $1 rm /boot/USB/filesystem.img', + "bullseye", + td / "filesystem.squashfs", + ] + ) - # Do the **BARE MINIMUM** to make a USB key that can boot on X86_64 UEFI. - # We use mtools so we do not ever need root privileges. - # We can't use mkfs.vfat, as that needs kpartx or losetup (i.e. root). - # We can't use mkfs.udf, as that needs mount (i.e. root). - # We can't use "refind-install --usedefault" as that runs mount(8) (i.e. root). - # We don't use genisoimage because - # 1) ISO9660 must die; - # 2) incomplete UDF 1.5+ support; - # 3) resulting filesystem can't be tweaked after flashing (e.g. debian-live/site.dir/etc/systemd/network/up.network). - # - # We use refind because 1) I hate grub; and 2) I like refind. - # If you want aarch64 or ia32 you need to install their BOOTxxx.EFI files. - # If you want kernel+initrd on something other than FAT, you need refind/drivers_xxx/xxx_xxx.EFI. - # - # FIXME: with qemu in UEFI mode (OVMF), I get dumped into startup.nsh (UEFI REPL). - # From there, I can manually type in "FS0:\EFI\BOOT\BOOTX64.EFI" to start refind, tho. - # So WTF is its problem? Does it not support fallback bootloader? - '--include=refind parted mtools', - '--essential-hook=echo refind refind/install_to_esp boolean false | chroot $1 debconf-set-selections', - '--customize-hook=echo refind refind/install_to_esp boolean true | chroot $1 debconf-set-selections', - '--customize-hook=chroot $1 mkdir -p /boot/USB /boot/EFI/BOOT', - '--customize-hook=chroot $1 cp /usr/share/refind/refind/refind_x64.efi /boot/EFI/BOOT/BOOTX64.EFI', - f'--customize-hook=chroot $1 truncate --size={filesystem_img_size} /boot/USB/filesystem.img', - f'--customize-hook=chroot $1 parted --script --align=optimal /boot/USB/filesystem.img mklabel gpt mkpart {esp_label} {esp_offset}b 100% set 1 esp on', - f'--customize-hook=chroot $1 mformat -i /boot/USB/filesystem.img@@{esp_offset} -F -v {esp_label}', - f'--customize-hook=chroot $1 mmd -i /boot/USB/filesystem.img@@{esp_offset} ::{live_media_path}', - f"""--customize-hook=echo '"Boot with default options" "boot=live live-media-path={live_media_path}"' >$1/boot/refind_linux.conf""", - # NOTE: find sidesteps the "glob expands before chroot applies" problem. - f"""--customize-hook=chroot $1 find -O3 /boot/ -xdev -mindepth 1 -maxdepth 1 -regextype posix-egrep -iregex '.*/(EFI|refind_linux.conf|vmlinuz.*|initrd.img.*)' -exec mcopy -vsbpm -i /boot/USB/filesystem.img@@{esp_offset} {{}} :: ';'""", - # FIXME: copy-out doesn't handle sparseness, so is REALLY slow (about 50 seconds). - # Therefore instead leave it in the squashfs, and extract it later. - # f'--customize-hook=copy-out /boot/USB/filesystem.img /tmp/', - # f'--customize-hook=chroot $1 rm /boot/USB/filesystem.img', - - 'bullseye', - td / 'filesystem.squashfs' - ]) - - with args.output_file.open('wb') as f: - subprocess.check_call(['rdsquashfs', '--cat=boot/USB/filesystem.img', td / 'filesystem.squashfs'], stdout=f) - subprocess.check_call([ - 'mcopy', '-i', f'{args.output_file}@@{esp_offset}', td / 'filesystem.squashfs', f'::{live_media_path}/filesystem.squashfs']) + with args.output_file.open("wb") as f: + subprocess.check_call( + ["rdsquashfs", "--cat=boot/USB/filesystem.img", td / "filesystem.squashfs"], + stdout=f, + ) + subprocess.check_call( + [ + "mcopy", + "-i", + f"{args.output_file}@@{esp_offset}", + td / "filesystem.squashfs", + f"::{live_media_path}/filesystem.squashfs", + ] + ) diff --git a/examples/twb/debian-sid-zfs.py b/examples/twb/debian-sid-zfs.py index de0c758..1d2e9ce 100644 --- a/examples/twb/debian-sid-zfs.py +++ b/examples/twb/debian-sid-zfs.py @@ -19,197 +19,194 @@ which in turn includes a bootloader (refind), kernel, ramdisk, and filesystem.sq """ parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument('output_file', nargs='?', default=pathlib.Path('filesystem.img'), type=pathlib.Path) -parser.add_argument('--timezone', default='Australia/Melbourne', type=lambda s: s.split('/'), help='NOTE: MUST be "Area/Zone" not e.g. "UTC", for now') -parser.add_argument('--locale', default='en_AU.UTF-8', help='NOTE: MUST end in ".UTF-8", for now') +parser.add_argument( + "output_file", nargs="?", default=pathlib.Path("filesystem.img"), type=pathlib.Path +) +parser.add_argument( + "--timezone", + default="Australia/Melbourne", + type=lambda s: s.split("/"), + help='NOTE: MUST be "Area/Zone" not e.g. "UTC", for now', +) +parser.add_argument( + "--locale", default="en_AU.UTF-8", help='NOTE: MUST end in ".UTF-8", for now' +) args = parser.parse_args() -filesystem_img_size = '512M' # big enough to include filesystem.squashfs + about 64M of bootloader, kernel, and ramdisk. -esp_offset = 1024 * 1024 # 1MiB -esp_label = 'UEFI-ESP' # max 8 bytes for FAT32 -live_media_path = 'debian-live' +filesystem_img_size = "512M" # big enough to include filesystem.squashfs + about 64M of bootloader, kernel, and ramdisk. +esp_offset = 1024 * 1024 # 1MiB +esp_label = "UEFI-ESP" # max 8 bytes for FAT32 +live_media_path = "debian-live" -with tempfile.TemporaryDirectory(prefix='debian-sid-zfs.') as td: +with tempfile.TemporaryDirectory(prefix="debian-sid-zfs.") as td: td = pathlib.Path(td) subprocess.check_call( - ['mmdebstrap', - '--mode=unshare', - '--variant=apt', - '--aptopt=Acquire::http::Proxy "http://apt-cacher-ng.cyber.com.au:3142"', - '--aptopt=Acquire::https::Proxy "DIRECT"', - '--dpkgopt=force-unsafe-io', - '--components=main contrib non-free', # needed for CPU security patches - - '--include=init initramfs-tools xz-utils live-boot netbase', - '--include=dbus', # https://bugs.debian.org/814758 - '--include=linux-image-amd64 firmware-linux', - - # Have ZFS 2.0 support. - '--include=zfs-dkms zfsutils-linux zfs-zed build-essential linux-headers-amd64', # ZFS 2 support - - # Make the initrd a little smaller (41MB -> 20MB), at the expensive of significantly slower image build time. - '--include=zstd', - '--essential-hook=mkdir -p $1/etc/initramfs-tools/conf.d', - '--essential-hook=>$1/etc/initramfs-tools/conf.d/zstd echo COMPRESS=zstd', - - # Be the equivalent of Debian Live GNOME - # '--include=live-task-gnome', - #'--include=live-task-xfce', - # FIXME: enable this? It makes live-task-xfce go from 1G to 16G... so no. - #'--aptopt=Apt::Install-Recommends "true"', - # ...cherry-pick instead - # UPDATE: debian-installer-launcher DOES NOT WORK because we don't load crap SPECIFICALLY into /live/installer, in the ESP. - # UPDATE: network-manager-gnome DOES NOT WORK, nor is systemd-networkd auto-started... WTF? - # end result is no networking. - #'--include=live-config user-setup sudo firmware-linux haveged', - #'--include=calamares-settings-debian udisks2', # 300MB weirdo Qt GUI debian installer - #'--include=xfce4-terminal', - - # x86_64 CPUs are undocumented proprietary RISC chips that EMULATE a documented x86_64 CISC ISA. - # The emulator is called "microcode", and is full of security vulnerabilities. - # Make sure security patches for microcode for *ALL* CPUs are included. - # By default, it tries to auto-detect the running CPU, so only patches the CPU of the build server. - '--include=intel-microcode amd64-microcode iucode-tool', - '--essential-hook=>$1/etc/default/intel-microcode echo IUCODE_TOOL_INITRAMFS=yes IUCODE_TOOL_SCANCPUS=no', - '--essential-hook=>$1/etc/default/amd64-microcode echo AMD64UCODE_INITRAMFS=yes', - '--dpkgopt=force-confold', # Work around https://bugs.debian.org/981004 - - # DHCP/DNS/SNTP clients... - # FIXME: use live-config ? - '--include=libnss-resolve libnss-myhostname systemd-timesyncd', - '--customize-hook=chroot $1 cp -alf /lib/systemd/resolv.conf /etc/resolv.conf', # This probably needs to happen LAST - # FIXME: fix resolv.conf to point to resolved, not "copy from the build-time OS" - # FIXME: fix hostname & hosts to not exist, not "copy from the build-time OS" - '--customize-hook=systemctl --root=$1 enable systemd-networkd systemd-timesyncd', # is this needed? - # Run a DHCP client on *ALL* ifaces. - # Consider network "up" (start sshd and local login prompt) when *ANY* (not ALL) ifaces are up. - "--customize-hook=>$1/etc/systemd/network/up.network printf '%s\n' '[Match]' Name='en*' '[Network]' DHCP=yes", # try DHCP on all ethernet ifaces - '--customize-hook=mkdir $1/etc/systemd/system/systemd-networkd-wait-online.service.d', - "--customize-hook=>$1/etc/systemd/system/systemd-networkd-wait-online.service.d/any-not-all.conf printf '%s\n' '[Service]' 'ExecStart=' 'ExecStart=/lib/systemd/systemd-networkd-wait-online --any'", - - # Hope there's a central smarthost SMTP server called "mail" in the local search domain. - # FIXME: can live-config do this? - '--include=msmtp-mta', - "--customize-hook=>$1/etc/msmtprc printf '%s\n' 'account default' 'syslog LOG_MAIL' 'host mail' 'auto_from on'", - - # Hope there's a central RELP logserver called "logserv" in the local domain. - # FIXME: can live-config do this? - '--include=rsyslog-relp', - """--customize-hook=>$1/etc/rsyslog.conf printf '%s\n' 'module(load="imuxsock")' 'module(load="imklog")' 'module(load="omrelp")' 'action(type="omrelp" target="logserv" port="2514" template="RSYSLOG_SyslogProtocol23Format")'""", - - # Run self-tests on all discoverable hard disks, and (try to) email if something goes wrong. - '--include=smartmontools bsd-mailx', - "--customize-hook=>$1/etc/smartd.conf echo 'DEVICESCAN -n standby,15 -a -o on -S on -s (S/../../7/00|L/../01/./01) -t -H -m root -M once'", - - # For rarely-updated, rarely-rebooted SOEs, apply what security updates we can into transient tmpfs COW. - # This CANNOT apply kernel security updates (though it will download them). - # This CANNOT make the upgrades persistent across reboots (they re-download each boot). - # FIXME: Would it be cleaner to set Environment=NEEDRESTART_MODE=a in - # apt-daily-upgrade.service and/or - # unattended-upgrades.service, so - # needrestart is noninteractive only when apt is noninteractive? - '--include=unattended-upgrades needrestart', - "--customize-hook=echo 'unattended-upgrades unattended-upgrades/enable_auto_updates boolean true' | chroot $1 debconf-set-selections", - """--customize-hook=>$1/etc/needrestart/conf.d/unattended-needrestart.conf echo '$nrconf{restart} = "a";'""", # https://bugs.debian.org/894444 - # Do an apt update & apt upgrade at boot time (as well as @daily). - # The lack of /etc/machine-id causes these to be implicitly enabled. - # FIXME: use dropin in /etc. - "--customize-hook=>>$1/lib/systemd/system/apt-daily.service printf '%s\n' '[Install]' 'WantedBy=multi-user.target'", - "--customize-hook=>>$1/lib/systemd/system/apt-daily-upgrade.service printf '%s\n' '[Install]' 'WantedBy=multi-user.target'", - - # FIXME: add support for this stuff (for the non-live final install this happens via ansible): - # - # unattended-upgrades - # smartd - # networkd (boot off ANY NIC, not EVERY NIC -- https://github.com/systemd/systemd/issues/9714) - # refind (bootloader config) - # misc safety nets - # double-check that mmdebstrap's machine-id support works properly - - # Bare minimum to let me SSH in. - # FIXME: make this configurable. - # FIXME: trust a CA certificate instead -- see Zero Trust SSH, Jeremy Stott, LCA 2020 - # WARNING: tinysshd does not support RSA, nor MaxStartups, nor sftp (unless you also install openssh-client, which is huge). - # FIXME: double-check no host keys are baked into the image (openssh-server and dropbear do this). - '--include=tinysshd rsync', - '--essential-hook=install -dm700 $1/root/.ssh', - '--essential-hook=echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIapAZ0E0353DaY6xBnasvu/DOvdWdKQ6RQURwq4l6Wu twb@cyber.com.au (Trent W. Buck)" >$1/root/.ssh/authorized_keys', - - # Bare minimum to let me log in locally. - # DO NOT use this on production builds! - '--essential-hook=chroot $1 passwd --delete root', - - # Configure language (not needed to boot). - # Racism saves a **LOT** of space -- something like 2GB for Debian Live images. - # FIXME: use live-config instead? - '--include=locales localepurge', - f'--essential-hook=echo locales locales/default_environment_locale select {args.locale} | chroot $1 debconf-set-selections', - f'--essential-hook=echo locales locales/locales_to_be_generated multiselect {args.locale} UTF-8 | chroot $1 debconf-set-selections', - # FIXME: https://bugs.debian.org/603700 - "--customize-hook=chroot $1 sed -i /etc/locale.nopurge -e 's/^USE_DPKG/#ARGH#&/'", - "--customize-hook=chroot $1 localepurge", - "--customize-hook=chroot $1 sed -i /etc/locale.nopurge -e 's/^#ARGH#//'", - - - # Removing documentation also saves a LOT of space. - '--dpkgopt=path-exclude=/usr/share/doc/*', - '--dpkgopt=path-exclude=/usr/share/info/*', - '--dpkgopt=path-exclude=/usr/share/man/*', - '--dpkgopt=path-exclude=/usr/share/omf/*', - '--dpkgopt=path-exclude=/usr/share/help/*', - '--dpkgopt=path-exclude=/usr/share/gnome/help/*', - - - # Configure timezone (not needed to boot)` - # FIXME: use live-config instead? - '--include=tzdata', - f'--essential-hook=echo tzdata tzdata/Areas select {args.timezone[0]} | chroot $1 debconf-set-selections', - f'--essential-hook=echo tzdata tzdata/Zones/{args.timezone[0]} select {args.timezone[1]} | chroot $1 debconf-set-selections', - - - # Do the **BARE MINIMUM** to make a USB key that can boot on X86_64 UEFI. - # We use mtools so we do not ever need root privileges. - # We can't use mkfs.vfat, as that needs kpartx or losetup (i.e. root). - # We can't use mkfs.udf, as that needs mount (i.e. root). - # We can't use "refind-install --usedefault" as that runs mount(8) (i.e. root). - # We don't use genisoimage because - # 1) ISO9660 must die; - # 2) incomplete UDF 1.5+ support; - # 3) resulting filesystem can't be tweaked after flashing (e.g. debian-live/site.dir/etc/systemd/network/up.network). - # - # We use refind because 1) I hate grub; and 2) I like refind. - # If you want aarch64 or ia32 you need to install their BOOTxxx.EFI files. - # If you want kernel+initrd on something other than FAT, you need refind/drivers_xxx/xxx_xxx.EFI. - # - # FIXME: with qemu in UEFI mode (OVMF), I get dumped into startup.nsh (UEFI REPL). - # From there, I can manually type in "FS0:\EFI\BOOT\BOOTX64.EFI" to start refind, tho. - # So WTF is its problem? Does it not support fallback bootloader? - '--include=refind parted mtools', - '--essential-hook=echo refind refind/install_to_esp boolean false | chroot $1 debconf-set-selections', - '--customize-hook=echo refind refind/install_to_esp boolean true | chroot $1 debconf-set-selections', - '--customize-hook=chroot $1 mkdir -p /boot/USB /boot/EFI/BOOT', - '--customize-hook=chroot $1 cp /usr/share/refind/refind/refind_x64.efi /boot/EFI/BOOT/BOOTX64.EFI', - '--customize-hook=chroot $1 cp /usr/share/refind/refind/refind.conf-sample /boot/EFI/BOOT/refind.conf', - f'--customize-hook=chroot $1 truncate --size={filesystem_img_size} /boot/USB/filesystem.img', - f'--customize-hook=chroot $1 parted --script --align=optimal /boot/USB/filesystem.img mklabel gpt mkpart {esp_label} {esp_offset}b 100% set 1 esp on', - f'--customize-hook=chroot $1 mformat -i /boot/USB/filesystem.img@@{esp_offset} -F -v {esp_label}', - f'--customize-hook=chroot $1 mmd -i /boot/USB/filesystem.img@@{esp_offset} ::{live_media_path}', - f"""--customize-hook=echo '"Boot with default options" "boot=live live-media-path={live_media_path}"' >$1/boot/refind_linux.conf""", - - f"""--customize-hook=chroot $1 find /boot/ -xdev -mindepth 1 -maxdepth 1 -not -name filesystem.img -not -name USB -exec mcopy -vsbpm -i /boot/USB/filesystem.img@@{esp_offset} {{}} :: ';'""", - # FIXME: copy-out doesn't handle sparseness, so is REALLY slow (about 50 seconds). - # Therefore instead leave it in the squashfs, and extract it later. - # f'--customize-hook=copy-out /boot/USB/filesystem.img /tmp/', - # f'--customize-hook=chroot $1 rm /boot/USB/filesystem.img', - - - 'sid', - td / 'filesystem.squashfs' - ]) - - with args.output_file.open('wb') as f: - subprocess.check_call(['rdsquashfs', '--cat=boot/USB/filesystem.img', td / 'filesystem.squashfs'], stdout=f) - subprocess.check_call([ - 'mcopy', '-i', f'{args.output_file}@@{esp_offset}', td / 'filesystem.squashfs', f'::{live_media_path}/filesystem.squashfs']) + [ + "mmdebstrap", + "--mode=unshare", + "--variant=apt", + '--aptopt=Acquire::http::Proxy "http://apt-cacher-ng.cyber.com.au:3142"', + '--aptopt=Acquire::https::Proxy "DIRECT"', + "--dpkgopt=force-unsafe-io", + "--components=main contrib non-free", # needed for CPU security patches + "--include=init initramfs-tools xz-utils live-boot netbase", + "--include=dbus", # https://bugs.debian.org/814758 + "--include=linux-image-amd64 firmware-linux", + # Have ZFS 2.0 support. + "--include=zfs-dkms zfsutils-linux zfs-zed build-essential linux-headers-amd64", # ZFS 2 support + # Make the initrd a little smaller (41MB -> 20MB), at the expensive of significantly slower image build time. + "--include=zstd", + "--essential-hook=mkdir -p $1/etc/initramfs-tools/conf.d", + "--essential-hook=>$1/etc/initramfs-tools/conf.d/zstd echo COMPRESS=zstd", + # Be the equivalent of Debian Live GNOME + # '--include=live-task-gnome', + #'--include=live-task-xfce', + # FIXME: enable this? It makes live-task-xfce go from 1G to 16G... so no. + #'--aptopt=Apt::Install-Recommends "true"', + # ...cherry-pick instead + # UPDATE: debian-installer-launcher DOES NOT WORK because we don't load crap SPECIFICALLY into /live/installer, in the ESP. + # UPDATE: network-manager-gnome DOES NOT WORK, nor is systemd-networkd auto-started... WTF? + # end result is no networking. + #'--include=live-config user-setup sudo firmware-linux haveged', + #'--include=calamares-settings-debian udisks2', # 300MB weirdo Qt GUI debian installer + #'--include=xfce4-terminal', + # x86_64 CPUs are undocumented proprietary RISC chips that EMULATE a documented x86_64 CISC ISA. + # The emulator is called "microcode", and is full of security vulnerabilities. + # Make sure security patches for microcode for *ALL* CPUs are included. + # By default, it tries to auto-detect the running CPU, so only patches the CPU of the build server. + "--include=intel-microcode amd64-microcode iucode-tool", + "--essential-hook=>$1/etc/default/intel-microcode echo IUCODE_TOOL_INITRAMFS=yes IUCODE_TOOL_SCANCPUS=no", + "--essential-hook=>$1/etc/default/amd64-microcode echo AMD64UCODE_INITRAMFS=yes", + "--dpkgopt=force-confold", # Work around https://bugs.debian.org/981004 + # DHCP/DNS/SNTP clients... + # FIXME: use live-config ? + "--include=libnss-resolve libnss-myhostname systemd-timesyncd", + "--customize-hook=chroot $1 cp -alf /lib/systemd/resolv.conf /etc/resolv.conf", # This probably needs to happen LAST + # FIXME: fix resolv.conf to point to resolved, not "copy from the build-time OS" + # FIXME: fix hostname & hosts to not exist, not "copy from the build-time OS" + "--customize-hook=systemctl --root=$1 enable systemd-networkd systemd-timesyncd", # is this needed? + # Run a DHCP client on *ALL* ifaces. + # Consider network "up" (start sshd and local login prompt) when *ANY* (not ALL) ifaces are up. + "--customize-hook=>$1/etc/systemd/network/up.network printf '%s\n' '[Match]' Name='en*' '[Network]' DHCP=yes", # try DHCP on all ethernet ifaces + "--customize-hook=mkdir $1/etc/systemd/system/systemd-networkd-wait-online.service.d", + "--customize-hook=>$1/etc/systemd/system/systemd-networkd-wait-online.service.d/any-not-all.conf printf '%s\n' '[Service]' 'ExecStart=' 'ExecStart=/lib/systemd/systemd-networkd-wait-online --any'", + # Hope there's a central smarthost SMTP server called "mail" in the local search domain. + # FIXME: can live-config do this? + "--include=msmtp-mta", + "--customize-hook=>$1/etc/msmtprc printf '%s\n' 'account default' 'syslog LOG_MAIL' 'host mail' 'auto_from on'", + # Hope there's a central RELP logserver called "logserv" in the local domain. + # FIXME: can live-config do this? + "--include=rsyslog-relp", + """--customize-hook=>$1/etc/rsyslog.conf printf '%s\n' 'module(load="imuxsock")' 'module(load="imklog")' 'module(load="omrelp")' 'action(type="omrelp" target="logserv" port="2514" template="RSYSLOG_SyslogProtocol23Format")'""", + # Run self-tests on all discoverable hard disks, and (try to) email if something goes wrong. + "--include=smartmontools bsd-mailx", + "--customize-hook=>$1/etc/smartd.conf echo 'DEVICESCAN -n standby,15 -a -o on -S on -s (S/../../7/00|L/../01/./01) -t -H -m root -M once'", + # For rarely-updated, rarely-rebooted SOEs, apply what security updates we can into transient tmpfs COW. + # This CANNOT apply kernel security updates (though it will download them). + # This CANNOT make the upgrades persistent across reboots (they re-download each boot). + # FIXME: Would it be cleaner to set Environment=NEEDRESTART_MODE=a in + # apt-daily-upgrade.service and/or + # unattended-upgrades.service, so + # needrestart is noninteractive only when apt is noninteractive? + "--include=unattended-upgrades needrestart", + "--customize-hook=echo 'unattended-upgrades unattended-upgrades/enable_auto_updates boolean true' | chroot $1 debconf-set-selections", + """--customize-hook=>$1/etc/needrestart/conf.d/unattended-needrestart.conf echo '$nrconf{restart} = "a";'""", # https://bugs.debian.org/894444 + # Do an apt update & apt upgrade at boot time (as well as @daily). + # The lack of /etc/machine-id causes these to be implicitly enabled. + # FIXME: use dropin in /etc. + "--customize-hook=>>$1/lib/systemd/system/apt-daily.service printf '%s\n' '[Install]' 'WantedBy=multi-user.target'", + "--customize-hook=>>$1/lib/systemd/system/apt-daily-upgrade.service printf '%s\n' '[Install]' 'WantedBy=multi-user.target'", + # FIXME: add support for this stuff (for the non-live final install this happens via ansible): + # + # unattended-upgrades + # smartd + # networkd (boot off ANY NIC, not EVERY NIC -- https://github.com/systemd/systemd/issues/9714) + # refind (bootloader config) + # misc safety nets + # double-check that mmdebstrap's machine-id support works properly + # Bare minimum to let me SSH in. + # FIXME: make this configurable. + # FIXME: trust a CA certificate instead -- see Zero Trust SSH, Jeremy Stott, LCA 2020 + # WARNING: tinysshd does not support RSA, nor MaxStartups, nor sftp (unless you also install openssh-client, which is huge). + # FIXME: double-check no host keys are baked into the image (openssh-server and dropbear do this). + "--include=tinysshd rsync", + "--essential-hook=install -dm700 $1/root/.ssh", + '--essential-hook=echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIapAZ0E0353DaY6xBnasvu/DOvdWdKQ6RQURwq4l6Wu twb@cyber.com.au (Trent W. Buck)" >$1/root/.ssh/authorized_keys', + # Bare minimum to let me log in locally. + # DO NOT use this on production builds! + "--essential-hook=chroot $1 passwd --delete root", + # Configure language (not needed to boot). + # Racism saves a **LOT** of space -- something like 2GB for Debian Live images. + # FIXME: use live-config instead? + "--include=locales localepurge", + f"--essential-hook=echo locales locales/default_environment_locale select {args.locale} | chroot $1 debconf-set-selections", + f"--essential-hook=echo locales locales/locales_to_be_generated multiselect {args.locale} UTF-8 | chroot $1 debconf-set-selections", + # FIXME: https://bugs.debian.org/603700 + "--customize-hook=chroot $1 sed -i /etc/locale.nopurge -e 's/^USE_DPKG/#ARGH#&/'", + "--customize-hook=chroot $1 localepurge", + "--customize-hook=chroot $1 sed -i /etc/locale.nopurge -e 's/^#ARGH#//'", + # Removing documentation also saves a LOT of space. + "--dpkgopt=path-exclude=/usr/share/doc/*", + "--dpkgopt=path-exclude=/usr/share/info/*", + "--dpkgopt=path-exclude=/usr/share/man/*", + "--dpkgopt=path-exclude=/usr/share/omf/*", + "--dpkgopt=path-exclude=/usr/share/help/*", + "--dpkgopt=path-exclude=/usr/share/gnome/help/*", + # Configure timezone (not needed to boot)` + # FIXME: use live-config instead? + "--include=tzdata", + f"--essential-hook=echo tzdata tzdata/Areas select {args.timezone[0]} | chroot $1 debconf-set-selections", + f"--essential-hook=echo tzdata tzdata/Zones/{args.timezone[0]} select {args.timezone[1]} | chroot $1 debconf-set-selections", + # Do the **BARE MINIMUM** to make a USB key that can boot on X86_64 UEFI. + # We use mtools so we do not ever need root privileges. + # We can't use mkfs.vfat, as that needs kpartx or losetup (i.e. root). + # We can't use mkfs.udf, as that needs mount (i.e. root). + # We can't use "refind-install --usedefault" as that runs mount(8) (i.e. root). + # We don't use genisoimage because + # 1) ISO9660 must die; + # 2) incomplete UDF 1.5+ support; + # 3) resulting filesystem can't be tweaked after flashing (e.g. debian-live/site.dir/etc/systemd/network/up.network). + # + # We use refind because 1) I hate grub; and 2) I like refind. + # If you want aarch64 or ia32 you need to install their BOOTxxx.EFI files. + # If you want kernel+initrd on something other than FAT, you need refind/drivers_xxx/xxx_xxx.EFI. + # + # FIXME: with qemu in UEFI mode (OVMF), I get dumped into startup.nsh (UEFI REPL). + # From there, I can manually type in "FS0:\EFI\BOOT\BOOTX64.EFI" to start refind, tho. + # So WTF is its problem? Does it not support fallback bootloader? + "--include=refind parted mtools", + "--essential-hook=echo refind refind/install_to_esp boolean false | chroot $1 debconf-set-selections", + "--customize-hook=echo refind refind/install_to_esp boolean true | chroot $1 debconf-set-selections", + "--customize-hook=chroot $1 mkdir -p /boot/USB /boot/EFI/BOOT", + "--customize-hook=chroot $1 cp /usr/share/refind/refind/refind_x64.efi /boot/EFI/BOOT/BOOTX64.EFI", + "--customize-hook=chroot $1 cp /usr/share/refind/refind/refind.conf-sample /boot/EFI/BOOT/refind.conf", + f"--customize-hook=chroot $1 truncate --size={filesystem_img_size} /boot/USB/filesystem.img", + f"--customize-hook=chroot $1 parted --script --align=optimal /boot/USB/filesystem.img mklabel gpt mkpart {esp_label} {esp_offset}b 100% set 1 esp on", + f"--customize-hook=chroot $1 mformat -i /boot/USB/filesystem.img@@{esp_offset} -F -v {esp_label}", + f"--customize-hook=chroot $1 mmd -i /boot/USB/filesystem.img@@{esp_offset} ::{live_media_path}", + f"""--customize-hook=echo '"Boot with default options" "boot=live live-media-path={live_media_path}"' >$1/boot/refind_linux.conf""", + f"""--customize-hook=chroot $1 find /boot/ -xdev -mindepth 1 -maxdepth 1 -not -name filesystem.img -not -name USB -exec mcopy -vsbpm -i /boot/USB/filesystem.img@@{esp_offset} {{}} :: ';'""", + # FIXME: copy-out doesn't handle sparseness, so is REALLY slow (about 50 seconds). + # Therefore instead leave it in the squashfs, and extract it later. + # f'--customize-hook=copy-out /boot/USB/filesystem.img /tmp/', + # f'--customize-hook=chroot $1 rm /boot/USB/filesystem.img', + "sid", + td / "filesystem.squashfs", + ] + ) + with args.output_file.open("wb") as f: + subprocess.check_call( + ["rdsquashfs", "--cat=boot/USB/filesystem.img", td / "filesystem.squashfs"], + stdout=f, + ) + subprocess.check_call( + [ + "mcopy", + "-i", + f"{args.output_file}@@{esp_offset}", + td / "filesystem.squashfs", + f"::{live_media_path}/filesystem.squashfs", + ] + )