Fix Marlin boot backend and filesystem defaults

CONTEXT.md

@@ -178,9 +178,9 @@
 filesystem with `podman cp` — never executing it, since Universal Blue images
 ship Rust uutils that `podman --entrypoint` cannot dispatch:
 
-- **bootupd present** → ostree backend, `xfs`, `--generic-image`
-- **systemd-boot, no bootupd** → composefs backend, `btrfs`, `--composefs-backend`
-- **neither** → ostree
+- **GRUB bootupd update payload present** → ostree backend, `xfs`, `--generic-image`
+- **systemd-boot, no GRUB update payload** → composefs backend, `ext4` when its formatter is present (otherwise `btrfs`), `--composefs-backend`
+- **neither** → legacy bootupd fallback; local probes reject images without either bootloader
 
 `--generic-image` is load-bearing rather than cosmetic. bootupd installs the
 bootloader to `EFI/<vendor>/` plus an efibootmgr NVRAM entry, and a fresh VM

cmd/corral-bootc/main.go

@@ -214,7 +214,7 @@ func main() {
 	create.Flags().StringVar(&sshKey, "ssh-key", "", "SSH public key (default: ~/.ssh/*.pub)")
 	create.Flags().StringVarP(&storageClass, "storage-class", "s", "", "StorageClass for the disk PVC (default: cluster preference)")
 	create.Flags().BoolVar(&resume, "resume", false, "Finish a build that completed after a previous `create` was interrupted")
-	create.Flags().StringVar(&filesystem, "filesystem", "", "Override the root filesystem a local build would pick from the image (xfs for ostree, btrfs for composefs)")
+	create.Flags().StringVar(&filesystem, "filesystem", "", "Override the root filesystem a local build would pick from the image (xfs for ostree, ext4 or btrfs for composefs)")
 	create.Flags().BoolVar(&useSudo, "sudo", false, "Run the local build's podman under sudo — bootc install needs root (off-cluster backends only)")
 
 	images := &cobra.Command{

cmd/create.go

@@ -687,8 +687,8 @@ func runLocalBootcCreate(name string) error {
 	// Backend-aware install (mirrors pkg/kubevirt/bootc.go's builder). ostree/
 	// server images ship bootupctl and install to xfs with --generic-image (so
 	// the disk boots the removable EFI path under plain QEMU). composefs images
-	// (Universal Blue / TunaOS desktops: no bootupctl, systemd-boot on the ESP)
-	// REQUIRE btrfs — their initramfs is btrfs-only, so installing them the
+	// (Universal Blue / TunaOS desktops: systemd-boot without GRUB payloads)
+	// require fs-verity (ext4 when available, otherwise btrfs). Installing the
 	// ostree/xfs way drops to dracut emergency mode at initrd-switch-root
 	// (tuna-os sailfin desktop Gate). They also need the real kernel/initrd
 	// re-extracted over bootc's EROFS-zeroed ESP copies, and the SSH key written
@@ -698,10 +698,14 @@ func runLocalBootcCreate(name string) error {
 		createBootc, "sh", "-c",
 		fmt.Sprintf(`set -e
 DISK=%s
-if command -v bootupctl >/dev/null 2>&1; then
+if ls /usr/lib/bootupd/updates/EFI/*/grub*.efi >/dev/null 2>&1 \
+  || { [ -f /usr/lib/bootupd/updates/EFI.json ] \
+    && find /usr/lib/efi/grub2 -type f -name 'grub*.efi' -print -quit 2>/dev/null | grep -q . \
+    && find /usr/lib/efi/shim -type f -name 'shim*.efi' -print -quit 2>/dev/null | grep -q .; }; then
   COMPOSEFS=0; FS=xfs; BACKEND=--generic-image
 elif [ -f /usr/lib/systemd/boot/efi/systemd-bootx64.efi ] || [ -f /usr/lib/systemd/boot/efi/systemd-bootaa64.efi ]; then
   COMPOSEFS=1; FS=btrfs; BACKEND=--composefs-backend
+  if command -v mkfs.ext4 >/dev/null 2>&1; then FS=ext4; fi
 else
   COMPOSEFS=0; FS=xfs; BACKEND=--generic-image
 fi

pkg/bootc/bootc_test.go

@@ -1,7 +1,10 @@
 package bootc
 
 import (
+	"archive/tar"
+	"bytes"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -169,7 +172,7 @@ func TestDetectBackend_FollowsTheClusterRule(t *testing.T) {
 		"bootupd present":          {[]string{"/usr/sbin/bootupctl"}, ostreeBackend},
 		"bootupd in /usr/bin":      {[]string{"/usr/bin/bootupctl"}, ostreeBackend},
 		"systemd-boot, no bootupd": {[]string{"/usr/lib/systemd/boot/efi/systemd-bootx64.efi"}, composefsBackend},
-		"neither marker":           {nil, ostreeBackend},
+		"Marlin ships both tools":  {[]string{"/usr/bin/bootupctl", "/usr/lib/systemd/boot/efi/systemd-bootx64.efi", "/usr/bin/mkfs.ext4"}, Backend{Kind: "composefs", Filesystem: "ext4", Flag: "--composefs-backend"}},
 	} {
 		t.Run(name, func(t *testing.T) {
 			fake := withFake(t)
@@ -201,6 +204,7 @@ func TestDetectBackend_NeverExecutesTheImage(t *testing.T) {
 	fake.AddPrefixResponse("podman create", "container123", nil)
 	fake.AddPrefixResponse("podman rm", "", nil)
 	fake.AddPrefixResponse("podman cp", "", errors.New("absent"))
+	fake.AddResponse("podman cp container123:/usr/bin/bootupctl -", "", nil)
 
 	if _, err := (LocalBuilder{}).DetectBackend("img"); err != nil {
 		t.Fatal(err)
@@ -531,6 +535,7 @@ func TestDetectBackend_GivesPodmanACommandToRecord(t *testing.T) {
 	fake.AddPrefixResponse("podman create", "container123", nil)
 	fake.AddPrefixResponse("podman rm", "", nil)
 	fake.AddPrefixResponse("podman cp", "", errors.New("absent"))
+	fake.AddResponse("podman cp container123:/usr/bin/bootupctl -", "", nil)
 
 	if _, err := (LocalBuilder{}).DetectBackend("ghcr.io/ublue-os/bluefin:stable"); err != nil {
 		t.Fatal(err)
@@ -553,3 +558,53 @@ func TestDetectBackend_GivesPodmanACommandToRecord(t *testing.T) {
 		t.Errorf("the placeholder should be an obviously fake path: %q", probeCommand[0])
 	}
 }
+
+// GRUB payloads win over systemd-boot binaries installed by the same systemd
+// package. Cover both bootupd layouts without requiring a container engine.
+func TestDetectBackend_GRUBPayloadWins(t *testing.T) {
+	for _, modern := range []bool{false, true} {
+		t.Run(fmt.Sprint(modern), func(t *testing.T) {
+			fake := withFake(t)
+			fake.AddPrefixResponse("podman create", "container123", nil)
+			fake.AddPrefixResponse("podman rm", "", nil)
+			fake.AddPrefixResponse("podman cp", "", errors.New("absent"))
+			for _, p := range []string{"/usr/lib/systemd/boot/efi/systemd-bootx64.efi", "/usr/bin/mkfs.xfs"} {
+				fake.AddResponse("podman cp container123:"+p+" -", "", nil)
+			}
+			addTar := func(dir, filename string) {
+				var b bytes.Buffer
+				tw := tar.NewWriter(&b)
+				if err := tw.WriteHeader(&tar.Header{Name: filename, Mode: 0644, Size: 1}); err != nil {
+					t.Fatal(err)
+				}
+				if _, err := tw.Write([]byte("x")); err != nil {
+					t.Fatal(err)
+				}
+				if err := tw.Close(); err != nil {
+					t.Fatal(err)
+				}
+				fake.AddResponse("podman cp container123:"+dir+" -", b.String(), nil)
+			}
+			if modern {
+				fake.AddResponse("podman cp container123:/usr/lib/bootupd/updates/EFI.json -", "", nil)
+				addTar("/usr/lib/efi/grub2", "grub2/2.14/grubaa64.efi")
+				addTar("/usr/lib/efi/shim", "shim/16/shimaa64.efi")
+			} else {
+				addTar("/usr/lib/bootupd/updates/EFI", "EFI/fedora/grubx64.efi")
+			}
+			got, err := (LocalBuilder{}).DetectBackend("img")
+			if err != nil || got != ostreeBackend {
+				t.Fatalf("got %+v, %v", got, err)
+			}
+		})
+	}
+}
+func TestDetectBackend_RejectsUnknownImage(t *testing.T) {
+	fake := withFake(t)
+	fake.AddPrefixResponse("podman create", "container123", nil)
+	fake.AddPrefixResponse("podman rm", "", nil)
+	fake.AddPrefixResponse("podman cp", "", errors.New("absent"))
+	if _, err := (LocalBuilder{}).DetectBackend("img"); err == nil {
+		t.Fatal("unknown image silently chose OSTree")
+	}
+}

pkg/bootc/local.go

@@ -1,9 +1,13 @@
 package bootc
 
 import (
+	"archive/tar"
+	"bytes"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -230,8 +234,8 @@ func firstLineOf(msg string) string {
 }
 
 // Backend is the bootc storage backend an image needs. It is a property of
-// the image, not a choice: composefs images ship a btrfs-only initramfs and
-// no bootupd, ostree images ship bootupd and want xfs.
+// the image, not a choice: GRUB update payloads identify OSTree; systemd-boot
+// without those payloads needs composefs and a filesystem with fs-verity.
 type Backend struct {
 	Kind       string // "ostree" or "composefs"
 	Filesystem string
@@ -247,8 +251,8 @@ var (
 // what pkg/kubevirt's cluster builder does — the rule was worked out there
 // (f63917e) and getting it wrong produces a disk that does not boot.
 //
-// Per the bootc docs the composefs-rs backend requires systemd-boot and no
-// bootupd; traditional ostree images ship bootupd.
+// GRUB update payloads identify traditional OSTree images. A bootupd
+// executable alone does not: Marlin also ships it with systemd-boot.
 //
 // The probe inspects the image *filesystem* with `podman cp` and never
 // executes anything in it. Universal Blue desktop images ship Rust uutils,
@@ -272,17 +276,48 @@ func (b LocalBuilder) DetectBackend(image string) (Backend, error) {
 		_, err := runner.Run(cp, cpArgs...)
 		return err == nil
 	}
-	switch {
-	case has("/usr/sbin/bootupctl"), has("/usr/bin/bootupctl"):
-		return ostreeBackend, nil
-	case has("/usr/lib/systemd/boot/efi/systemd-bootx64.efi"),
-		has("/usr/lib/systemd/boot/efi/systemd-bootaa64.efi"):
-		return composefsBackend, nil
-	default:
-		// Neither marker: treat as ostree, which is what the cluster builder
-		// does and the safer guess — xfs and a removable bootloader path.
-		return ostreeBackend, nil
+	// bootupctl alone is not a payload: Marlin ships the binary alongside
+	// systemd-boot, without GRUB update metadata. Inspect both bootupd layouts.
+	containsEFI := func(dir, prefix string) bool {
+		cp, args := b.podman("cp", container+":"+dir, "-")
+		data, err := runner.Run(cp, args...)
+		if err != nil {
+			return false
+		}
+		tr := tar.NewReader(bytes.NewReader(data))
+		for {
+			h, err := tr.Next()
+			if err == io.EOF {
+				return false
+			}
+			if err != nil {
+				return false
+			}
+			name := path.Base(h.Name)
+			if h.Typeflag == tar.TypeReg && strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ".efi") {
+				return true
+			}
+		}
+	}
+	payload := containsEFI("/usr/lib/bootupd/updates/EFI", "grub") ||
+		(has("/usr/lib/bootupd/updates/EFI.json") && containsEFI("/usr/lib/efi/grub2", "grub") && containsEFI("/usr/lib/efi/shim", "shim"))
+	backend := ostreeBackend
+	if !payload {
+		if has("/usr/lib/systemd/boot/efi/systemd-bootx64.efi") || has("/usr/lib/systemd/boot/efi/systemd-bootaa64.efi") {
+			backend = composefsBackend
+		} else if !has("/usr/sbin/bootupctl") && !has("/usr/bin/bootupctl") {
+			return Backend{}, fmt.Errorf("image has no supported bootloader payload")
+		}
+	}
+	// Ext4 provides fs-verity for sealed roots; keep the existing btrfs
+	// default when the image only supplies that formatter.
+	if backend.Kind == "composefs" && (has("/usr/sbin/mkfs.ext4") || has("/usr/bin/mkfs.ext4")) {
+		backend.Filesystem = "ext4"
+	}
+	if backend.Kind == "ostree" && !has("/usr/sbin/mkfs.xfs") && !has("/usr/bin/mkfs.xfs") && (has("/usr/sbin/mkfs.btrfs") || has("/usr/bin/mkfs.btrfs")) {
+		backend.Filesystem = "btrfs"
 	}
+	return backend, nil
 }
 
 // installArgs builds the podman invocation. Split out so a test can assert the

pkg/kubevirt/bootc.go

@@ -449,21 +449,14 @@ write_files:
       printf '[storage]\ndriver = "overlay"\nrunroot = "/run/containers/storage"\ngraphroot = "/var/lib/containers/storage"\n\n[storage.options.pull_options]\nenable_partial_images = "false"\n' > /etc/containers/storage.conf
       IMG=__IMAGE__
       podman pull "$IMG" || { echo CORRAL_BUILD_FAIL pull; echo CORRAL_BUILD_FAIL pull > /dev/ttyS0 2>/dev/null; sync; sleep 2; poweroff; exit 1; }
-      # Read the image to pick the bootc storage backend (per the bootc docs the
-      # composefs-rs backend requires systemd-boot and NO bootupd; traditional
-      # ostree images ship bootupd):
-      #   - bootupd present                       -> ostree backend  (xfs)
-      #   - else systemd-boot present, no bootupd -> composefs backend (btrfs)
-      #   - neither                               -> ostree backend  (xfs)
-      # Probe by inspecting the image FILESYSTEM with podman cp (no execution):
-      # UB desktop images ship Rust uutils, where /usr/bin/test is a symlink to a
-      # multicall binary that podman --entrypoint can't dispatch (argv[0] breaks),
-      # so any "run a binary in the image" probe misdetects them. The same CTR is
-      # reused for the composefs kernel/initrd extraction below. The backend also
-      # drives the root fs: composefs images ship a btrfs-only initramfs, ostree xfs.
-      CTR=$(podman create "$IMG")
+      # Inspect real GRUB update payloads, not just bootupctl: Marlin ships
+      # both executables but installs with systemd-boot/composefs. Never run
+      # image commands for this probe (foreign architecture/uutils).
+      CTR=$(podman create "$IMG" /corral-probe-does-not-execute) || { echo CORRAL_BUILD_FAIL probe; poweroff; exit 1; }
       imghas() { podman cp "$CTR:$1" - >/dev/null 2>&1; }
-      if imghas /usr/sbin/bootupctl || imghas /usr/bin/bootupctl; then
+      imgEFI() { podman cp "$CTR:$1" - 2>/dev/null | tar -tf - 2>/dev/null | grep -Eq "/$2[^/]*\\.efi$"; }
+      if imgEFI /usr/lib/bootupd/updates/EFI grub \
+        || { imghas /usr/lib/bootupd/updates/EFI.json && imgEFI /usr/lib/efi/grub2 grub && imgEFI /usr/lib/efi/shim shim; }; then
         BACKEND_KIND=ostree
       elif imghas /usr/lib/systemd/boot/efi/systemd-bootx64.efi \
         || imghas /usr/lib/systemd/boot/efi/systemd-bootaa64.efi; then
@@ -475,6 +468,7 @@ write_files:
       if [ "$BACKEND_KIND" = composefs ]; then
         # composefs backend installs systemd-boot to the ESP removable path itself.
         COMPOSEFS=1; FS=btrfs; BACKEND=--composefs-backend
+        if imghas /usr/sbin/mkfs.ext4 || imghas /usr/bin/mkfs.ext4; then FS=ext4; fi
       else
         # ostree backend installs the bootloader via bootupd, which by default
         # only writes EFI/<vendor>/ + an efibootmgr NVRAM entry. A fresh VM has