From a31d81eb8263e0df9af62af9064d2dea5ce0152b Mon Sep 17 00:00:00 2001 From: Yaron Bar Date: Wed, 2 Sep 2026 11:43:05 +0300 Subject: [PATCH 1/3] fix: match full RHEL and Rocky product names in OS normalization NormalizeOSImageName matched the literal "RHEL", but Kubernetes reports osImage as "Red Hat Enterprise Linux 9.7 (Plow)", so every RHEL node normalized to "unknown-os" and its builder containers were named after it. Rocky had the same gap: "rocky\s*(\d+)" never matched "Rocky Linux 8.10". Widen both patterns and add a table test covering the spellings Kubernetes actually emits. Co-Authored-By: Claude Opus 5 (1M context) --- internal/drivers/drivers.go | 13 +++++++------ internal/drivers/drivers_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/internal/drivers/drivers.go b/internal/drivers/drivers.go index e9a2d51f4..2a014d8bd 100644 --- a/internal/drivers/drivers.go +++ b/internal/drivers/drivers.go @@ -13,17 +13,18 @@ import ( var ( ubuntuRe = regexp.MustCompile(`(?i)ubuntu\s+(\d+)\.(\d+)`) - rhelRe = regexp.MustCompile(`(?i)rhel\s*(\d+)`) - rockyRe = regexp.MustCompile(`(?i)rocky\s*(\d+)`) + rhelRe = regexp.MustCompile(`(?i)(?:rhel|red\s+hat\s+enterprise\s+linux)\s*(\d+)`) + rockyRe = regexp.MustCompile(`(?i)rocky(?:\s+linux)?\s*(\d+)`) ) // NormalizeOSImageName converts OS image names into short, DNS-1123 compliant canonical IDs. // Examples: // -// "Ubuntu 22.04.5 LTS" -> "ubuntu-22-04" -// "Ubuntu 24.04.3 LTS" -> "ubuntu-24" -// "RHEL 9.4" -> "rhel09" -// "Rocky Linux 8.10" -> "rocky08" +// "Ubuntu 22.04.5 LTS" -> "ubuntu-22-04" +// "Ubuntu 24.04.3 LTS" -> "ubuntu-24" +// "RHEL 9.4" -> "rhel09" +// "Red Hat Enterprise Linux 9.7 (Plow)" -> "rhel09" +// "Rocky Linux 8.10" -> "rocky08" func NormalizeOSImageName(input string) string { s := strings.TrimSpace(input) diff --git a/internal/drivers/drivers_test.go b/internal/drivers/drivers_test.go index 715a6f0b7..43eae1736 100644 --- a/internal/drivers/drivers_test.go +++ b/internal/drivers/drivers_test.go @@ -37,6 +37,33 @@ var _ = AfterSuite(func() { } }) +func TestNormalizeOSImageName(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + {"Ubuntu 22.04.5 LTS", "Ubuntu 22.04.5 LTS", "ubuntu-22-04"}, + {"Ubuntu 24.04.3 LTS", "Ubuntu 24.04.3 LTS", "ubuntu-24"}, + {"Ubuntu 24.04.4 LTS", "Ubuntu 24.04.4 LTS", "ubuntu-24"}, + {"RHEL 9.4", "RHEL 9.4", "rhel09"}, + {"Red Hat Enterprise Linux 9.7 (Plow)", "Red Hat Enterprise Linux 9.7 (Plow)", "rhel09"}, + {"Red Hat Enterprise Linux 8.10", "Red Hat Enterprise Linux 8.10", "rhel08"}, + {"Rocky Linux 8.10", "Rocky Linux 8.10", "rocky08"}, + {"empty string", "", "unknown-os"}, + {"Some Weird Distro 1.2", "Some Weird Distro 1.2", "unknown-os"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NormalizeOSImageName(tt.input) + if got != tt.want { + t.Errorf("NormalizeOSImageName(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + var _ = Describe("Driver Image Selection", func() { BeforeEach(func() { From f6d58f8488622098673b55cc553266678de1af11 Mon Sep 17 00:00:00 2001 From: Yaron Bar Date: Wed, 2 Sep 2026 11:51:02 +0300 Subject: [PATCH 2/3] fix: stage the cluster image weka CLI for drivers-builder The copy-cli init container staged whatever `weka` resolved to in the builder image, so the driver extraction ran an old CLI pinned in builder-ubuntu*-v1 rather than the one shipped with the cluster image being built. Pick the CLI source via weka_get_copy_local_driver_files, the same flag the loader already keys on, and route both callers through one helper so the two cannot drift apart. Co-Authored-By: Claude Opus 5 (1M context) --- .../wekacontainer/funcs_pod_ensure.go | 7 +++++- internal/drivers/drivers.go | 22 ++++++++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/internal/controllers/wekacontainer/funcs_pod_ensure.go b/internal/controllers/wekacontainer/funcs_pod_ensure.go index 46171b58f..cbd053562 100644 --- a/internal/controllers/wekacontainer/funcs_pod_ensure.go +++ b/internal/controllers/wekacontainer/funcs_pod_ensure.go @@ -123,9 +123,14 @@ func (r *containerReconcilerLoop) ensurePod(ctx context.Context) error { image = drivers.GetBuilderImageForNode(node) } + builderFlags, ffErr := r.GetFeatureFlags(ctx) + if ffErr != nil { + return errors.Wrap(ffErr, "failed to get feature flags for drivers-builder") + } + payloadBytes, _ := json.Marshal(map[string]string{ //nolint:errcheck // error return value intentionally not checked "targetImage": container.Spec.Image, - "cliImage": image, + "cliImage": drivers.GetBuilderCliImage(builderFlags, container.Spec.Image, image), }) container.Spec.Instructions = &weka.Instructions{ Type: weka.InstructionCopyWekaFilesToDriverLoader, diff --git a/internal/drivers/drivers.go b/internal/drivers/drivers.go index 2a014d8bd..5fec24090 100644 --- a/internal/drivers/drivers.go +++ b/internal/drivers/drivers.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/weka/weka-operator/internal/config" + "github.com/weka/weka-operator/internal/pkg/domain" "github.com/weka/weka-operator/internal/services" v1 "k8s.io/api/core/v1" ) @@ -66,13 +67,22 @@ func GetBuilderImageForNode(node *v1.Node) string { func GetLoaderImageForNode(ctx context.Context, node *v1.Node, image string) string { flags, err := services.GetFeatureFlags(ctx, image) - if err == nil && flags != nil { + if err != nil { + flags = nil + } + + return GetBuilderCliImage(flags, image, GetBuilderImageForNode(node)) +} + +// GetBuilderCliImage picks the image that supplies the weka CLI staged for the +// drivers-builder init containers. The extraction step bind-mounts over /opt/weka, +// where the cluster image keeps its CLI, so a CLI is always staged out of band; +// prefer the cluster image's own CLI when it can copy driver files itself. +func GetBuilderCliImage(flags *domain.FeatureFlags, clusterImage, builderImage string) string { + if flags != nil && flags.WekaGetCopyLocalDriverFiles { // innovation cli --kernel-build-id etc. - if flags.WekaGetCopyLocalDriverFiles { - return image - } + return clusterImage } - // else - can use the builder image that has "innovation" cli - return GetBuilderImageForNode(node) + return builderImage } From ded1b12a84dc77f3443851036e16f4ce9068cd11 Mon Sep 17 00:00:00 2001 From: Yaron Bar Date: Wed, 2 Sep 2026 13:57:07 +0300 Subject: [PATCH 3/3] fix: run the staged weka CLI in the drivers-builder container copy-cli resolved the CLI through PATH, which in the cluster image points at the release binary rather than the wekactl one, and staging it was inert anyway: the builder invokes a bare `weka`, and the operator wrapper at /usr/local/bin/weka execs /usr/bin/weka, so the staged copy was never reached. Glob the wekactl binary by arch, make it executable (the source is 0644), and shadow /usr/bin/weka with a subPath mount so the container runs it without weka_runtime.py having to name the path. Fall back to the previous PATH resolution for images with no wekactl-*, and fail loudly if neither exists. Co-Authored-By: Claude Opus 5 (1M context) --- .../controllers/resources/init_containers.go | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/internal/controllers/resources/init_containers.go b/internal/controllers/resources/init_containers.go index 31d433469..30484830a 100644 --- a/internal/controllers/resources/init_containers.go +++ b/internal/controllers/resources/init_containers.go @@ -97,10 +97,26 @@ func (f *PodFactory) copyWekaVersionToContainer(pod *v1.Pod) { Command: []string{"sh", "-c"}, Args: []string{ ` - # Copy the actual binary file that command resolves to (follows symlinks) - mkdir -p /shared-weka-version/cli && - cp -a -- "$(readlink -f -- "$(command -v weka)")" /shared-weka-version/cli/weka && - echo "copy-cli init container done" + # Stage the weka CLI outside /opt/weka: the extraction step below bind-mounts + # over /opt/weka, where the CLI lives, and would otherwise hide it. + mkdir -p /shared-weka-version/cli || exit 1 + # the wekactl filename carries a per-image hash and the machine arch + ARCH=$(uname -m) + CLI=$(ls -1 /opt/weka/dist/image/wekactl-*-"$ARCH" 2>/dev/null | head -1) + if [ -z "$CLI" ]; then + # older images ship only the weka binary; resolve the path we shadow + # rather than PATH, so a wrapper script can never be staged onto the + # very path it delegates to + CLI=$(readlink -f -- /usr/bin/weka) + fi + if [ -z "$CLI" ] || [ ! -f "$CLI" ]; then + echo "ERROR: no weka CLI found to stage" >&2 + exit 1 + fi + cp -- "$CLI" /shared-weka-version/cli/weka || exit 1 + # the source is not executable in the image, so set the bit explicitly + chmod 0755 /shared-weka-version/cli/weka || exit 1 + echo "copy-cli init container done: staged $CLI" `, }, VolumeMounts: []v1.VolumeMount{ @@ -148,6 +164,13 @@ func (f *PodFactory) copyWekaVersionToContainer(pod *v1.Pod) { Name: sharedVolumeName, MountPath: sharedVolumeMountPath, }) + // the wrapper at /usr/local/bin/weka execs /usr/bin/weka, so shadowing that path makes the + // container use the CLI staged from the cluster image instead of its own older one + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, v1.VolumeMount{ + Name: sharedVolumeName, + MountPath: "/usr/bin/weka", + SubPath: "cli/weka", + }) pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ Name: sharedVolumeName, VolumeSource: v1.VolumeSource{