feat: rewrite weka_runtime.py in Go (phase 2 - logic implementation per mode)

.ainav/index.md

@@ -30,7 +30,23 @@ internal/consts/             # Shared constants, annotations, resources
 internal/rest_api/           # Optional REST API server (cluster CRUD)
 internal/node_agent/         # Per-node agent server
 pkg/weka-k8s-api/           # CRD type definitions
-charts/weka-operator/        # Helm chart and Python runtime
+charts/weka-operator/        # Helm chart and Python runtime (weka_runtime.py)
+internal/runtime/            # Go rewrite of weka_runtime.py (pod-side process)
+  config/                   # Config loading from env vars
+  modes/                    # Per-mode entry points (compute, drive, client, ...)
+  agent/                    # Weka agent configuration and driver readiness
+  cpuaffinity/              # CPU core selection and affinity management
+  generation/               # Runtime generation file (takeover detection)
+  network/                  # Management IP discovery, net device reconciliation
+  persistency/              # Persistent storage bind-mount setup
+  ports/                    # Client port allocation
+  resources/                # Wait and load resources.json from operator
+  shutdown/                 # Shutdown instruction polling, drive release
+  syslog/                   # Syslog daemon (syslog-ng or go-syslog)
+  weka/                     # Weka container lifecycle (ensure, traces, features)
+  wekadrive/                # Drive discovery and VFIO validation
+  daemon/                   # Process supervisor
+  cmdutil/                  # Command execution helpers
 ```
 
 ## Key Areas by Functionality

.typos.toml

@@ -11,3 +11,6 @@ CROS = "CROS"
 ba = "ba"
 # umounted follows the umount(8) command naming convention
 umounted = "umounted"
+# SER is a serial-number prefix used in test fixtures (e.g. SER1, SER2 in
+# internal/runtime/wekadrive/sign_test.go); not a misspelling of "SET"
+SER = "SER"

Makefile

@@ -29,8 +29,10 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)
 CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
 ifneq ($(findstring release/,$(CURRENT_BRANCH)),)
 REPO ?= quay.io/weka.io/weka-operator
+REPO_POD_RUNTIME ?= quay.io/weka.io/weka-pod-runtime
 else
 REPO ?= quay.io/weka.io/weka-operator-dev
+REPO_POD_RUNTIME ?= quay.io/weka.io/weka-pod-runtime-dev
 endif
 VERSION ?= dev-$(shell git rev-parse --short HEAD)
 DEPLOY_CONTROLLER ?= true
@@ -284,7 +286,7 @@ endif
 
 .PHONY: install
 install: manifests ## Install CRDs into the K8s cluster specified in ~/.kube/config.
-	if [ "$(SKIP_CRD_INSTALL)" = "false" ]; then kubectl apply --server-side -f charts/weka-operator/crds; fi
+	if [ "$(SKIP_CRD_INSTALL)" = "false" ]; then kubectl apply --server-side --force-conflicts -f charts/weka-operator/crds; fi
 
 .PHONY: uninstall
 uninstall: manifests ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
@@ -393,3 +395,12 @@ OPERATOR_SDK = $(shell which operator-sdk)
 endif
 endif
 
+.PHONY: build-pod-runtime
+build-pod-runtime:
+	go build -o bin/weka-pod-runtime ./cmd/weka-pod-runtime/main.go
+
+.PHONY: docker-push-pod-runtime
+docker-push-pod-runtime:
+	docker buildx build --platform linux/amd64,linux/arm64 \
+	  -t $(REPO_POD_RUNTIME):$(VERSION) --push -f pod-runtime.Dockerfile .
+

cmd/weka-pod-runtime/main.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/weka/go-weka-observability/instrumentation"
 	obslogger "github.com/weka/go-weka-observability/logger"
@@ -25,20 +26,35 @@ func main() {
 		// observability is non-critical, log and continue
 		logger.Info("failed to set up OTel SDK", "err", err)
 	}
-	if err := modes.Run(ctx, cfg); err != nil {
-		logger.Error(err, "mode failed", "mode", cfg.Mode)
-		if shutdown != nil {
-			if shutdownErr := shutdown(ctx); shutdownErr != nil {
-				logger.Info("failed to shutdown OTel", "err", shutdownErr)
-			}
-		}
-		stop()
-		os.Exit(1)
-	}
+
+	modeErr := modes.Run(ctx, cfg)
+
 	if shutdown != nil {
 		if shutdownErr := shutdown(ctx); shutdownErr != nil {
 			logger.Info("failed to shutdown OTel", "err", shutdownErr)
 		}
 	}
 	stop()
+
+	// Mirror Python debug-sleep at weka_runtime.py:4655-4661:
+	//   debug_sleep = int(WEKA_OPERATOR_DEBUG_SLEEP or 3)
+	//   start = now; while now-start < debug_sleep: if /tmp/.cancel-debug-sleep: break; sleep(1)
+	// i.e. poll the cancel file once per second so an externally-created flag aborts the sleep.
+	debugSleep := cfg.DebugSleep
+	if debugSleep == 0 {
+		debugSleep = 3
+	}
+	logger.Info("debug sleep before exit", "seconds", debugSleep)
+	for i := 0; i < debugSleep; i++ {
+		if _, err := os.Stat("/tmp/.cancel-debug-sleep"); err == nil {
+			logger.Info("debug sleep cancelled by /tmp/.cancel-debug-sleep")
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+
+	if modeErr != nil {
+		logger.Error(modeErr, "mode failed", "mode", cfg.Mode)
+		os.Exit(1)
+	}
 }

internal/pkg/osinfo/osinfo.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"sync"
 )
 
 const (
@@ -30,9 +31,22 @@ func (n *NodeInfo) IsRhCos() bool  { return n.Os == OsNameRhCos }
 func (n *NodeInfo) IsCos() bool    { return n.Os == OsNameCos }
 func (n *NodeInfo) IsUbuntu() bool { return n.Os == OsNameUbuntu }
 
+var (
+	nodeInfoOnce   sync.Once
+	nodeInfoCached *NodeInfo
+	nodeInfoErr    error
+)
+
 // Load reads /hostside/etc/os-release and returns the detected NodeInfo.
-// This is the host-side path mounted into the pod.
+// Results are cached after the first call; the OS does not change during a pod's lifetime.
 func Load() (*NodeInfo, error) {
+	nodeInfoOnce.Do(func() {
+		nodeInfoCached, nodeInfoErr = load()
+	})
+	return nodeInfoCached, nodeInfoErr
+}
+
+func load() (*NodeInfo, error) {
 	raw, err := parseOsRelease("/hostside/etc/os-release")
 	if err != nil {
 		return nil, fmt.Errorf("reading os-release: %w", err)

internal/runtime/adhoc/force_resign_drives.go

@@ -34,6 +34,10 @@ func RunForceResignDrives(ctx context.Context, cfg *config.Config) error {
 		for _, serial := range payload.DeviceSerials {
 			p, err := blockdev.GetDevicePathBySerial(ctx, serial)
 			if err != nil {
+				// DELIBERATE DEVIATION from Python (weka_runtime.py:962): Python's
+				// force_resign_drives_by_serials appends None to device_paths when serial
+				// resolution fails, causing a downstream crash in sign_device_path_for_proxy.
+				// Go skips unresolvable serials instead, which is safer.  Do not revert.
 				logger.Info("force-resign-drives: failed to resolve serial to path, skipping", "serial", serial, "err", err.Error())
 				continue
 			}