diff --git a/.ainav/index.md b/.ainav/index.md index b47cd8985..d9ee3323a 100644 --- a/.ainav/index.md +++ b/.ainav/index.md @@ -30,7 +30,23 @@ internal/consts/ # Shared constants, annotations, resources internal/rest_api/ # Optional REST API server (cluster CRUD) internal/node_agent/ # Per-node agent server pkg/weka-k8s-api/ # CRD type definitions -charts/weka-operator/ # Helm chart and Python runtime +charts/weka-operator/ # Helm chart and Python runtime (weka_runtime.py) +internal/runtime/ # Go rewrite of weka_runtime.py (pod-side process) + config/ # Config loading from env vars + modes/ # Per-mode entry points (compute, drive, client, ...) + agent/ # Weka agent configuration and driver readiness + cpuaffinity/ # CPU core selection and affinity management + generation/ # Runtime generation file (takeover detection) + network/ # Management IP discovery, net device reconciliation + persistency/ # Persistent storage bind-mount setup + ports/ # Client port allocation + resources/ # Wait and load resources.json from operator + shutdown/ # Shutdown instruction polling, drive release + syslog/ # Syslog daemon (syslog-ng or go-syslog) + weka/ # Weka container lifecycle (ensure, traces, features) + wekadrive/ # Drive discovery and VFIO validation + daemon/ # Process supervisor + cmdutil/ # Command execution helpers ``` ## Key Areas by Functionality diff --git a/.typos.toml b/.typos.toml index 6d5edaf47..efd6d0692 100644 --- a/.typos.toml +++ b/.typos.toml @@ -11,3 +11,6 @@ CROS = "CROS" ba = "ba" # umounted follows the umount(8) command naming convention umounted = "umounted" +# SER is a serial-number prefix used in test fixtures (e.g. SER1, SER2 in +# internal/runtime/wekadrive/sign_test.go); not a misspelling of "SET" +SER = "SER" diff --git a/Makefile b/Makefile index d5f7f773b..89e154f07 100644 --- a/Makefile +++ b/Makefile @@ -29,8 +29,10 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") ifneq ($(findstring release/,$(CURRENT_BRANCH)),) REPO ?= quay.io/weka.io/weka-operator +REPO_POD_RUNTIME ?= quay.io/weka.io/weka-pod-runtime else REPO ?= quay.io/weka.io/weka-operator-dev +REPO_POD_RUNTIME ?= quay.io/weka.io/weka-pod-runtime-dev endif VERSION ?= dev-$(shell git rev-parse --short HEAD) DEPLOY_CONTROLLER ?= true @@ -284,7 +286,7 @@ endif .PHONY: install install: manifests ## Install CRDs into the K8s cluster specified in ~/.kube/config. - if [ "$(SKIP_CRD_INSTALL)" = "false" ]; then kubectl apply --server-side -f charts/weka-operator/crds; fi + if [ "$(SKIP_CRD_INSTALL)" = "false" ]; then kubectl apply --server-side --force-conflicts -f charts/weka-operator/crds; fi .PHONY: uninstall uninstall: manifests ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. @@ -393,3 +395,12 @@ OPERATOR_SDK = $(shell which operator-sdk) endif endif +.PHONY: build-pod-runtime +build-pod-runtime: + go build -o bin/weka-pod-runtime ./cmd/weka-pod-runtime/main.go + +.PHONY: docker-push-pod-runtime +docker-push-pod-runtime: + docker buildx build --platform linux/amd64,linux/arm64 \ + -t $(REPO_POD_RUNTIME):$(VERSION) --push -f pod-runtime.Dockerfile . + diff --git a/cmd/weka-pod-runtime/main.go b/cmd/weka-pod-runtime/main.go index 1643a5491..2fa9457c9 100644 --- a/cmd/weka-pod-runtime/main.go +++ b/cmd/weka-pod-runtime/main.go @@ -5,6 +5,7 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/weka/go-weka-observability/instrumentation" obslogger "github.com/weka/go-weka-observability/logger" @@ -25,20 +26,35 @@ func main() { // observability is non-critical, log and continue logger.Info("failed to set up OTel SDK", "err", err) } - if err := modes.Run(ctx, cfg); err != nil { - logger.Error(err, "mode failed", "mode", cfg.Mode) - if shutdown != nil { - if shutdownErr := shutdown(ctx); shutdownErr != nil { - logger.Info("failed to shutdown OTel", "err", shutdownErr) - } - } - stop() - os.Exit(1) - } + + modeErr := modes.Run(ctx, cfg) + if shutdown != nil { if shutdownErr := shutdown(ctx); shutdownErr != nil { logger.Info("failed to shutdown OTel", "err", shutdownErr) } } stop() + + // Mirror Python debug-sleep at weka_runtime.py:4655-4661: + // debug_sleep = int(WEKA_OPERATOR_DEBUG_SLEEP or 3) + // start = now; while now-start < debug_sleep: if /tmp/.cancel-debug-sleep: break; sleep(1) + // i.e. poll the cancel file once per second so an externally-created flag aborts the sleep. + debugSleep := cfg.DebugSleep + if debugSleep == 0 { + debugSleep = 3 + } + logger.Info("debug sleep before exit", "seconds", debugSleep) + for i := 0; i < debugSleep; i++ { + if _, err := os.Stat("/tmp/.cancel-debug-sleep"); err == nil { + logger.Info("debug sleep cancelled by /tmp/.cancel-debug-sleep") + break + } + time.Sleep(1 * time.Second) + } + + if modeErr != nil { + logger.Error(modeErr, "mode failed", "mode", cfg.Mode) + os.Exit(1) + } } diff --git a/internal/pkg/osinfo/osinfo.go b/internal/pkg/osinfo/osinfo.go index 981ec87d7..41d69ffbf 100644 --- a/internal/pkg/osinfo/osinfo.go +++ b/internal/pkg/osinfo/osinfo.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "strings" + "sync" ) const ( @@ -30,9 +31,22 @@ func (n *NodeInfo) IsRhCos() bool { return n.Os == OsNameRhCos } func (n *NodeInfo) IsCos() bool { return n.Os == OsNameCos } func (n *NodeInfo) IsUbuntu() bool { return n.Os == OsNameUbuntu } +var ( + nodeInfoOnce sync.Once + nodeInfoCached *NodeInfo + nodeInfoErr error +) + // Load reads /hostside/etc/os-release and returns the detected NodeInfo. -// This is the host-side path mounted into the pod. +// Results are cached after the first call; the OS does not change during a pod's lifetime. func Load() (*NodeInfo, error) { + nodeInfoOnce.Do(func() { + nodeInfoCached, nodeInfoErr = load() + }) + return nodeInfoCached, nodeInfoErr +} + +func load() (*NodeInfo, error) { raw, err := parseOsRelease("/hostside/etc/os-release") if err != nil { return nil, fmt.Errorf("reading os-release: %w", err) diff --git a/internal/runtime/adhoc/force_resign_drives.go b/internal/runtime/adhoc/force_resign_drives.go index e302d63fa..939088449 100644 --- a/internal/runtime/adhoc/force_resign_drives.go +++ b/internal/runtime/adhoc/force_resign_drives.go @@ -34,6 +34,10 @@ func RunForceResignDrives(ctx context.Context, cfg *config.Config) error { for _, serial := range payload.DeviceSerials { p, err := blockdev.GetDevicePathBySerial(ctx, serial) if err != nil { + // DELIBERATE DEVIATION from Python (weka_runtime.py:962): Python's + // force_resign_drives_by_serials appends None to device_paths when serial + // resolution fails, causing a downstream crash in sign_device_path_for_proxy. + // Go skips unresolvable serials instead, which is safer. Do not revert. logger.Info("force-resign-drives: failed to resolve serial to path, skipping", "serial", serial, "err", err.Error()) continue } diff --git a/internal/runtime/agent/agent.go b/internal/runtime/agent/agent.go new file mode 100644 index 000000000..99750f969 --- /dev/null +++ b/internal/runtime/agent/agent.go @@ -0,0 +1,296 @@ +// Package agent configures and manages the weka-agent process. +// Mirrors configure_agent, get_agent_cmd, await_agent, ensure_drivers, override_dependencies_flag +// at weka_runtime.py:1208–3128. +package agent + +import ( + "context" + "fmt" + "os" + "os/exec" + "time" + + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/pkg/osinfo" + "github.com/weka/weka-operator/internal/runtime/cmdutil" + "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/drivers" +) + +// Configure patches /etc/wekaio/service.conf and writes /etc/wekaio/service.json. +// handleDrivers=false means agent should NOT handle drivers (compute/drive/client pass false). +// Mirrors Python configure_agent() at weka_runtime.py:2924. +func Configure(ctx context.Context, cfg *config.Config, handleDrivers bool) error { + _, logger := instrumentation.CreateLogSpan(ctx, "agent.Configure") + defer logger.End() + + ignoreDriverFlag := "true" + if handleDrivers { + ignoreDriverFlag = "false" + } + + expandConditionMounts := "" + if cfg.Mode == "s3" || cfg.Mode == "envoy" { + expandConditionMounts = ",envoy-data" + } + + skipEnvoySetup := "" + if cfg.Mode == "s3" { + skipEnvoySetup = "sed -i 's/skip_envoy_setup=.*/skip_envoy_setup=true/g' /etc/wekaio/service.conf || true" + } + + // M5: Envoy agent env vars. + // Mirrors Python configure_agent() at weka_runtime.py:2934-2936: + // if MODE == "envoy": + // env_vars['RESTART_EPOCH_WANTED'] = str(int(os.environ.get("envoy_restart_epoch", time.time()))) + // env_vars['BASE_ID'] = PORT + envoyEnvExports := "" + if cfg.Mode == "envoy" { + restartEpoch := os.Getenv("envoy_restart_epoch") + if restartEpoch == "" { + restartEpoch = fmt.Sprintf("%d", time.Now().Unix()) + } + envoyEnvExports = fmt.Sprintf("export RESTART_EPOCH_WANTED=%s\nexport BASE_ID=%d\n", + restartEpoch, cfg.Port) + } + + script := fmt.Sprintf(`%s +CONFFILE="/etc/wekaio/service.conf" +PATTERN="skip_driver_install" + +# Remove trailing skip_driver_install line if present +if tail -n 1 "$CONFFILE" | grep -q "$PATTERN"; then + sed -i '$d' "$CONFFILE" +fi + +if ! grep -q "skip_driver_install" /etc/wekaio/service.conf; then + sed -i "/\[os\]/a skip_driver_install=%s" /etc/wekaio/service.conf + sed -i "/\[os\]/a ignore_driver_spec=%s" /etc/wekaio/service.conf +else + sed -i "s/skip_driver_install=.*/skip_driver_install=%s/g" /etc/wekaio/service.conf +fi +sed -i "s/ignore_driver_spec=.*/ignore_driver_spec=%s/g" /etc/wekaio/service.conf || true + +sed -i "s@external_mounts=.*@external_mounts=/opt/weka/external-mounts@g" /etc/wekaio/service.conf || true +sed -i "s@conditional_mounts_ids=.*@conditional_mounts_ids=kube-serviceaccount,etc-hosts,etc-resolv%s@g" /etc/wekaio/service.conf || true +%s +sed -i 's/cgroups_mode=auto/cgroups_mode=none/g' /etc/wekaio/service.conf || true +sed -i 's/override_core_pattern=true/override_core_pattern=false/g' /etc/wekaio/service.conf || true +sed -i "s/port=14100/port=%d/g" /etc/wekaio/service.conf || true +echo '{"agent": {"port": "%d"}}' > /etc/wekaio/service.json +`, + envoyEnvExports, + ignoreDriverFlag, ignoreDriverFlag, ignoreDriverFlag, ignoreDriverFlag, + expandConditionMounts, skipEnvoySetup, + cfg.AgentPort, cfg.AgentPort, + ) + + if err := cmdutil.Run(ctx, "sh", "-c", script); err != nil { + return fmt.Errorf("agent.Configure: %w", err) + } + + if cfg.MachineIdentifier != "" { + logger.Info("setting machine-id", "id", cfg.MachineIdentifier) + if err := os.MkdirAll("/opt/weka/data/agent", 0o755); err != nil { + return err + } + idPath := "/opt/weka/data/agent/machine-identifier" + if err := os.WriteFile(idPath, []byte(cfg.MachineIdentifier), 0o644); err != nil { + return fmt.Errorf("agent.Configure machine-identifier: %w", err) + } + } + + return nil +} + +// GetCmd returns the shell command string that starts the weka agent. +// Mirrors Python get_agent_cmd() at weka_runtime.py:3126. +func GetCmd(cfg *config.Config) string { + return fmt.Sprintf("exec /usr/bin/weka --agent --socket-name weka_agent_ud_socket_%d", cfg.AgentPort) +} + +// AwaitReady polls "weka local ps" until exit 0, with a timeout. +// Timeout is 60s normally, 1500s for global persistence mode. +// Mirrors Python await_agent() at weka_runtime.py:2075. +func AwaitReady(ctx context.Context, cfg *config.Config) error { + _, logger := instrumentation.CreateLogSpan(ctx, "agent.AwaitReady") + defer logger.End() + + timeout := 60 * time.Second + if cfg.WekaPersistenceMode == "global" { + timeout = 1500 * time.Second + } + + deadline := time.Now().Add(timeout) + for { + if err := cmdutil.Run(ctx, "weka", "local", "ps"); err == nil { + logger.Info("weka-agent started successfully") + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("agent.AwaitReady: agent did not come up in %s", timeout) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(300 * time.Millisecond): + } + logger.Info("waiting for weka-agent to start") + } +} + +// OverrideDependenciesFlag hard-codes the dependency success marker so the dist container can start. +// Mirrors Python override_dependencies_flag() at weka_runtime.py:2988. +func OverrideDependenciesFlag(ctx context.Context, cfg *config.Config) error { + _, logger := instrumentation.CreateLogSpan(ctx, "agent.OverrideDependenciesFlag") + defer logger.End() + + logger.Info("overriding dependencies flag") + + // M2: drive both branch and dep version from ResolveVersionParams. + // Mirrors Python weka_runtime.py:2988-3010: + // dep_version = version_params.get('dependencies', DEFAULT_DEPENDENCY_VERSION) + // if WEKA_DRIVERS_HANDLING: touch .../skip else: mkdir .../dep_version/$(uname -r)/ && touch .../successful + vp := drivers.ResolveVersionParams(cfg.ImageName) + if vp.WekaDriversHandling { + script := ` +mkdir -p /opt/weka/data/dependencies +touch /opt/weka/data/dependencies/skip +` + if err := cmdutil.Run(ctx, "sh", "-c", script); err != nil { + return fmt.Errorf("agent.OverrideDependenciesFlag (new): %w", err) + } + return nil + } + + depVersion := vp.EffectiveDependencies() + script := fmt.Sprintf(` +mkdir -p /opt/weka/data/dependencies/%s/$(uname -r)/ +touch /opt/weka/data/dependencies/%s/$(uname -r)/successful +`, depVersion, depVersion) + if err := cmdutil.Run(ctx, "sh", "-c", script); err != nil { + return fmt.Errorf("agent.OverrideDependenciesFlag (legacy): %w", err) + } + return nil +} + +// EnsureDrivers polls until all required kernel drivers are loaded. +// Mirrors Python ensure_drivers() at weka_runtime.py:1208. +func EnsureDrivers(ctx context.Context, cfg *config.Config) error { + _, logger := instrumentation.CreateLogSpan(ctx, "agent.EnsureDrivers") + defer logger.End() + + logger.Info("waiting for drivers", "mode", cfg.Mode) + + // Client / s3 / nfs: use "weka driver ready" command (new driver mode). + if !isLegacyDriverMode(cfg) && isClientLikeMode(cfg.Mode) { + // M6: read version from release spec (as Python's get_weka_version() does), + // instead of shelling out to "weka version | grep '*' | awk ..." which requires + // the agent to already be running. + // Mirrors Python ensure_drivers() at weka_runtime.py:1217-1221: + // version = await get_weka_version() + // run_command(f"weka driver ready --without-agent --version {version}") + wekaVersion, err := drivers.GetWekaVersion() + if err != nil { + return fmt.Errorf("EnsureDrivers: get weka version: %w", err) + } + if err := cmdutil.PollUntil(ctx, 1*time.Second, func() bool { + if err := cmdutil.Run(ctx, "weka", "driver", "ready", "--without-agent", "--version", wekaVersion); err == nil { + return true + } + logger.Warn("drivers not ready, waiting") + if e := writeDriverLog("weka-drivers-loading"); e != nil { + logger.Warn("failed to write driver status log", "err", e) + } + return false + }); err != nil { + return err + } + if err := writeDriverLog(""); err != nil { + return fmt.Errorf("EnsureDrivers: clearing driver status log: %w", err) + } + logger.Info("all drivers loaded successfully") + return nil + } + + // Compute / drive: poll lsmod for each driver. + driverModules := []string{"wekafsio", "wekafsgw", "mpin_user"} + + nodeInfo, err := osinfo.Load() + isCOS := err == nil && nodeInfo.IsCos() + if !isCOS { + driverModules = append(driverModules, "igb_uio") + if !skipUIOPCIGeneric(cfg) { + driverModules = append(driverModules, "uio_pci_generic") + } + } + + for _, driver := range driverModules { + if err := cmdutil.PollUntil(ctx, 1*time.Second, func() bool { + if err := cmdutil.Run(ctx, "sh", "-c", fmt.Sprintf("lsmod | grep -w %s", driver)); err == nil { + return true + } + logger.Info("driver not loaded, waiting", "driver", driver) + if e := writeDriverLog(driver); e != nil { + logger.Warn("failed to write driver status log", "err", e) + } + return false + }); err != nil { + return err + } + } + + if err := writeDriverLog(""); err != nil { + return fmt.Errorf("EnsureDrivers: clearing driver status log: %w", err) + } + logger.Info("all drivers loaded successfully") + return nil +} + +// ---- helpers ---------------------------------------------------------------- + +// isLegacyDriverMode returns true when the old lsmod-based driver check should be used. +// Python: is_legacy_driver_cmd() at weka_runtime.py:3215 — checks if "weka driver --help | grep pack" succeeds. +// In Go we run the same check. +func isLegacyDriverMode(cfg *config.Config) bool { + err := exec.Command("sh", "-c", "weka driver --help | grep pack").Run() //nolint:gosec // command args are operator-controlled, not user input + if err == nil { + return false // new mode: "pack" command available + } + return true // legacy mode +} + +// isClientLikeMode returns true for modes that use weka driver ready instead of lsmod. +func isClientLikeMode(mode string) bool { + switch mode { + case "client", "s3", "nfs": + return true + } + return false +} + +// skipUIOPCIGeneric returns true when uio_pci_generic should not be loaded. +// On COS we always skip it. +func skipUIOPCIGeneric(cfg *config.Config) bool { + // M1: mirror Python should_skip_uio_pci_generic() at weka_runtime.py:1416-1417: + // return version_params.get('uio_pci_generic') is False or should_skip_uio() + // where should_skip_uio() == is_google_cos(). The version-params branch is what makes + // all 4.3.x and DEFAULT_PARAMS images skip uio_pci_generic even on non-COS nodes. + if drivers.ResolveVersionParams(cfg.ImageName).ShouldSkipUioPciGeneric() { + return true + } + nodeInfo, err := osinfo.Load() + if err == nil && nodeInfo.IsCos() { + return true + } + return false +} + +// writeDriverLog writes the driver name to /tmp/weka-drivers.log atomically. +func writeDriverLog(content string) error { + const tmp = "/tmp/weka-drivers.log_tmp" + if err := os.WriteFile(tmp, []byte(content), 0o644); err != nil { + return err + } + return os.Rename(tmp, "/tmp/weka-drivers.log") +} diff --git a/internal/runtime/blockdev/blockdev.go b/internal/runtime/blockdev/blockdev.go index bf5e27290..96f0abd3c 100644 --- a/internal/runtime/blockdev/blockdev.go +++ b/internal/runtime/blockdev/blockdev.go @@ -45,6 +45,27 @@ func (d *lsblkDevice) hasMountpoint() bool { return false } +// disksFromLsblk parses raw lsblk JSON output and returns Disk entries for every +// device whose Type=="disk". Per-device I/O (serial, capacity) is NOT done here. +func disksFromLsblk(out []byte) ([]Disk, error) { + var parsed lsblkOutput + if err := json.Unmarshal(out, &parsed); err != nil { + return nil, fmt.Errorf("lsblk JSON parse: %w", err) + } + var disks []Disk + for i := range parsed.BlockDevices { + dev := &parsed.BlockDevices[i] + if dev.Type != "disk" { + continue + } + disks = append(disks, Disk{ + Path: dev.Name, + IsMounted: dev.hasMountpoint(), + }) + } + return disks, nil +} + // FindDisks enumerates all disk-type block devices visible from the host PID namespace. func FindDisks(ctx context.Context) ([]Disk, error) { out, err := cmdutil.Output(ctx, @@ -55,34 +76,29 @@ func FindDisks(ctx context.Context) ([]Disk, error) { return nil, fmt.Errorf("lsblk: %w", err) } - var parsed lsblkOutput - if err := json.Unmarshal(out, &parsed); err != nil { - return nil, fmt.Errorf("lsblk JSON parse: %w", err) + disks, err := disksFromLsblk(out) + if err != nil { + return nil, err } - var disks []Disk - for i := range parsed.BlockDevices { - dev := &parsed.BlockDevices[i] - if dev.Type != "disk" { - continue - } - isMounted := dev.hasMountpoint() - serialID, err := GetDeviceSerialID(ctx, dev.Name) + var result []Disk + for _, d := range disks { + serialID, err := GetDeviceSerialID(ctx, d.Path) if err != nil { serialID = "" } - devCap, err := GetCapacityGiB(ctx, dev.Name) + devCap, err := GetCapacityGiB(ctx, d.Path) if err != nil || devCap == 0 { continue } - disks = append(disks, Disk{ - Path: dev.Name, - IsMounted: isMounted, + result = append(result, Disk{ + Path: d.Path, + IsMounted: d.IsMounted, SerialID: serialID, CapacityGiB: devCap, }) } - return disks, nil + return result, nil } // GetDeviceSerialID returns the serial ID for the given block device path. @@ -123,12 +139,31 @@ func GetDeviceSerialID(ctx context.Context, devicePath string) (string, error) { if err != nil { return "", fmt.Errorf("reading udev data %s: %w", udevPath, err) } + return parseUdevSerial(data), nil +} + +// parseUdevSerial finds the first line containing the substring "ID_SERIAL=" and +// returns the text after the first "=" on that line, trimmed. This matches the +// Python original (grep 'ID_SERIAL=' | cut -d= -f2-). +// +// Bug fix: the previous Go code used strings.CutPrefix(line, "ID_SERIAL="), which +// only matched lines *starting* with "ID_SERIAL=". Real udev data lines are +// "E:"-prefixed (e.g. "E:ID_SERIAL=Samsung_SSD_970"), so the old code always +// returned "" for real SATA/SCSI drives. Note that "ID_SERIAL_SHORT=" does NOT +// contain the substring "ID_SERIAL=" so it is correctly excluded. +func parseUdevSerial(data []byte) string { for _, line := range strings.Split(string(data), "\n") { - if after, ok := strings.CutPrefix(line, "ID_SERIAL="); ok { - return after, nil + if !strings.Contains(line, "ID_SERIAL=") { + continue + } + // Return everything after the first "=" on this line, trimmed. + idx := strings.Index(line, "=") + if idx < 0 { + continue } + return strings.TrimSpace(line[idx+1:]) } - return "", nil + return "" } // GetCapacityGiB returns the capacity of a block device in GiB. diff --git a/internal/runtime/blockdev/blockdev_test.go b/internal/runtime/blockdev/blockdev_test.go new file mode 100644 index 000000000..c1ed0b77f --- /dev/null +++ b/internal/runtime/blockdev/blockdev_test.go @@ -0,0 +1,204 @@ +package blockdev + +import ( + "testing" +) + +// realLsblkJSON is a fixture derived from a live node. +// The node's ~37 type:"loop" entries are trimmed here to 3 representative ones +// (a snap mount, the "/opt/weka/logs" mount, and an unmounted one) since the +// loop-filtering behavior is fully exercised by the dedicated test case below. +// The substance is the 2 type:"disk" nvme devices (/dev/nvme4n1, /dev/nvme6n1): +// each has mountpoint null but a part2 -> md0 (raid1) child that is mounted at +// "/opt/weka/data/agent/sockets/...". +var realLsblkJSON = []byte(`{ + "blockdevices": [ + {"name":"/dev/loop0","type":"loop","mountpoint":"/snap/core20/1611","serial":null,"children":null}, + {"name":"/dev/loop35","type":"loop","mountpoint":"/opt/weka/logs","serial":null,"children":null}, + {"name":"/dev/loop36","type":"loop","mountpoint":null,"serial":null,"children":null}, + {"name":"/dev/nvme4n1","type":"disk","mountpoint":null,"serial":null,"children":[ + {"name":"/dev/nvme4n1p1","type":"part","mountpoint":null,"serial":null,"children":null}, + {"name":"/dev/nvme4n1p2","type":"part","mountpoint":null,"serial":null,"children":[ + {"name":"/dev/md0","type":"raid1","mountpoint":"/opt/weka/data/agent/sockets/000","serial":null,"children":null} + ]} + ]}, + {"name":"/dev/nvme6n1","type":"disk","mountpoint":null,"serial":null,"children":[ + {"name":"/dev/nvme6n1p1","type":"part","mountpoint":null,"serial":null,"children":null}, + {"name":"/dev/nvme6n1p2","type":"part","mountpoint":null,"serial":null,"children":[ + {"name":"/dev/md0","type":"raid1","mountpoint":"/opt/weka/data/agent/sockets/001","serial":null,"children":null} + ]} + ]} + ] +}`) + +func TestDisksFromLsblk(t *testing.T) { + tests := []struct { + name string + input []byte + wantErr bool + wantCount int + wantPaths []string + wantMounted []bool + }{ + { + name: "real fixture: two nvme disks, both mounted via raid child", + input: realLsblkJSON, + wantCount: 2, + wantPaths: []string{"/dev/nvme4n1", "/dev/nvme6n1"}, + // IsMounted==true because the recursion bubbles the raid child's mountpoint up + // two levels through part2 to the disk. + wantMounted: []bool{true, true}, + }, + { + name: "malformed JSON returns error", + input: []byte(`{not valid json`), + wantErr: true, + }, + { + name: "empty blockdevices returns empty slice", + input: []byte(`{"blockdevices":[]}`), + wantCount: 0, + }, + { + name: "only loop devices are filtered out", + input: []byte(`{"blockdevices":[ + {"name":"/dev/loop0","type":"loop","mountpoint":null,"children":null}, + {"name":"/dev/loop1","type":"loop","mountpoint":"/mnt/x","children":null} + ]}`), + wantCount: 0, + }, + { + name: "raid and part top-level types are filtered out", + input: []byte(`{"blockdevices":[ + {"name":"/dev/md0","type":"raid1","mountpoint":"/data","children":null}, + {"name":"/dev/sda1","type":"part","mountpoint":null,"children":null} + ]}`), + wantCount: 0, + }, + { + name: "unmounted disk", + input: []byte(`{"blockdevices":[ + {"name":"/dev/sda","type":"disk","mountpoint":null,"children":null} + ]}`), + wantCount: 1, + wantPaths: []string{"/dev/sda"}, + wantMounted: []bool{false}, + }, + { + name: "disk with direct mountpoint", + input: []byte(`{"blockdevices":[ + {"name":"/dev/sdb","type":"disk","mountpoint":"/data","children":null} + ]}`), + wantCount: 1, + wantPaths: []string{"/dev/sdb"}, + wantMounted: []bool{true}, + }, + { + name: "disk with mountpoint only in nested child", + input: []byte(`{"blockdevices":[ + {"name":"/dev/sdc","type":"disk","mountpoint":null,"children":[ + {"name":"/dev/sdc1","type":"part","mountpoint":null,"children":[ + {"name":"/dev/sdc1a","type":"part","mountpoint":"/boot","children":null} + ]} + ]} + ]}`), + wantCount: 1, + wantPaths: []string{"/dev/sdc"}, + wantMounted: []bool{true}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := disksFromLsblk(tt.input) + if tt.wantErr { + if err == nil { + t.Fatalf("disksFromLsblk(): want error, got nil") + } + return + } + if err != nil { + t.Fatalf("disksFromLsblk(): unexpected error: %v", err) + } + if len(got) != tt.wantCount { + t.Fatalf("disksFromLsblk(): got %d disks, want %d; disks=%v", len(got), tt.wantCount, got) + } + for i, d := range got { + if tt.wantPaths != nil && d.Path != tt.wantPaths[i] { + t.Errorf("disk[%d].Path = %q, want %q", i, d.Path, tt.wantPaths[i]) + } + if tt.wantMounted != nil && d.IsMounted != tt.wantMounted[i] { + t.Errorf("disk[%d].IsMounted = %v, want %v", i, d.IsMounted, tt.wantMounted[i]) + } + } + }) + } +} + +func TestParseUdevSerial(t *testing.T) { + tests := []struct { + name string + input []byte + want string + }{ + { + // Primary case: realistic E:-prefixed udev data line. + // OLD CODE (CutPrefix "ID_SERIAL=") would return "" here — the bug. + // NEW CODE (substring Contains + index after first "=") returns correct value. + name: "E:-prefixed line: Samsung_SSD_970", + input: []byte("E:ID_SERIAL=Samsung_SSD_970\nE:ID_SERIAL_SHORT=970\n"), + want: "Samsung_SSD_970", + }, + { + // ID_SERIAL_SHORT= must NOT match: it does not contain "ID_SERIAL=" as a substring + // because after "ID_SERIAL" comes "_SHORT=", not "=". + name: "only ID_SERIAL_SHORT present returns empty", + input: []byte("E:ID_SERIAL_SHORT=foo\n"), + want: "", + }, + { + name: "empty input returns empty", + input: []byte(""), + want: "", + }, + { + // First matching line wins when two ID_SERIAL= lines are present. + name: "first ID_SERIAL= line wins", + input: []byte("E:ID_SERIAL=First_Match\nE:ID_SERIAL=Second_Match\n"), + want: "First_Match", + }, + { + // A bare "ID_SERIAL=bare" (no E: prefix) still works — the match is by substring. + name: "bare line without prefix", + input: []byte("ID_SERIAL=bare\n"), + want: "bare", + }, + { + // Value is trimmed of surrounding whitespace. + name: "value is trimmed", + input: []byte("E:ID_SERIAL= spaced \n"), + want: "spaced", + }, + { + // A line with ID_SERIAL= but no value returns "". + name: "empty value after equals", + input: []byte("E:ID_SERIAL=\n"), + want: "", + }, + { + // Mixed: other fields before the serial line. + name: "serial line among other udev fields", + input: []byte("E:DEVTYPE=disk\nE:ID_PATH=pci-0000:00:17.0\nE:ID_SERIAL=WDC_WD40EFRX\nE:ID_MODEL=WDC\n"), + want: "WDC_WD40EFRX", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseUdevSerial(tt.input) + if got != tt.want { + t.Errorf("parseUdevSerial(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} diff --git a/internal/runtime/cmdutil/cmdutil.go b/internal/runtime/cmdutil/cmdutil.go index 11fc54b98..802c0c9e4 100644 --- a/internal/runtime/cmdutil/cmdutil.go +++ b/internal/runtime/cmdutil/cmdutil.go @@ -7,10 +7,26 @@ import ( "fmt" "os/exec" "strings" + "time" "github.com/weka/go-weka-observability/instrumentation" ) +// PollUntil calls fn every interval until it returns true, or ctx is done. +// fn should perform any per-iteration logging/side effects itself before returning false. +func PollUntil(ctx context.Context, interval time.Duration, fn func() bool) error { + for { + if fn() { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } +} + // Output runs the named command with args under ctx and returns its stdout. // Stderr is captured: logged as a warning when non-empty and appended to any error. func Output(ctx context.Context, name string, args ...string) ([]byte, error) { diff --git a/internal/runtime/config/config.go b/internal/runtime/config/config.go index 1086f52c7..1b7f0a02b 100644 --- a/internal/runtime/config/config.go +++ b/internal/runtime/config/config.go @@ -8,8 +8,8 @@ import ( "strconv" "strings" - "github.com/weka/weka-operator/internal/pkg/domain" v1alpha1 "github.com/weka/weka-k8s-api/api/v1alpha1" + "github.com/weka/weka-operator/internal/pkg/domain" ) var version = "dev" // set via -ldflags at build time @@ -25,6 +25,7 @@ type Config struct { FailureDomain string MachineIdentifier string Version string + Drives []string // populated at runtime from NodeResources // Resource allocation Cores []int @@ -91,63 +92,63 @@ func Load() *Config { cfg := &Config{ Version: version, } - cfg.Mode = os.Getenv("MODE") - cfg.Name = os.Getenv("NAME") - cfg.NodeName = os.Getenv("NODE_NAME") - cfg.PodName = os.Getenv("POD_NAME") - cfg.PodNamespace = os.Getenv("POD_NAMESPACE") - cfg.PodID = os.Getenv("POD_ID") - cfg.FailureDomain = os.Getenv("FAILURE_DOMAIN") + cfg.Mode = os.Getenv("MODE") + cfg.Name = os.Getenv("NAME") + cfg.NodeName = os.Getenv("NODE_NAME") + cfg.PodName = os.Getenv("POD_NAME") + cfg.PodNamespace = os.Getenv("POD_NAMESPACE") + cfg.PodID = os.Getenv("POD_ID") + cfg.FailureDomain = os.Getenv("FAILURE_DOMAIN") cfg.MachineIdentifier = os.Getenv("MACHINE_IDENTIFIER") - cfg.Cores = parseIntSlice(os.Getenv("CORES")) - cfg.CoreIDs = parseIntSlice(os.Getenv("CORE_IDS")) + cfg.Cores = parseIntSlice(os.Getenv("CORES")) + cfg.CoreIDs = parseIntSlice(os.Getenv("CORE_IDS")) cfg.CPUPolicy = os.Getenv("CPU_POLICY") - cfg.Memory = os.Getenv("MEMORY") + cfg.Memory = os.Getenv("MEMORY") cfg.DPDKBaseMemMB = parseInt(os.Getenv("DPDK_BASE_MEMORY_MB")) - cfg.NetworkDevice = os.Getenv("NETWORK_DEVICE") - cfg.Subnets = parseStringSlice(os.Getenv("SUBNETS")) - cfg.NetworkSelectors = parseStringSlice(os.Getenv("NETWORK_SELECTORS")) + cfg.NetworkDevice = os.Getenv("NETWORK_DEVICE") + cfg.Subnets = parseStringSlice(os.Getenv("SUBNETS")) + cfg.NetworkSelectors = parseStringSlice(os.Getenv("NETWORK_SELECTORS")) cfg.ManagementIPSelectors = parseStringSlice(os.Getenv("MANAGEMENT_IPS_SELECTORS")) - cfg.Port = parseInt(os.Getenv("PORT")) + cfg.Port = parseInt(os.Getenv("PORT")) cfg.AgentPort = parseInt(os.Getenv("AGENT_PORT")) - cfg.BasePort = parseInt(os.Getenv("BASE_PORT")) + cfg.BasePort = parseInt(os.Getenv("BASE_PORT")) cfg.PortRange = parseInt(os.Getenv("PORT_RANGE")) - cfg.JoinIPs = parseStringSlice(os.Getenv("JOIN_IPS")) - cfg.IsIPv6 = parseBool(os.Getenv("IS_IPV6")) - cfg.UDPMode = parseBool(os.Getenv("UDP_MODE")) + cfg.JoinIPs = parseStringSlice(os.Getenv("JOIN_IPS")) + cfg.IsIPv6 = parseBool(os.Getenv("IS_IPV6")) + cfg.UDPMode = parseBool(os.Getenv("UDP_MODE")) cfg.BindManagementAll = parseBool(os.Getenv("BIND_MANAGEMENT_ALL")) cfg.ManagementIP = os.Getenv("MANAGEMENT_IP") - cfg.NetGateway = os.Getenv("NET_GATEWAY") + cfg.NetGateway = os.Getenv("NET_GATEWAY") cfg.NvidiaVFSingleIP = parseBool(os.Getenv("NVIDIA_VF_SINGLE_IP")) - cfg.DistService = os.Getenv("DIST_SERVICE") - cfg.DriversBuildID = os.Getenv("DRIVERS_BUILD_ID") - cfg.DumperConfigMode = os.Getenv("DUMPER_CONFIG_MODE") - cfg.WekaContainerID = os.Getenv("WEKA_CONTAINER_ID") + cfg.DistService = os.Getenv("DIST_SERVICE") + cfg.DriversBuildID = os.Getenv("DRIVERS_BUILD_ID") + cfg.DumperConfigMode = os.Getenv("DUMPER_CONFIG_MODE") + cfg.WekaContainerID = os.Getenv("WEKA_CONTAINER_ID") cfg.WekaPersistenceMode = os.Getenv("WEKA_PERSISTENCE_MODE") - cfg.AutoRemoveTimeout = parseInt(os.Getenv("AUTO_REMOVE_TIMEOUT")) - cfg.PreRunScript = os.Getenv("PRE_RUN_SCRIPT") - cfg.ImageName = os.Getenv("IMAGE_NAME") - cfg.TargetImageName = os.Getenv("TARGET_IMAGE_NAME") - cfg.SyslogPackage = os.Getenv("SYSLOG_PACKAGE") + cfg.AutoRemoveTimeout = parseInt(os.Getenv("AUTO_REMOVE_TIMEOUT")) + cfg.PreRunScript = os.Getenv("PRE_RUN_SCRIPT") + cfg.ImageName = os.Getenv("IMAGE_NAME") + cfg.TargetImageName = os.Getenv("TARGET_IMAGE_NAME") + cfg.SyslogPackage = os.Getenv("SYSLOG_PACKAGE") - cfg.COSAllowHugepageConfig = parseBool(os.Getenv("WEKA_COS_ALLOW_HUGEPAGE_CONFIG")) + cfg.COSAllowHugepageConfig = parseBool(os.Getenv("WEKA_COS_ALLOW_HUGEPAGE_CONFIG")) cfg.COSAllowDisableDriverSign = parseBool(os.Getenv("WEKA_COS_ALLOW_DISABLE_DRIVER_SIGNING")) - cfg.COSGlobalHugepageSize = os.Getenv("WEKA_COS_GLOBAL_HUGEPAGE_SIZE") - cfg.COSGlobalHugepageCount = parseInt(os.Getenv("WEKA_COS_GLOBAL_HUGEPAGE_COUNT")) - - cfg.OtelEndpoint = os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") - cfg.OtelLogsEndpoint = os.Getenv("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT") - cfg.OtelHeaders = os.Getenv("OTEL_EXPORTER_OTLP_HEADERS") - cfg.OtelLogsHeaders = os.Getenv("OTEL_EXPORTER_OTLP_LOGS_HEADERS") - cfg.OtelServiceName = os.Getenv("OTEL_SERVICE_NAME") + cfg.COSGlobalHugepageSize = os.Getenv("WEKA_COS_GLOBAL_HUGEPAGE_SIZE") + cfg.COSGlobalHugepageCount = parseInt(os.Getenv("WEKA_COS_GLOBAL_HUGEPAGE_COUNT")) + + cfg.OtelEndpoint = os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + cfg.OtelLogsEndpoint = os.Getenv("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT") + cfg.OtelHeaders = os.Getenv("OTEL_EXPORTER_OTLP_HEADERS") + cfg.OtelLogsHeaders = os.Getenv("OTEL_EXPORTER_OTLP_LOGS_HEADERS") + cfg.OtelServiceName = os.Getenv("OTEL_SERVICE_NAME") cfg.OtelServiceVersion = os.Getenv("OTEL_SERVICE_VERSION") - cfg.OtelLogsEnabled = parseBool(os.Getenv("OTEL_LOGS_ENABLED")) + cfg.OtelLogsEnabled = parseBool(os.Getenv("OTEL_LOGS_ENABLED")) cfg.MaxTraceCapacityGB = parseInt(os.Getenv("MAX_TRACE_CAPACITY_GB")) - cfg.EnsureFreeSpaceGB = parseInt(os.Getenv("ENSURE_FREE_SPACE_GB")) - cfg.DebugSleep = parseInt(os.Getenv("WEKA_OPERATOR_DEBUG_SLEEP")) + cfg.EnsureFreeSpaceGB = parseInt(os.Getenv("ENSURE_FREE_SPACE_GB")) + cfg.DebugSleep = parseInt(os.Getenv("WEKA_OPERATOR_DEBUG_SLEEP")) if raw := os.Getenv("INSTRUCTIONS"); raw != "" { var inst v1alpha1.Instructions diff --git a/internal/runtime/config/config_test.go b/internal/runtime/config/config_test.go index 8257fca35..8f8ef17fb 100644 --- a/internal/runtime/config/config_test.go +++ b/internal/runtime/config/config_test.go @@ -4,20 +4,182 @@ import ( "testing" ) +// ---- parseInt tests ---- + +func TestParseInt(t *testing.T) { + tests := []struct { + name string + input string + want int + }{ + {name: "empty string returns 0", input: "", want: 0}, + {name: "valid positive", input: "42", want: 42}, + {name: "invalid string returns 0", input: "abc", want: 0}, + {name: "negative value", input: "-5", want: -5}, + {name: "zero string", input: "0", want: 0}, + {name: "whitespace returns 0", input: " ", want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseInt(tt.input) + if got != tt.want { + t.Errorf("parseInt(%q) = %d, want %d", tt.input, got, tt.want) + } + }) + } +} + +// ---- parseBool tests ---- + +func TestParseBool(t *testing.T) { + tests := []struct { + name string + input string + want bool + }{ + {name: "true string", input: "true", want: true}, + {name: "1 string", input: "1", want: true}, + {name: "false string", input: "false", want: false}, + {name: "0 string", input: "0", want: false}, + {name: "empty string returns false", input: "", want: false}, + {name: "invalid string returns false", input: "abc", want: false}, + {name: "TRUE uppercase", input: "TRUE", want: true}, + {name: "FALSE uppercase", input: "FALSE", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseBool(tt.input) + if got != tt.want { + t.Errorf("parseBool(%q) = %v, want %v", tt.input, got, tt.want) + } + }) + } +} + +// ---- parseStringSlice tests ---- + +func TestParseStringSlice(t *testing.T) { + tests := []struct { + name string + input string + wantNil bool + wantLen int + wantVal []string + }{ + { + name: "empty string returns nil", + input: "", + wantNil: true, + }, + { + name: "three elements", + input: "a,b,c", + wantLen: 3, + wantVal: []string{"a", "b", "c"}, + }, + { + name: "single element", + input: "x", + wantLen: 1, + wantVal: []string{"x"}, + }, + { + name: "trailing comma produces empty last element", + input: "a,", + wantLen: 2, + wantVal: []string{"a", ""}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseStringSlice(tt.input) + if tt.wantNil { + if got != nil { + t.Errorf("parseStringSlice(%q) = %v, want nil", tt.input, got) + } + return + } + if len(got) != tt.wantLen { + t.Fatalf("parseStringSlice(%q) len = %d, want %d (got %v)", tt.input, len(got), tt.wantLen, got) + } + for i, want := range tt.wantVal { + if got[i] != want { + t.Errorf("parseStringSlice(%q)[%d] = %q, want %q", tt.input, i, got[i], want) + } + } + }) + } +} + +// ---- parseIntSlice tests ---- + +func TestParseIntSlice(t *testing.T) { + tests := []struct { + name string + input string + want []int + }{ + { + name: "trimmed spaces around values", + input: "1, 2 ,3", + want: []int{1, 2, 3}, + }, + { + name: "zero string is kept", + input: "0,1", + want: []int{0, 1}, + }, + { + name: "non-numeric values are dropped", + input: "x,2", + want: []int{2}, + }, + { + name: "empty string returns empty", + input: "", + want: []int{}, + }, + { + name: "all invalid returns empty", + input: "a,b,c", + want: []int{}, + }, + { + name: "negative value preserved", + input: "-1,2", + want: []int{-1, 2}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseIntSlice(tt.input) + if len(got) != len(tt.want) { + t.Fatalf("parseIntSlice(%q) = %v (len %d), want %v (len %d)", + tt.input, got, len(got), tt.want, len(tt.want)) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Errorf("parseIntSlice(%q)[%d] = %d, want %d", tt.input, i, got[i], tt.want[i]) + } + } + }) + } +} + func TestFeatureFlagsFromBitmap(t *testing.T) { tests := []struct { name string bitmap string // expected field values - tracesOverridePartial bool - tracesOverrideSlash bool - supportsBindingNotAll bool - agentValidate60Ports bool - allowPerContainerDrivers bool - wekaGetCopyLocalDrivers bool - driverSupportsAutoDrain bool - ssdProxyIommuSupport bool - ssdProxyIncludesDpdk bool + tracesOverridePartial bool + tracesOverrideSlash bool + supportsBindingNotAll bool + agentValidate60Ports bool + allowPerContainerDrivers bool + wekaGetCopyLocalDrivers bool + driverSupportsAutoDrain bool + ssdProxyIommuSupport bool + ssdProxyIncludesDpdk bool }{ { name: "invalid base64 returns all false", @@ -38,13 +200,13 @@ func TestFeatureFlagsFromBitmap(t *testing.T) { tracesOverrideSlash: true, }, { - name: "bit 2 sets SupportsBindingToNotAllInterfaces only", - bitmap: "BA==", // 0x04 + name: "bit 2 sets SupportsBindingToNotAllInterfaces only", + bitmap: "BA==", // 0x04 supportsBindingNotAll: true, }, { - name: "bit 7 sets SsdProxyIommuSupport only", - bitmap: "gA==", // 0x80 + name: "bit 7 sets SsdProxyIommuSupport only", + bitmap: "gA==", // 0x80 ssdProxyIommuSupport: true, }, { @@ -53,8 +215,8 @@ func TestFeatureFlagsFromBitmap(t *testing.T) { // all flags remain false: bit 8 is explicitly unused }, { - name: "bit 9 sets SsdProxyIncludesDpdkMemory only", - bitmap: "AAI=", // byte[0]=0x00, byte[1]=0x02 → bit 9 set + name: "bit 9 sets SsdProxyIncludesDpdkMemory only", + bitmap: "AAI=", // byte[0]=0x00, byte[1]=0x02 → bit 9 set ssdProxyIncludesDpdk: true, }, { @@ -63,16 +225,16 @@ func TestFeatureFlagsFromBitmap(t *testing.T) { bitmap: "Bw==", tracesOverridePartial: true, tracesOverrideSlash: true, - supportsBindingNotAll: true, + supportsBindingNotAll: true, }, { // "Hw==" from 5.1.0 release: 0x1F = bits 0,1,2,3,4 - name: "real bitmap Hw== from 5.1.0", - bitmap: "Hw==", - tracesOverridePartial: true, - tracesOverrideSlash: true, - supportsBindingNotAll: true, - agentValidate60Ports: true, + name: "real bitmap Hw== from 5.1.0", + bitmap: "Hw==", + tracesOverridePartial: true, + tracesOverrideSlash: true, + supportsBindingNotAll: true, + agentValidate60Ports: true, allowPerContainerDrivers: true, }, } diff --git a/internal/runtime/cos/hugepages.go b/internal/runtime/cos/hugepages.go index 7bde6752e..ffe5c8e31 100644 --- a/internal/runtime/cos/hugepages.go +++ b/internal/runtime/cos/hugepages.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/pkg/osinfo" "github.com/weka/weka-operator/internal/runtime/cmdutil" "github.com/weka/weka-operator/internal/runtime/config" ) @@ -22,13 +23,14 @@ func ConfigureHugepages(ctx context.Context, cfg *config.Config) error { ctx, logger := instrumentation.CreateLogSpan(ctx, "ConfigureHugepages") defer logger.End() - if !cfg.COSAllowHugepageConfig { - logger.Info("Skipping hugepages configuration (WEKA_COS_ALLOW_HUGEPAGE_CONFIG not set)") - // still check and error if hugepages are missing — fall through to build sedCmds + nodeInfo, err := osinfo.Load() + if err != nil || !nodeInfo.IsCos() { + logger.Info("Skipping hugepages configuration (non-COS node)") + return nil } // Check if hugepages already configured - if count, err := currentHugepageCount(); err == nil && count > 0 { + if count, countErr := currentHugepageCount(); countErr == nil && count > 0 { logger.Info("Node already has hugepages configured, skipping", "count", count) return nil } diff --git a/internal/runtime/cpuaffinity/cpuaffinity.go b/internal/runtime/cpuaffinity/cpuaffinity.go new file mode 100644 index 000000000..7441c56d2 --- /dev/null +++ b/internal/runtime/cpuaffinity/cpuaffinity.go @@ -0,0 +1,422 @@ +// Package cpuaffinity implements CPU core selection and affinity management. +// Mirrors find_full_cores, manage_cpu_affinities, periodic_cpu_affinity_management +// at weka_runtime.py:1647–2073. +package cpuaffinity + +import ( + "bufio" + "context" + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/runtime/config" +) + +// FindFullCores selects n CPU core IDs suitable for Weka data-path processes. +// Mirrors Python find_full_cores() at weka_runtime.py:1716. +func FindFullCores(ctx context.Context, cfg *config.Config, n int) ([]string, error) { + _, logger := instrumentation.CreateLogSpan(ctx, "cpuaffinity.FindFullCores") + defer logger.End() + + // If explicit core IDs are set and not "auto", use them directly. + if len(cfg.CoreIDs) > 0 { + result := make([]string, len(cfg.CoreIDs)) + for i, id := range cfg.CoreIDs { + result[i] = strconv.Itoa(id) + } + return result, nil + } + + available, err := parseCPUAllowedList("/proc/1/status") + if err != nil { + return nil, fmt.Errorf("cpuaffinity: reading allowed CPUs: %w", err) + } + + if cfg.CPUPolicy == "dedicated" { + selected := make([]string, 0, n) + for _, c := range available { + if c != 0 { + selected = append(selected, strconv.Itoa(c)) + if len(selected) == n { + return selected, nil + } + } + } + return nil, fmt.Errorf("cpuaffinity: cannot find %d dedicated cores (found %d)", n, len(selected)) + } + + // Shared (HT) mode: pick one sibling from each fully-available HT pair. + availSet := make(map[int]struct{}, len(available)) + for _, c := range available { + availSet[c] = struct{}{} + } + + var zeroSiblings []int + if _, ok := availSet[0]; ok { + if s, err := readSiblingsList(0); err == nil { + zeroSiblings = s + } + } + zeroSibSet := make(map[int]struct{}, len(zeroSiblings)) + for _, s := range zeroSiblings { + zeroSibSet[s] = struct{}{} + } + + var selected []string + selectedSet := make(map[int]struct{}) + + for _, cpu := range available { + if _, skip := zeroSibSet[cpu]; skip { + continue + } + siblings, err := readSiblingsList(cpu) + if err != nil { + continue + } + // All siblings must be in the allowed set. + allAvail := true + for _, sib := range siblings { + if _, ok := availSet[sib]; !ok { + allAvail = false + break + } + } + if !allAvail { + continue + } + // None of the siblings may already be selected. + alreadySelected := false + for _, sib := range siblings { + if _, ok := selectedSet[sib]; ok { + alreadySelected = true + break + } + } + if alreadySelected { + continue + } + selected = append(selected, strconv.Itoa(cpu)) + selectedSet[cpu] = struct{}{} + if len(selected) == n { + return selected, nil + } + } + return nil, fmt.Errorf("cpuaffinity: cannot find %d full HT core pairs (found %d)", n, len(selected)) +} + +// parseCPUAllowedList reads Cpus_allowed_list from /proc/1/status and returns sorted int slice. +// Mirrors Python parse_cpu_allowed_list() at weka_runtime.py:1647. +func parseCPUAllowedList(path string) ([]int, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() //nolint:errcheck // close error on read-only file is not actionable + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "Cpus_allowed_list") { + parts := strings.SplitN(line, ":\t", 2) + if len(parts) == 2 { + return expandRanges(strings.TrimSpace(parts[1])), nil + } + } + } + return nil, fmt.Errorf("cpus_allowed_list not found in %s", path) +} + +// expandRanges expands a range string like "0-3,8-11" into a sorted int slice. +// Mirrors Python expand_ranges() at weka_runtime.py:1655. +func expandRanges(rangesStr string) []int { + var result []int + for _, part := range strings.Split(rangesStr, ",") { + part = strings.TrimSpace(part) + if idx := strings.Index(part, "-"); idx >= 0 { + start, err := strconv.Atoi(part[:idx]) + if err != nil { + continue + } + end, err := strconv.Atoi(part[idx+1:]) + if err != nil { + continue + } + for i := start; i <= end; i++ { + result = append(result, i) + } + } else if part != "" { + if v, err := strconv.Atoi(part); err == nil { + result = append(result, v) + } + } + } + return result +} + +// readSiblingsList reads the thread_siblings_list for a given CPU index. +// Mirrors Python read_siblings_list() at weka_runtime.py:1666. +func readSiblingsList(cpuIndex int) ([]int, error) { + path := fmt.Sprintf("/sys/devices/system/cpu/cpu%d/topology/thread_siblings_list", cpuIndex) + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return expandRanges(strings.TrimSpace(string(data))), nil +} + +// Manager periodically adjusts CPU affinities of non-datapath processes. +type Manager struct { + cfg *config.Config +} + +// NewManager creates a new Manager. +func NewManager(cfg *config.Config) *Manager { + return &Manager{cfg: cfg} +} + +// RunPeriodic runs the affinity management loop in the background. +// Initial delay is 30s, then runs every 60s. +// Mirrors Python periodic_cpu_affinity_management() at weka_runtime.py:2055. +func (m *Manager) RunPeriodic(ctx context.Context) { + _, logger := instrumentation.CreateLogSpan(ctx, "cpuaffinity.RunPeriodic") + defer logger.End() + + select { + case <-ctx.Done(): + return + case <-time.After(30 * time.Second): + } + + logger.Info("starting periodic CPU affinity management (every 60 seconds)") + + for { + if err := m.Manage(ctx); err != nil { + logger.Warn("periodic CPU affinity management failed (non-fatal)", "err", err) + } + select { + case <-ctx.Done(): + return + case <-time.After(60 * time.Second): + } + } +} + +// Manage identifies processes that need reassignment and tasksets them. +// Mirrors Python manage_cpu_affinities() at weka_runtime.py:1963. +func (m *Manager) Manage(ctx context.Context) error { + _, logger := instrumentation.CreateLogSpan(ctx, "cpuaffinity.Manage") + defer logger.End() + + available, err := parseCPUAllowedList("/proc/1/status") + if err != nil { + return fmt.Errorf("cpuaffinity.Manage: %w", err) + } + + dataPathCores := getDataPathCores() + reservedSet := getAllReservedCores(dataPathCores) + + var remaining []int + for _, c := range available { + if _, reserved := reservedSet[c]; !reserved { + remaining = append(remaining, c) + } + } + + if len(remaining) == 0 { + logger.Warn("no remaining cores available for CPU affinity management") + return nil + } + + coresStr := intsToCSV(remaining) + targetSet := make(map[int]struct{}, len(remaining)) + for _, c := range remaining { + targetSet[c] = struct{}{} + } + + pids := getProcessesToReassign() + var needChange []string + for _, pid := range pids { + current := getProcessAffinity(pid) + if current == nil { + continue + } + if mapsEqual(current, targetSet) { + continue + } + needChange = append(needChange, pid) + } + if len(needChange) == 0 { + return nil + } + + logger.Debug("managing CPU affinities", + "cores", coresStr, + "processes_needing_change", len(needChange), + ) + + changesMade := 0 + for _, pid := range needChange { + cmd := exec.CommandContext(ctx, "taskset", "-cp", coresStr, pid) //nolint:gosec // pid is sourced from ps output, not user input + if err := cmd.Run(); err != nil { + logger.Debug("failed to set affinity", "pid", pid, "err", err) + continue + } + changesMade++ + } + if changesMade > 0 { + logger.Info("CPU affinity management: adjusted processes", "count", changesMade) + } + return nil +} + +// getDataPathCores reads wekanode data-path process core affinities. +// Mirrors Python get_data_path_cores() at weka_runtime.py:1767. +func getDataPathCores() []int { + out, err := exec.Command("ps", "aux").Output() //nolint:gosec // ps with fixed args, no user input + if err != nil { + return nil + } + + var dpPIDs []string + for _, line := range strings.Split(string(out), "\n") { + if strings.Contains(line, "/weka/wekanode") && + strings.Contains(line, "--slot") && + !strings.Contains(line, "--slot 0") { + parts := strings.Fields(line) + if len(parts) > 1 { + dpPIDs = append(dpPIDs, parts[1]) + } + } + } + + coreSet := make(map[int]struct{}) + for _, pid := range dpPIDs { + affinity := getProcessAffinity(pid) + for c := range affinity { + coreSet[c] = struct{}{} + } + } + var result []int + for c := range coreSet { + result = append(result, c) + } + return result +} + +// getAllReservedCores returns data-path cores plus all their HT siblings. +// Mirrors Python get_all_reserved_cores() at weka_runtime.py:1818. +func getAllReservedCores(dataPathCores []int) map[int]struct{} { + reserved := make(map[int]struct{}) + for _, c := range dataPathCores { + reserved[c] = struct{}{} + if siblings, err := readSiblingsList(c); err == nil { + for _, s := range siblings { + reserved[s] = struct{}{} + } + } + } + return reserved +} + +// getProcessesToReassign collects PIDs eligible for affinity reassignment. +// Mirrors Python get_processes_to_reassign() at weka_runtime.py:1881. +func getProcessesToReassign() []string { + out, err := exec.Command("ps", "aux").Output() //nolint:gosec // ps with fixed args, no user input + if err != nil { + return nil + } + + var pids []string + for _, line := range strings.Split(string(out), "\n") { + parts := strings.Fields(line) + if len(parts) < 2 { + continue + } + pid := parts[1] + if pid == "PID" || pid == "1" { + continue + } + if strings.Contains(line, "/weka/wekanode") && !strings.Contains(line, "--slot 0") { + continue + } + if uptime := processUptime(pid); uptime < 10.0 { + continue + } + pids = append(pids, pid) + } + return pids +} + +// getProcessAffinity returns the current affinity set for a PID, or nil on error. +// Mirrors Python get_process_affinity() at weka_runtime.py:1943. +func getProcessAffinity(pid string) map[int]struct{} { + out, err := exec.Command("taskset", "-cp", pid).Output() //nolint:gosec // pid is sourced from ps output, not user input + if err != nil { + return nil + } + for _, line := range strings.Split(string(out), "\n") { + if strings.Contains(line, "affinity list:") { + parts := strings.SplitN(line, "affinity list:", 2) + if len(parts) == 2 { + cores := expandRanges(strings.TrimSpace(parts[1])) + result := make(map[int]struct{}, len(cores)) + for _, c := range cores { + result[c] = struct{}{} + } + return result + } + } + } + return nil +} + +// processUptime returns how long (seconds) a process has been running, or 0 on error. +func processUptime(pid string) float64 { + statData, err := os.ReadFile("/proc/" + pid + "/stat") + if err != nil { + return 0 + } + parts := strings.Fields(string(statData)) + if len(parts) < 22 { + return 0 + } + startTicks, err := strconv.ParseFloat(parts[21], 64) + if err != nil { + return 0 + } + uptimeData, err := os.ReadFile("/proc/uptime") + if err != nil { + return 0 + } + systemUptime, err := strconv.ParseFloat(strings.Fields(string(uptimeData))[0], 64) + if err != nil { + return 0 + } + clkTck := float64(100) // SC_CLK_TCK default + return systemUptime - startTicks/clkTck +} + +func intsToCSV(ints []int) string { + parts := make([]string, len(ints)) + for i, v := range ints { + parts[i] = strconv.Itoa(v) + } + return strings.Join(parts, ",") +} + +func mapsEqual(a, b map[int]struct{}) bool { + if len(a) != len(b) { + return false + } + for k := range a { + if _, ok := b[k]; !ok { + return false + } + } + return true +} diff --git a/internal/runtime/cpuaffinity/cpuaffinity_test.go b/internal/runtime/cpuaffinity/cpuaffinity_test.go new file mode 100644 index 000000000..989f8af65 --- /dev/null +++ b/internal/runtime/cpuaffinity/cpuaffinity_test.go @@ -0,0 +1,245 @@ +package cpuaffinity + +import ( + "os" + "path/filepath" + "testing" +) + +// ---- expandRanges tests ---- + +func TestExpandRanges(t *testing.T) { + tests := []struct { + name string + input string + want []int + }{ + { + name: "range and range", + input: "0-3,8-11", + want: []int{0, 1, 2, 3, 8, 9, 10, 11}, + }, + { + name: "single value", + input: "5", + want: []int{5}, + }, + { + name: "mixed single and range", + input: "0,2-3", + want: []int{0, 2, 3}, + }, + { + name: "empty string returns empty", + input: "", + want: nil, + }, + { + name: "whitespace trimmed around segment", + input: " 0-2 , 5 ", + want: []int{0, 1, 2, 5}, + }, + { + name: "single range", + input: "4-7", + want: []int{4, 5, 6, 7}, + }, + { + name: "multiple singles", + input: "0,4,8", + want: []int{0, 4, 8}, + }, + { + name: "range with single value equals", + input: "3-3", + want: []int{3}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := expandRanges(tt.input) + if len(got) != len(tt.want) { + t.Fatalf("expandRanges(%q) = %v (len %d), want %v (len %d)", + tt.input, got, len(got), tt.want, len(tt.want)) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Errorf("expandRanges(%q)[%d] = %d, want %d", tt.input, i, got[i], tt.want[i]) + } + } + }) + } +} + +// ---- parseCPUAllowedList tests ---- + +func TestParseCPUAllowedList(t *testing.T) { + tests := []struct { + name string + content string // file content; empty means file not created + want []int + wantErr bool + noFile bool // pass a path that doesn't exist + }{ + { + name: "Cpus_allowed_list with tab separator - range", + content: "Name:\tsome_process\nCpus_allowed:\tff\nCpus_allowed_list:\t0-3\nVmRSS:\t1234 kB\n", + want: []int{0, 1, 2, 3}, + }, + { + name: "Cpus_allowed_list single cpu", + content: "Cpus_allowed_list:\t5\n", + want: []int{5}, + }, + { + name: "Cpus_allowed_list multiple ranges", + content: "Cpus_allowed_list:\t0-3,8-11\n", + want: []int{0, 1, 2, 3, 8, 9, 10, 11}, + }, + { + name: "file without Cpus_allowed_list line returns error", + content: "Name:\ttest\nVmRSS:\t100 kB\n", + wantErr: true, + }, + { + name: "missing file returns error", + noFile: true, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var path string + if tt.noFile { + path = filepath.Join(t.TempDir(), "nonexistent_status") + } else { + dir := t.TempDir() + path = filepath.Join(dir, "status") + if err := os.WriteFile(path, []byte(tt.content), 0644); err != nil { + t.Fatalf("write test file: %v", err) + } + } + + got, err := parseCPUAllowedList(path) + if (err != nil) != tt.wantErr { + t.Fatalf("parseCPUAllowedList() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + + if len(got) != len(tt.want) { + t.Fatalf("parseCPUAllowedList() = %v (len %d), want %v (len %d)", + got, len(got), tt.want, len(tt.want)) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Errorf("parseCPUAllowedList()[%d] = %d, want %d", i, got[i], tt.want[i]) + } + } + }) + } +} + +// ---- intsToCSV tests ---- + +func TestIntsToCSV(t *testing.T) { + tests := []struct { + name string + input []int + want string + }{ + {name: "single element", input: []int{5}, want: "5"}, + {name: "multiple elements", input: []int{0, 3, 7}, want: "0,3,7"}, + {name: "empty slice", input: []int{}, want: ""}, + {name: "nil slice", input: nil, want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := intsToCSV(tt.input) + if got != tt.want { + t.Errorf("intsToCSV(%v) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +// ---- mapsEqual tests ---- + +func makeIntSet(vals ...int) map[int]struct{} { + m := make(map[int]struct{}, len(vals)) + for _, v := range vals { + m[v] = struct{}{} + } + return m +} + +func TestMapsEqual(t *testing.T) { + tests := []struct { + name string + a map[int]struct{} + b map[int]struct{} + want bool + }{ + { + name: "equal maps", + a: makeIntSet(1, 2, 3), + b: makeIntSet(3, 2, 1), + want: true, + }, + { + name: "different lengths — a larger", + a: makeIntSet(1, 2, 3), + b: makeIntSet(1, 2), + want: false, + }, + { + name: "different lengths — b larger", + a: makeIntSet(1, 2), + b: makeIntSet(1, 2, 3), + want: false, + }, + { + name: "disjoint keys", + a: makeIntSet(1, 2), + b: makeIntSet(3, 4), + want: false, + }, + { + name: "both empty", + a: makeIntSet(), + b: makeIntSet(), + want: true, + }, + { + name: "nil maps", + a: nil, + b: nil, + want: true, + }, + { + name: "one nil one empty", + a: nil, + b: makeIntSet(), + want: true, + }, + { + name: "partial overlap", + a: makeIntSet(1, 2, 3), + b: makeIntSet(1, 2, 4), + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := mapsEqual(tt.a, tt.b) + if got != tt.want { + t.Errorf("mapsEqual(%v, %v) = %v, want %v", tt.a, tt.b, got, tt.want) + } + }) + } +} diff --git a/internal/runtime/drivers/build.go b/internal/runtime/drivers/build.go new file mode 100644 index 000000000..25a99dfdb --- /dev/null +++ b/internal/runtime/drivers/build.go @@ -0,0 +1,112 @@ +// Package drivers provides helpers for building and loading Weka kernel drivers. +package drivers + +import ( + "fmt" + "os" + "regexp" + "strings" + + "github.com/weka/weka-operator/internal/pkg/osinfo" +) + +const ubuntu24BuildID = "ubuntu24.04" + +// GetWekaVersion returns the version string from the release spec directory. +// Scans /opt/weka/dist/release and /shared-weka-version/opt-weka/dist/release; +// expects exactly one .spec file in whichever directory is found first. +func GetWekaVersion() (string, error) { + dirs := []string{ + "/opt/weka/dist/release", + "/shared-weka-version/opt-weka/dist/release", + } + for _, dir := range dirs { + entries, err := os.ReadDir(dir) + if err != nil || len(entries) == 0 { + continue + } + if len(entries) != 1 { + return "", fmt.Errorf("expected one release spec in %s, found %d", dir, len(entries)) + } + name := entries[0].Name() + version := strings.TrimSuffix(name, ".spec") + version = strings.SplitN(version, ".spec", 2)[0] + return version, nil + } + return "", fmt.Errorf("no release files found in any of: %v", dirs) +} + +// KernelBuildID returns the kernel build ID to pass to weka driver commands. +// An empty string means the caller should omit the --kernel-build-id flag (weka uses uname -r). +func KernelBuildID(driversBuildID, distService string) (string, error) { + if driversBuildID != "" && driversBuildID != "auto" { + return driversBuildID, nil + } + + nodeInfo, err := osinfo.Load() + if err != nil { + return "", fmt.Errorf("KernelBuildID: load osinfo: %w", err) + } + + switch { + case nodeInfo.IsCos(): + if nodeInfo.OsBuildId == "" { + return "", fmt.Errorf("OS_BUILD_ID is required for Google COS driver builds") + } + return nodeInfo.OsBuildId, nil + + case isUbuntu24(nodeInfo): + if distService != "" { + return ubuntu24BuildID, nil + } + // No dist service: weka will use uname -r internally. + return "", nil + + default: + // RHCOS and others use the OS build ID. + return nodeInfo.OsBuildId, nil + } +} + +// KernelSignature scans driversDir for a file matching +// weka-driver--.zip and returns the kernel signature hex string. +func KernelSignature(driversDir string) (string, error) { + entries, err := os.ReadDir(driversDir) + if err != nil { + return "", fmt.Errorf("KernelSignature: read dir %s: %w", driversDir, err) + } + re := regexp.MustCompile(`^weka-driver-[a-f0-9]+-([a-f0-9]+)\.zip$`) + for _, e := range entries { + if m := re.FindStringSubmatch(e.Name()); m != nil { + return m[1], nil + } + } + return "", fmt.Errorf("no weka-driver zip found in %s", driversDir) +} + +// WekaDriversHandling returns true when the Weka version uses new driver handling. +// Delegates to ResolveVersionParams(imageName).WekaDriversHandling, which faithfully +// mirrors the VERSION_TO_DRIVERS_MAP_WEKAFS / DEFAULT_PARAMS lookup in +// weka_runtime.py:1339-1351. The old "4.2." string heuristic was incorrect for +// all explicit map entries (they all set weka_drivers_handling=False regardless of prefix). +func WekaDriversHandling(imageName string) bool { + return ResolveVersionParams(imageName).WekaDriversHandling +} + +func isUbuntu24(nodeInfo *osinfo.NodeInfo) bool { + if !nodeInfo.IsUbuntu() { + return false + } + parts := strings.SplitN(nodeInfo.OsBuildId, ".", 2) + if len(parts) == 0 { + return false + } + major := 0 + for _, c := range parts[0] { + if c < '0' || c > '9' { + return false + } + major = major*10 + int(c-'0') + } + return major >= 24 +} diff --git a/internal/runtime/drivers/build_test.go b/internal/runtime/drivers/build_test.go new file mode 100644 index 000000000..b60d0f142 --- /dev/null +++ b/internal/runtime/drivers/build_test.go @@ -0,0 +1,192 @@ +package drivers + +import ( + "os" + "path/filepath" + "testing" + + "github.com/weka/weka-operator/internal/pkg/osinfo" +) + +// ---- KernelSignature tests ---- + +func TestKernelSignature(t *testing.T) { + tests := []struct { + name string + setup func(dir string) // creates files inside dir + want string + wantErr bool + nonExist bool // pass a non-existent dir entirely + }{ + { + name: "valid weka-driver zip returns signature", + setup: func(dir string) { + name := "weka-driver-abc123def456-deadbeefcafe.zip" + _ = os.WriteFile(filepath.Join(dir, name), []byte(""), 0644) + }, + want: "deadbeefcafe", + }, + { + name: "dir with no matching zip returns error", + setup: func(dir string) { + _ = os.WriteFile(filepath.Join(dir, "some-other-file.txt"), []byte(""), 0644) + }, + wantErr: true, + }, + { + name: "malformed zip name (no signature group) returns error", + setup: func(dir string) { + // File that starts with weka-driver but has only one hex segment (no sig). + _ = os.WriteFile(filepath.Join(dir, "weka-driver-abc123.zip"), []byte(""), 0644) + }, + wantErr: true, + }, + { + name: "non-existent dir returns error", + nonExist: true, + wantErr: true, + }, + { + name: "empty dir returns error", + setup: func(dir string) { + // nothing created + }, + wantErr: true, + }, + { + name: "correct file alongside irrelevant files is found", + setup: func(dir string) { + _ = os.WriteFile(filepath.Join(dir, "unrelated.tar.gz"), []byte(""), 0644) + _ = os.WriteFile(filepath.Join(dir, "weka-driver-ff00aa11bb22-cc33dd44.zip"), []byte(""), 0644) + }, + want: "cc33dd44", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var dir string + if tt.nonExist { + dir = filepath.Join(t.TempDir(), "nonexistent-subdir") + } else { + dir = t.TempDir() + if tt.setup != nil { + tt.setup(dir) + } + } + + got, err := KernelSignature(dir) + if (err != nil) != tt.wantErr { + t.Fatalf("KernelSignature() error = %v, wantErr %v", err, tt.wantErr) + } + if !tt.wantErr && got != tt.want { + t.Errorf("KernelSignature() = %q, want %q", got, tt.want) + } + }) + } +} + +// ---- isUbuntu24 tests ---- + +func TestIsUbuntu24(t *testing.T) { + tests := []struct { + name string + nodeInfo *osinfo.NodeInfo + want bool + }{ + { + name: "Ubuntu OsBuildId 24.04 → true", + nodeInfo: &osinfo.NodeInfo{Os: osinfo.OsNameUbuntu, OsBuildId: "24.04"}, + want: true, + }, + { + name: "Ubuntu OsBuildId 22.04 → false", + nodeInfo: &osinfo.NodeInfo{Os: osinfo.OsNameUbuntu, OsBuildId: "22.04"}, + want: false, + }, + { + name: "Ubuntu OsBuildId 24 (no dot) → true", + nodeInfo: &osinfo.NodeInfo{Os: osinfo.OsNameUbuntu, OsBuildId: "24"}, + want: true, + }, + { + name: "Ubuntu OsBuildId 20.04 → false", + nodeInfo: &osinfo.NodeInfo{Os: osinfo.OsNameUbuntu, OsBuildId: "20.04"}, + want: false, + }, + { + name: "non-Ubuntu (cos) → false regardless of build id", + nodeInfo: &osinfo.NodeInfo{Os: osinfo.OsNameCos, OsBuildId: "24.04"}, + want: false, + }, + { + name: "non-Ubuntu (rhcos) → false", + nodeInfo: &osinfo.NodeInfo{Os: osinfo.OsNameRhCos, OsBuildId: "24.04"}, + want: false, + }, + { + name: "Ubuntu non-numeric major → false", + nodeInfo: &osinfo.NodeInfo{Os: osinfo.OsNameUbuntu, OsBuildId: "focal.04"}, + want: false, + }, + { + name: "Ubuntu OsBuildId 26.04 → true", + nodeInfo: &osinfo.NodeInfo{Os: osinfo.OsNameUbuntu, OsBuildId: "26.04"}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isUbuntu24(tt.nodeInfo) + if got != tt.want { + t.Errorf("isUbuntu24(%+v) = %v, want %v", tt.nodeInfo, got, tt.want) + } + }) + } +} + +// ---- WekaDriversHandling tests ---- + +func TestWekaDriversHandling(t *testing.T) { + tests := []struct { + name string + imageName string + want bool + }{ + { + name: "unknown tag → default params → true", + imageName: "quay.io/weka/weka:99.99.99-unknown", + want: true, + }, + { + name: "known tag 4.3.3 → explicit map entry → false", + imageName: "quay.io/weka/weka:4.3.3", + want: false, + }, + { + name: "known 4.2.10-k8so.0 entry → false", + imageName: "quay.io/weka/weka:4.2.10-k8so.0", + want: false, + }, + { + name: "s3multitenancy override → false (WekaDriversHandling absent in override dict)", + imageName: "quay.io/weka/weka:4.2.7.64-s3multitenancy.3", + want: false, + }, + { + name: "empty image name → unknown tag → default params → true", + imageName: "", + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := WekaDriversHandling(tt.imageName) + if got != tt.want { + t.Errorf("WekaDriversHandling(%q) = %v, want %v", tt.imageName, got, tt.want) + } + }) + } +} diff --git a/internal/runtime/drivers/load.go b/internal/runtime/drivers/load.go new file mode 100644 index 000000000..b495921a7 --- /dev/null +++ b/internal/runtime/drivers/load.go @@ -0,0 +1,178 @@ +package drivers + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/pkg/osinfo" + "github.com/weka/weka-operator/internal/runtime/cmdutil" +) + +// SetupOverlayfsForLibModules mounts a tmpfs-backed overlayfs over /lib/modules so +// that the kernel driver installer can write into what is typically a read-only host mount. +func SetupOverlayfsForLibModules(ctx context.Context) error { + ctx, logger := instrumentation.CreateLogSpan(ctx, "drivers.SetupOverlayfsForLibModules") + defer logger.End() + + realPathBytes, err := cmdutil.Output(ctx, "readlink", "-f", "/lib/modules") + if err != nil { + return fmt.Errorf("SetupOverlayfsForLibModules: readlink: %w", err) + } + realPath := strings.TrimSpace(string(realPathBytes)) + + const ovlBase = "/tmp/ovl-libmodules" + upperDir := ovlBase + "/upper" + workDir := ovlBase + "/work" + ovlMnt := ovlBase + "/mnt" + + if err := os.MkdirAll(ovlBase, 0o755); err != nil { + return fmt.Errorf("SetupOverlayfsForLibModules: mkdir %s: %w", ovlBase, err) + } + + // L4: Skip the tmpfs mount if ovlBase is already a mountpoint (idempotency guard). + // Mirrors Python weka_runtime.py:1565-1570: + // if (await run_command(f"mountpoint -q {ovl_root}"))[2] != 0: mount tmpfs ... + if err := cmdutil.Run(ctx, "mountpoint", "-q", ovlBase); err != nil { + if err := cmdutil.Run(ctx, "mount", "-t", "tmpfs", "-o", "size=512m", "tmpfs", ovlBase); err != nil { + return fmt.Errorf("SetupOverlayfsForLibModules: mount tmpfs: %w", err) + } + } + + for _, dir := range []string{upperDir, workDir, ovlMnt} { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("SetupOverlayfsForLibModules: mkdir %s: %w", dir, err) + } + } + + overlayOpts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", realPath, upperDir, workDir) + if err := cmdutil.Run(ctx, "mount", "-t", "overlay", "overlay", "-o", overlayOpts, ovlMnt); err != nil { + return fmt.Errorf("SetupOverlayfsForLibModules: mount overlay: %w", err) + } + + if err := cmdutil.Run(ctx, "mount", "--bind", ovlMnt, realPath); err != nil { + return fmt.Errorf("SetupOverlayfsForLibModules: bind mount to %s: %w", realPath, err) + } + + if realPath != "/lib/modules" { + if err := cmdutil.Run(ctx, "mount", "--bind", realPath, "/lib/modules"); err != nil { + return fmt.Errorf("SetupOverlayfsForLibModules: bind mount to /lib/modules: %w", err) + } + } + + logger.Info("overlayfs for /lib/modules set up", "realPath", realPath) + return nil +} + +// DisableDriverSigning handles COS-specific kernel module signature enforcement. +// allowDisableSign should be cfg.COSAllowDisableDriverSign. On non-COS nodes it is a no-op. +func DisableDriverSigning(ctx context.Context, allowDisableSign bool) error { + ctx, logger := instrumentation.CreateLogSpan(ctx, "drivers.DisableDriverSigning") + defer logger.End() + + nodeInfo, err := osinfo.Load() + if err != nil || !nodeInfo.IsCos() { + return nil + } + + logger.Info("checking kernel driver signing enforcement on COS") + return cosDisableDriverSigning(ctx, allowDisableSign) +} + +func cosDisableDriverSigning(ctx context.Context, allowDisableSign bool) error { + cmdlineData, err := os.ReadFile("/hostside/proc/cmdline") + if err != nil { + return fmt.Errorf("cosDisableDriverSigning: read cmdline: %w", err) + } + line := string(cmdlineData) + + type sedCmd struct{ from, to string } + var cmds []sedCmd + + if strings.Contains(line, "module.sig_enforce") { + if strings.Contains(line, "module.sig_enforce=1") { + cmds = append(cmds, sedCmd{"module.sig_enforce=1", "module.sig_enforce=0"}) + } + } else { + cmds = append(cmds, sedCmd{"cros_efi", "cros_efi module.sig_enforce=0"}) + } + if strings.Contains(line, "loadpin.enabled") { + if strings.Contains(line, "loadpin.enabled=1") { + cmds = append(cmds, sedCmd{"loadpin.enabled=1", "loadpin.enabled=0"}) + } + } else { + cmds = append(cmds, sedCmd{"cros_efi", "cros_efi loadpin.enabled=0"}) + } + if strings.Contains(line, "loadpin.enforce") { + if strings.Contains(line, "loadpin.enforce=1") { + cmds = append(cmds, sedCmd{"loadpin.enforce=1", "loadpin.enforce=0"}) + } + } else { + cmds = append(cmds, sedCmd{"cros_efi", "cros_efi loadpin.enforce=0"}) + } + + if len(cmds) == 0 { + return nil + } + + if !allowDisableSign { + return fmt.Errorf("node driver signing must be disabled but WEKA_COS_ALLOW_DISABLE_DRIVER_SIGNING is not set") + } + + const espPartition = "/dev/disk/by-partlabel/EFI-SYSTEM" + const mountPath = "/tmp/esp" + const grubCfg = "efi/boot/grub.cfg" + + if err := os.MkdirAll(mountPath, 0o755); err != nil { + return err + } + if err := cmdutil.Run(ctx, "mount", espPartition, mountPath); err != nil { + return fmt.Errorf("cosDisableDriverSigning: mount ESP: %w", err) + } + defer func() { _ = cmdutil.Run(ctx, "umount", mountPath) }() //nolint:errcheck // best-effort cleanup on defer + + for _, sc := range cmds { + script := fmt.Sprintf("cd %s && sed -i 's/%s/%s/g' %s", mountPath, sc.from, sc.to, grubCfg) + if err := cmdutil.Run(ctx, "sh", "-c", script); err != nil { + return fmt.Errorf("cosDisableDriverSigning: sed: %w", err) + } + } + + // Reboot via sysrq-trigger. + _ = os.WriteFile("/hostside/proc/sysrq-trigger", []byte("b"), 0o200) //nolint:errcheck // reboot trigger: process ends immediately after + return nil +} + +// LoadModules runs the post-load steps common to both legacy and new driver modes: +// vfio-pci, arp_tables, and optionally uio_pci_generic. +func LoadModules(ctx context.Context, skipUIOPCIGeneric bool) { + _, logger := instrumentation.CreateLogSpan(ctx, "drivers.LoadModules") + defer logger.End() + + nodeInfo, err := osinfo.Load() + isCOS := err == nil && nodeInfo != nil && nodeInfo.IsCos() + + loadVfioPCI := func() { + if isCOS { + _ = cmdutil.Run(ctx, "modprobe", "vfio-pci") //nolint:errcheck // best-effort: module may already be loaded + return + } + entries, err := os.ReadDir("/sys/kernel/iommu_groups/") + if err == nil && len(entries) > 0 { + _ = cmdutil.Run(ctx, "modprobe", "vfio-pci") //nolint:errcheck // best-effort: module may already be loaded + } + } + loadVfioPCI() + + if err := cmdutil.Run(ctx, "modprobe", "arp_tables"); err != nil { + logger.Warn("failed to load arp_tables (non-fatal)", "err", err) + } + + if !skipUIOPCIGeneric { + if err := cmdutil.Run(ctx, "modprobe", "uio_pci_generic"); err != nil { + logger.Warn("failed to load uio_pci_generic (non-fatal)", "err", err) + } + } +} diff --git a/internal/runtime/drivers/versionparams.go b/internal/runtime/drivers/versionparams.go new file mode 100644 index 000000000..30fde0888 --- /dev/null +++ b/internal/runtime/drivers/versionparams.go @@ -0,0 +1,199 @@ +// Package drivers provides helpers for building and loading Weka kernel drivers. +package drivers + +import "strings" + +// Driver version constants. +// Mirrors weka_runtime.py:1331-1334. +const ( + IgbUioDriverVersion = "weka1.0.2" + MpinUserDriverVersion = "1.0.1" + UioPciGenericDriverVersion = "5f49bb7dc1b5d192fb01b442b17ddc0451313ea2" + DefaultDependencyVersion = "1.0.0-024f0fdaa33ec66087bc6c5631b85819" +) + +// VersionParams holds the per-image-tag driver version parameters. +// +// Tri-state for uio_pci_generic (mirrors Python version_params.get('uio_pci_generic') is not False): +// - UioPciGenericDisabled = true → key was explicitly False → skip loading +// - UioPciGenericDisabled = false, UioPciGeneric = "" → key absent → load it +// - UioPciGenericDisabled = false, UioPciGeneric != "" → key is a version string → load that version +// +// Mirrors Python VersionParams dict and DEFAULT_PARAMS at weka_runtime.py:1254-1351. +type VersionParams struct { + // Wekafs driver version (wekafs key in map). Empty means the default from the image. + Wekafs string + // MpinUser driver version (mpin_user key in map). Empty means MPIN_USER_DRIVER_VERSION constant. + MpinUser string + // IgbUio driver version (igb_uio key in map). Empty means IGB_UIO_DRIVER_VERSION constant. + IgbUio string + // UioPciGeneric holds the version string when uio_pci_generic key is a string, else "". + UioPciGeneric string + // UioPciGenericDisabled is true when uio_pci_generic key is explicitly False (skip loading). + // Mirrors Python: version_params.get('uio_pci_generic') is False at weka_runtime.py:1418. + UioPciGenericDisabled bool + // Dependencies version (dependencies key in map). + Dependencies string + // WekaDriversHandling is true when the image uses new weka driver subcommands (DEFAULT_PARAMS). + // False for all explicit map entries (legacy handling). + // Mirrors Python WEKA_DRIVERS_HANDLING = True if version_params.get("weka_drivers_handling") at weka_runtime.py:1351. + WekaDriversHandling bool +} + +// versionToDriversMapWekafs is a verbatim copy of VERSION_TO_DRIVERS_MAP_WEKAFS. +// Mirrors weka_runtime.py:1254-1325. +// Keys are image tags (the substring after the last ':' in IMAGE_NAME). +var versionToDriversMapWekafs = map[string]VersionParams{ + // 4.3.x-dev entries: uio_pci_generic=False, dependencies="6b519d501ea82063", weka_drivers_handling absent (→ False) + "4.3.1.29791-9f57657d1fb70e71a3fb914ff7d75eee-dev": { + Wekafs: "cc9937c66eb1d0be-GW_556972ab1ad2a29b0db5451e9db18748", + UioPciGenericDisabled: true, + Dependencies: "6b519d501ea82063", + }, + "4.3.2.560-842278e2dca9375f84bd3784a4e7515c-dev3": { + Wekafs: "1acd22f9ddbda67d-GW_556972ab1ad2a29b0db5451e9db18748", + UioPciGenericDisabled: true, + Dependencies: "6b519d501ea82063", + }, + "4.3.2.560-842278e2dca9375f84bd3784a4e7515c-dev4": { + Wekafs: "1acd22f9ddbda67d-GW_556972ab1ad2a29b0db5451e9db18748", + UioPciGenericDisabled: true, + Dependencies: "6b519d501ea82063", + }, + "4.3.2.560-842278e2dca9375f84bd3784a4e7515c-dev5": { + Wekafs: "1acd22f9ddbda67d-GW_556972ab1ad2a29b0db5451e9db18748", + UioPciGenericDisabled: true, + Dependencies: "6b519d501ea82063", + }, + "4.3.2.783-f5fe2ec58286d9fa8fc033f920e6c842-dev": { + Wekafs: "1cb1639d52a2b9ca-GW_556972ab1ad2a29b0db5451e9db18748", + UioPciGenericDisabled: true, + Dependencies: "6b519d501ea82063", + }, + "4.3.3.28-k8s-alpha-dev": { + Wekafs: "1cb1639d52a2b9ca-GW_556972ab1ad2a29b0db5451e9db18748", + UioPciGenericDisabled: true, + Dependencies: "6b519d501ea82063", + }, + "4.3.3.28-k8s-alpha-dev2": { + Wekafs: "1cb1639d52a2b9ca-GW_556972ab1ad2a29b0db5451e9db18748", + UioPciGenericDisabled: true, + Dependencies: "6b519d501ea82063", + }, + "4.3.3.28-k8s-alpha-dev3": { + Wekafs: "1cb1639d52a2b9ca-GW_556972ab1ad2a29b0db5451e9db18748", + UioPciGenericDisabled: true, + Dependencies: "6b519d501ea82063", + }, + "4.3.2.783-f5fe2ec58286d9fa8fc033f920e6c842-dev2": { + Wekafs: "1cb1639d52a2b9ca-GW_556972ab1ad2a29b0db5451e9db18748", + UioPciGenericDisabled: true, + Dependencies: "6b519d501ea82063", + }, + "4.3.2.783-f5fe2ec58286d9fa8fc033f920e6c842-dev3": { + Wekafs: "1cb1639d52a2b9ca-GW_556972ab1ad2a29b0db5451e9db18748", + UioPciGenericDisabled: true, + Dependencies: "6b519d501ea82063", + }, + // 4.2.x entries: uio_pci_generic absent (→ load it), weka_drivers_handling absent (→ False) + "4.2.7.64-k8so-beta.10": { + Wekafs: "1.0.0-995f26b334137fd78d57c264d5b19852-GW_aedf44a11ca66c7bb599f302ae1dff86", + }, + "4.2.10.1693-251d3172589e79bd4960da8031a9a693-dev": { // dev 4.2.7-based version + Wekafs: "1.0.0-995f26b334137fd78d57c264d5b19852-GW_aedf44a11ca66c7bb599f302ae1dff86", + }, + "4.2.10.1290-e552f99e92504c69126da70e1740f6e4-dev": { + Wekafs: "1.0.0-c50570e208c935e9129c9054140ab11a-GW_aedf44a11ca66c7bb599f302ae1dff86", + }, + "4.2.10-k8so.0": { + Wekafs: "1.0.0-c50570e208c935e9129c9054140ab11a-GW_aedf44a11ca66c7bb599f302ae1dff86", + }, + "4.2.10.1671-363e1e8fcfb1290e061815445e973310-dev": { + Wekafs: "1.0.0-c50570e208c935e9129c9054140ab11a-GW_aedf44a11ca66c7bb599f302ae1dff86", + }, + // Plain 4.3.3: uio_pci_generic=False, dependencies="7955984e4bce9d8b", weka_drivers_handling=False + "4.3.3": { + Wekafs: "cbd05f716a3975f7-GW_556972ab1ad2a29b0db5451e9db18748", + UioPciGenericDisabled: true, + Dependencies: "7955984e4bce9d8b", + WekaDriversHandling: false, + }, +} + +// defaultParams mirrors Python DEFAULT_PARAMS at weka_runtime.py:1335-1338: +// +// DEFAULT_PARAMS = dict(weka_drivers_handling=True, uio_pci_generic=False) +// +// Unknown image tags fall back to this: new driver handling, uio_pci_generic disabled. +var defaultParams = VersionParams{ + WekaDriversHandling: true, + UioPciGenericDisabled: true, +} + +// ResolveVersionParams looks up the per-image-tag version parameters. +// +// Algorithm mirrors weka_runtime.py:1339-1348: +// 1. Extract tag = imageName after last ':'. +// 2. Look up tag in VERSION_TO_DRIVERS_MAP_WEKAFS; if absent use DEFAULT_PARAMS. +// 3. If "4.2.7.64-s3multitenancy." appears anywhere in imageName, override wholesale. +func ResolveVersionParams(imageName string) *VersionParams { + // Extract tag (substring after last ':'). + tag := imageName + if idx := strings.LastIndex(imageName, ":"); idx >= 0 { + tag = imageName[idx+1:] + } + + // Map lookup with DEFAULT_PARAMS fallback. + // Mirrors: version_params = VERSION_TO_DRIVERS_MAP_WEKAFS.get(IMAGE_NAME.split(":")[-1], DEFAULT_PARAMS) + params, found := versionToDriversMapWekafs[tag] + if !found { + params = defaultParams + } + + // Wholesale override for 4.2.7.64-s3multitenancy images. + // Mirrors weka_runtime.py:1342-1348: + // if "4.2.7.64-s3multitenancy." in IMAGE_NAME: + // version_params = dict(wekafs=..., mpin_user=..., igb_uio=..., uio_pci_generic=...) + if strings.Contains(imageName, "4.2.7.64-s3multitenancy.") { + params = VersionParams{ + Wekafs: "1.0.0-995f26b334137fd78d57c264d5b19852-GW_aedf44a11ca66c7bb599f302ae1dff86", + MpinUser: "f8c7f8b24611c2e458103da8de26d545", + IgbUio: "b64e22645db30b31b52f012cc75e9ea0", + UioPciGeneric: "1.0.0-929f279ce026ddd2e31e281b93b38f52", + // WekaDriversHandling absent from override dict → False + // UioPciGenericDisabled absent → false (string version present → load it) + } + } + + return ¶ms +} + +// EffectiveUioPciGenericVersion returns the effective uio_pci_generic driver version. +// When UioPciGeneric is set in params use it; otherwise fall back to the global constant. +// Mirrors Python: version_params.get("uio_pci_generic", UIO_PCI_GENERIC_DRIVER_VERSION) used +// in legacy load path at weka_runtime.py:1448. +func (p *VersionParams) EffectiveUioPciGenericVersion() string { + if p.UioPciGeneric != "" { + return p.UioPciGeneric + } + return UioPciGenericDriverVersion +} + +// EffectiveDependencies returns the dependency version, falling back to DefaultDependencyVersion. +// Mirrors Python: version_params.get('dependencies', DEFAULT_DEPENDENCY_VERSION) at weka_runtime.py:2999. +func (p *VersionParams) EffectiveDependencies() string { + if p.Dependencies != "" { + return p.Dependencies + } + return DefaultDependencyVersion +} + +// ShouldSkipUioPciGeneric returns true when uio_pci_generic loading should be skipped. +// Mirrors Python should_skip_uio_pci_generic() at weka_runtime.py:1418: +// +// return version_params.get('uio_pci_generic') is False or should_skip_uio() +// +// (should_skip_uio() = is_google_cos(); the COS check is the caller's responsibility.) +func (p *VersionParams) ShouldSkipUioPciGeneric() bool { + return p.UioPciGenericDisabled +} diff --git a/internal/runtime/drivers/versionparams_test.go b/internal/runtime/drivers/versionparams_test.go new file mode 100644 index 000000000..66eed5579 --- /dev/null +++ b/internal/runtime/drivers/versionparams_test.go @@ -0,0 +1,178 @@ +package drivers + +import "testing" + +// TestResolveVersionParams validates faithful parity with the Python +// VERSION_TO_DRIVERS_MAP_WEKAFS / DEFAULT_PARAMS lookup at weka_runtime.py:1254-1351. +func TestResolveVersionParams(t *testing.T) { + tests := []struct { + name string + imageName string + wantHandling bool // WekaDriversHandling + wantSkipUio bool // ShouldSkipUioPciGeneric + wantDependencies string // EffectiveDependencies() + wantWekafs string // exact Wekafs (only checked when non-empty) + wantEffectiveUioVer string // EffectiveUioPciGenericVersion() (only checked when non-empty) + }{ + { + // Unknown tag → DEFAULT_PARAMS: new handling, uio_pci_generic=False (skip), default deps. + name: "unknown image falls back to DEFAULT_PARAMS", + imageName: "quay.io/weka/weka-in-container:9.9.9-unknown", + wantHandling: true, + wantSkipUio: true, + wantDependencies: DefaultDependencyVersion, + }, + { + // Plain 4.3.3 → legacy handling, uio_pci_generic=False, explicit dependencies. + name: "plain 4.3.3 is legacy with explicit deps", + imageName: "quay.io/weka/weka-in-container:4.3.3", + wantHandling: false, + wantSkipUio: true, + wantDependencies: "7955984e4bce9d8b", + wantWekafs: "cbd05f716a3975f7-GW_556972ab1ad2a29b0db5451e9db18748", + }, + { + // 4.3.x-dev → legacy handling, uio_pci_generic=False, dev deps. + name: "4.3.x-dev is legacy and skips uio", + imageName: "quay.io/weka/weka-in-container:4.3.1.29791-9f57657d1fb70e71a3fb914ff7d75eee-dev", + wantHandling: false, + wantSkipUio: true, + wantDependencies: "6b519d501ea82063", + }, + { + // 4.2.x map entry → legacy handling, uio_pci_generic key ABSENT → load it (do not skip). + name: "4.2.7.64 entry loads uio and uses default deps", + imageName: "quay.io/weka/weka-in-container:4.2.7.64-k8so-beta.10", + wantHandling: false, + wantSkipUio: false, + wantDependencies: DefaultDependencyVersion, + }, + { + // s3multitenancy wholesale override (matched by substring, not tag) → + // legacy handling, uio_pci_generic is a version string → load that version (do not skip). + name: "4.2.7.64-s3multitenancy wholesale override", + imageName: "quay.io/weka/weka-in-container:4.2.7.64-s3multitenancy.5", + wantHandling: false, + wantSkipUio: false, + wantDependencies: DefaultDependencyVersion, + wantWekafs: "1.0.0-995f26b334137fd78d57c264d5b19852-GW_aedf44a11ca66c7bb599f302ae1dff86", + wantEffectiveUioVer: "1.0.0-929f279ce026ddd2e31e281b93b38f52", + }, + { + // Tag extraction: no colon → whole string is the tag (still resolves via override path here). + name: "image without registry colon", + imageName: "4.3.3", + wantHandling: false, + wantSkipUio: true, + wantDependencies: "7955984e4bce9d8b", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + p := ResolveVersionParams(tc.imageName) + if p.WekaDriversHandling != tc.wantHandling { + t.Errorf("WekaDriversHandling = %v, want %v", p.WekaDriversHandling, tc.wantHandling) + } + if got := p.ShouldSkipUioPciGeneric(); got != tc.wantSkipUio { + t.Errorf("ShouldSkipUioPciGeneric() = %v, want %v", got, tc.wantSkipUio) + } + if got := p.EffectiveDependencies(); got != tc.wantDependencies { + t.Errorf("EffectiveDependencies() = %q, want %q", got, tc.wantDependencies) + } + if tc.wantWekafs != "" && p.Wekafs != tc.wantWekafs { + t.Errorf("Wekafs = %q, want %q", p.Wekafs, tc.wantWekafs) + } + if tc.wantEffectiveUioVer != "" { + if got := p.EffectiveUioPciGenericVersion(); got != tc.wantEffectiveUioVer { + t.Errorf("EffectiveUioPciGenericVersion() = %q, want %q", got, tc.wantEffectiveUioVer) + } + } + }) + } +} + +// TestEffectiveUioPciGenericVersionFallback verifies the constant fallback when no version is set. +func TestEffectiveUioPciGenericVersionFallback(t *testing.T) { + p := ResolveVersionParams("quay.io/weka/weka-in-container:9.9.9-unknown") + if got := p.EffectiveUioPciGenericVersion(); got != UioPciGenericDriverVersion { + t.Errorf("EffectiveUioPciGenericVersion() = %q, want fallback %q", got, UioPciGenericDriverVersion) + } +} + +// TestEffectiveUioPciGenericVersion covers both the set-value and fallback paths explicitly. +func TestEffectiveUioPciGenericVersion(t *testing.T) { + tests := []struct { + name string + params VersionParams + want string + }{ + { + name: "UioPciGeneric set — returns set value", + params: VersionParams{UioPciGeneric: "1.0.0-929f279ce026ddd2e31e281b93b38f52"}, + want: "1.0.0-929f279ce026ddd2e31e281b93b38f52", + }, + { + name: "UioPciGeneric empty — falls back to global constant", + params: VersionParams{UioPciGeneric: ""}, + want: UioPciGenericDriverVersion, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.params.EffectiveUioPciGenericVersion() + if got != tt.want { + t.Errorf("EffectiveUioPciGenericVersion() = %q, want %q", got, tt.want) + } + }) + } +} + +// TestEffectiveDependencies covers both the set-value and fallback paths explicitly. +func TestEffectiveDependencies(t *testing.T) { + tests := []struct { + name string + params VersionParams + want string + }{ + { + name: "Dependencies set — returns set value", + params: VersionParams{Dependencies: "7955984e4bce9d8b"}, + want: "7955984e4bce9d8b", + }, + { + name: "Dependencies empty — falls back to global constant", + params: VersionParams{Dependencies: ""}, + want: DefaultDependencyVersion, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.params.EffectiveDependencies() + if got != tt.want { + t.Errorf("EffectiveDependencies() = %q, want %q", got, tt.want) + } + }) + } +} + +// TestShouldSkipUioPciGeneric verifies it directly reflects UioPciGenericDisabled. +func TestShouldSkipUioPciGeneric(t *testing.T) { + tests := []struct { + name string + disabled bool + want bool + }{ + {name: "UioPciGenericDisabled true → skip", disabled: true, want: true}, + {name: "UioPciGenericDisabled false → do not skip", disabled: false, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := VersionParams{UioPciGenericDisabled: tt.disabled} + got := p.ShouldSkipUioPciGeneric() + if got != tt.want { + t.Errorf("ShouldSkipUioPciGeneric() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/runtime/generation/generation.go b/internal/runtime/generation/generation.go new file mode 100644 index 000000000..fb454d4c1 --- /dev/null +++ b/internal/runtime/generation/generation.go @@ -0,0 +1,93 @@ +// Package generation manages the weka_runtime generation file used for takeover detection. +// Mirrors write_generation, obtain_lock, is_wrong_generation, get_boot_id at weka_runtime.py:2776–4344. +package generation + +import ( + "context" + "fmt" + "net" + "os" + "strings" + "time" + + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/runtime/cmdutil" + "github.com/weka/weka-operator/internal/runtime/config" +) + +const ( + wekaK8sRuntimeDir = "/opt/weka/k8s-runtime" + generationPath = "/opt/weka/k8s-runtime/runtime-generation" + persistencyMarker = "/opt/weka/k8s-runtime/persistency-configured" + persistBindsDir = "/host-binds/opt-weka" +) + +// currentGeneration is set once at program start as a float-like string (matching Python str(time.time())). +var currentGeneration = fmt.Sprintf("%f", float64(time.Now().UnixNano())/1e9) + +// Write waits for persistency to be configured (if needed), then writes the current generation. +// Mirrors Python write_generation() at weka_runtime.py:3290. +func Write(ctx context.Context, _ *config.Config) error { + _, logger := instrumentation.CreateLogSpan(ctx, "generation.Write") + defer logger.End() + + // Wait while /host-binds/opt-weka exists but persistency is not yet configured. + if err := cmdutil.PollUntil(ctx, 1*time.Second, func() bool { + _, errBinds := os.Stat(persistBindsDir) + _, errMarker := os.Stat(persistencyMarker) + if os.IsNotExist(errBinds) || errMarker == nil { + return true + } + logger.Info("Waiting for persistency to be configured") + return false + }); err != nil { + return fmt.Errorf("generation.Write: waiting for persistency: %w", err) + } + + logger.Info("Writing generation", "generation", currentGeneration) + if err := os.MkdirAll(wekaK8sRuntimeDir, 0o755); err != nil { + return fmt.Errorf("generation.Write mkdir: %w", err) + } + if err := os.WriteFile(generationPath, []byte(currentGeneration), 0o644); err != nil { + return fmt.Errorf("generation.Write: %w", err) + } + return nil +} + +// ObtainLock binds an abstract-namespace UNIX socket to provide an exclusive runtime lock. +// Mirrors Python obtain_lock() at weka_runtime.py:3312. +func ObtainLock(name string) (net.PacketConn, error) { + return net.ListenPacket("unixgram", "\x00weka_runtime_"+name) +} + +// IsWrongGeneration returns true when the on-disk generation differs from the current process. +// Mirrors Python is_wrong_generation() at weka_runtime.py:4325. +func IsWrongGeneration(cfg *config.Config) bool { + switch cfg.Mode { + case "drivers-loader", "discovery", "drivers-builder": + return false + } + + content, err := os.ReadFile(generationPath) + if err != nil || len(content) == 0 { + return false + } + onDisk := strings.TrimSpace(string(content)) + if onDisk == currentGeneration { + return false + } + // Log at error level (non-fatal — caller decides what to do). + fmt.Fprintf(os.Stderr, "generation mismatch: expected %s got %s\n", currentGeneration, onDisk) + return true +} + +// ReadBootID reads /proc/sys/kernel/random/boot_id. +// Returns empty string on error (non-fatal: callers treat empty as unknown). +func ReadBootID() string { + content, err := os.ReadFile("/proc/sys/kernel/random/boot_id") + if err != nil { + fmt.Fprintf(os.Stderr, "generation.ReadBootID: %v\n", err) + return "" + } + return strings.TrimSpace(string(content)) +} diff --git a/internal/runtime/modes/client.go b/internal/runtime/modes/client.go index 0cd109a75..91f528d2f 100644 --- a/internal/runtime/modes/client.go +++ b/internal/runtime/modes/client.go @@ -2,9 +2,14 @@ package modes import ( "context" - "fmt" + "github.com/weka/weka-operator/internal/runtime/agent" "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/cpuaffinity" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/persistency" + "github.com/weka/weka-operator/internal/runtime/ports" + "github.com/weka/weka-operator/internal/runtime/weka" ) func init() { @@ -12,5 +17,48 @@ func init() { } func runClient(ctx context.Context, cfg *config.Config) error { - return fmt.Errorf("mode %q not yet implemented", cfg.Mode) + if err := persistency.Configure(ctx, cfg); err != nil { + return err + } + if err := ports.AllocateClientPorts(ctx, cfg); err != nil { + return err + } + res, loadErr := loadResources(ctx, cfg) + if loadErr != nil { + return loadErr + } + if err := network.WriteManagementIPs(ctx, cfg); err != nil { + return err + } + lock, lockErr := runGenerationAndLock(ctx, cfg) + if lockErr != nil { + return lockErr + } + defer lock.Close() //nolint:errcheck // generation lock: close error on exit is not actionable + if err := waitForFrontendDisconnect(ctx, cfg.Name); err != nil { + return err + } + if err := agent.EnsureDrivers(ctx, cfg); err != nil { + return err + } + if err := runAgent(ctx, cfg); err != nil { + return err + } + if err := weka.EnsureWekaVersion(ctx); err != nil { + return err + } + if err := weka.EnsureWekaContainer(ctx, cfg, res); err != nil { + return err + } + if err := weka.ConfigureTraces(ctx, cfg, cfg.Name); err != nil { + return err + } + if err := startAndVerifyContainer(ctx, cfg); err != nil { + return err + } + if err := weka.WriteFeatureFlagsJSON(ctx, cfg); err != nil { + return err + } + go cpuaffinity.NewManager(cfg).RunPeriodic(ctx) + return runShutdownLoop(ctx, cfg) } diff --git a/internal/runtime/modes/compute.go b/internal/runtime/modes/compute.go index 58e8fed21..bd0cd74ea 100644 --- a/internal/runtime/modes/compute.go +++ b/internal/runtime/modes/compute.go @@ -2,9 +2,14 @@ package modes import ( "context" - "fmt" + "github.com/weka/weka-operator/internal/runtime/agent" "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/cpuaffinity" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/persistency" + "github.com/weka/weka-operator/internal/runtime/ports" + "github.com/weka/weka-operator/internal/runtime/weka" ) func init() { @@ -12,5 +17,48 @@ func init() { } func runCompute(ctx context.Context, cfg *config.Config) error { - return fmt.Errorf("mode %q not yet implemented", cfg.Mode) + if err := persistency.Configure(ctx, cfg); err != nil { + return err + } + res, loadErr := loadResources(ctx, cfg) + if loadErr != nil { + return loadErr + } + if err := ports.SavePorts(ctx, cfg); err != nil { + return err + } + if err := network.WriteManagementIPs(ctx, cfg); err != nil { + return err + } + lock, lockErr := runGenerationAndLock(ctx, cfg) + if lockErr != nil { + return lockErr + } + defer lock.Close() //nolint:errcheck // generation lock: close error on exit is not actionable + if err := agent.EnsureDrivers(ctx, cfg); err != nil { + return err + } + if err := runAgent(ctx, cfg); err != nil { + return err + } + if err := weka.EnsureWekaVersion(ctx); err != nil { + return err + } + if err := weka.EnsureWekaContainer(ctx, cfg, res); err != nil { + return err + } + if err := weka.ConfigureTraces(ctx, cfg, cfg.Name); err != nil { + return err + } + if err := weka.WriteTelemetryConfigOverride(ctx); err != nil { + return err + } + if err := startAndVerifyContainer(ctx, cfg); err != nil { + return err + } + if err := weka.WriteFeatureFlagsJSON(ctx, cfg); err != nil { + return err + } + go cpuaffinity.NewManager(cfg).RunPeriodic(ctx) + return runShutdownLoop(ctx, cfg) } diff --git a/internal/runtime/modes/data_services.go b/internal/runtime/modes/data_services.go new file mode 100644 index 000000000..6501f3c38 --- /dev/null +++ b/internal/runtime/modes/data_services.go @@ -0,0 +1,63 @@ +package modes + +import ( + "context" + + "github.com/weka/weka-operator/internal/runtime/agent" + "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/persistency" + "github.com/weka/weka-operator/internal/runtime/ports" + "github.com/weka/weka-operator/internal/runtime/weka" +) + +func init() { + register("data-services", runDataServices) +} + +func runDataServices(ctx context.Context, cfg *config.Config) error { + if err := persistency.Configure(ctx, cfg); err != nil { + return err + } + res, loadErr := loadResources(ctx, cfg) + if loadErr != nil { + return loadErr + } + // Mirror Python wait_for_resources() → save_weka_ports_data() at weka_runtime.py:3644. + if err := ports.SavePorts(ctx, cfg); err != nil { + return err + } + if err := network.WriteManagementIPs(ctx, cfg); err != nil { + return err + } + lock, lockErr := runGenerationAndLock(ctx, cfg) + if lockErr != nil { + return lockErr + } + defer lock.Close() //nolint:errcheck // generation lock: close error on exit is not actionable + if err := agent.EnsureDrivers(ctx, cfg); err != nil { + return err + } + if err := runAgent(ctx, cfg); err != nil { + return err + } + if err := weka.EnsureWekaVersion(ctx); err != nil { + return err + } + // EnsureWekaContainer uses --only-dataserv-cores and --allow-mix-setting for data-services. + if err := weka.EnsureWekaContainer(ctx, cfg, res); err != nil { + return err + } + if err := weka.ConfigureTraces(ctx, cfg, cfg.Name); err != nil { + return err + } + if err := startAndVerifyContainer(ctx, cfg); err != nil { + return err + } + if err := weka.WriteFeatureFlagsJSON(ctx, cfg); err != nil { + return err + } + // No CPU affinity periodic task for data-services. + // runShutdownLoop skips the shutdown-instruction gate for data-services (see modesNeedShutdownInstruction). + return runShutdownLoop(ctx, cfg) +} diff --git a/internal/runtime/modes/drive.go b/internal/runtime/modes/drive.go index a3e5b5b76..887b60c25 100644 --- a/internal/runtime/modes/drive.go +++ b/internal/runtime/modes/drive.go @@ -2,9 +2,17 @@ package modes import ( "context" - "fmt" + "time" + "github.com/weka/weka-operator/internal/runtime/agent" "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/cpuaffinity" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/persistency" + "github.com/weka/weka-operator/internal/runtime/ports" + "github.com/weka/weka-operator/internal/runtime/shutdown" + "github.com/weka/weka-operator/internal/runtime/weka" + "github.com/weka/weka-operator/internal/runtime/wekadrive" ) func init() { @@ -12,5 +20,51 @@ func init() { } func runDrive(ctx context.Context, cfg *config.Config) error { - return fmt.Errorf("mode %q not yet implemented", cfg.Mode) + if err := persistency.Configure(ctx, cfg); err != nil { + return err + } + res, loadErr := loadResources(ctx, cfg) + if loadErr != nil { + return loadErr + } + if err := ports.SavePorts(ctx, cfg); err != nil { + return err + } + if err := network.WriteManagementIPs(ctx, cfg); err != nil { + return err + } + lock, lockErr := runGenerationAndLock(ctx, cfg) + if lockErr != nil { + return lockErr + } + defer lock.Close() //nolint:errcheck // generation lock: close error on exit is not actionable + if err := agent.EnsureDrivers(ctx, cfg); err != nil { + return err + } + if err := runAgent(ctx, cfg); err != nil { + return err + } + if err := weka.EnsureWekaVersion(ctx); err != nil { + return err + } + if err := weka.EnsureWekaContainer(ctx, cfg, res); err != nil { + return err + } + if err := weka.ConfigureTraces(ctx, cfg, cfg.Name); err != nil { + return err + } + if err := startAndVerifyContainer(ctx, cfg); err != nil { + return err + } + if err := weka.WriteFeatureFlagsJSON(ctx, cfg); err != nil { + return err + } + go cpuaffinity.NewManager(cfg).RunPeriodic(ctx) + if err := wekadrive.EnsureDrives(ctx, cfg); err != nil { + return err + } + if err := runShutdownLoop(ctx, cfg); err != nil { + return err + } + return shutdown.WaitForDriveRelease(ctx, cfg.Drives, 60*time.Second) } diff --git a/internal/runtime/modes/drivers_builder.go b/internal/runtime/modes/drivers_builder.go index b2a8ed95b..4d8ea785b 100644 --- a/internal/runtime/modes/drivers_builder.go +++ b/internal/runtime/modes/drivers_builder.go @@ -3,14 +3,109 @@ package modes import ( "context" "fmt" + "net/http" + "os" + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/runtime/cmdutil" "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/drivers" + "github.com/weka/weka-operator/internal/runtime/results" ) func init() { register("drivers-builder", runDriversBuilder) } +type builderResult struct { + DriverBuilt bool `json:"driver_built"` + Err string `json:"err"` + WekaVersion string `json:"weka_version"` + KernelBuildID string `json:"kernel_build_id"` + KernelSignature string `json:"kernel_signature"` + WekaPackNotSupported bool `json:"weka_pack_not_supported"` + NoWekaDriversHandling bool `json:"no_weka_drivers_handling"` +} + func runDriversBuilder(ctx context.Context, cfg *config.Config) error { - return fmt.Errorf("mode %q not yet implemented", cfg.Mode) + ctx, logger := instrumentation.CreateLogSpan(ctx, "modes.runDriversBuilder") + defer logger.End() + + version, err := drivers.GetWekaVersion() + if err != nil { + return fmt.Errorf("runDriversBuilder: get weka version: %w", err) + } + logger.Info("building drivers", "version", version) + + versionGetCmd := fmt.Sprintf( + "weka version get --driver-only --without-agent --no-progress-bar --from file://shared-weka-version/opt-weka %s", + version, + ) + if runErr := cmdutil.Run(ctx, "sh", "-c", versionGetCmd); runErr != nil { + return fmt.Errorf("runDriversBuilder: weka version get: %w", runErr) + } + + kernelBuildID, err := drivers.KernelBuildID(cfg.DriversBuildID, cfg.DistService) + if err != nil { + return fmt.Errorf("runDriversBuilder: %w", err) + } + + packCmd := fmt.Sprintf("weka driver pack --without-agent --version %s", version) + if kernelBuildID != "" { + packCmd += " --kernel-build-id " + kernelBuildID + } + if runErr := cmdutil.Run(ctx, "sh", "-c", packCmd); runErr != nil { + return fmt.Errorf("runDriversBuilder: weka driver pack: %w", runErr) + } + + if mkdirErr := os.MkdirAll("/opt/weka/dist", 0o755); mkdirErr != nil { + return fmt.Errorf("runDriversBuilder: mkdir /opt/weka/dist: %w", mkdirErr) + } + // v1 symlink makes GET /dist/v1/drivers/... resolve to /opt/weka/dist/drivers/... + _ = os.Remove("/opt/weka/dist/v1") //nolint:errcheck // best-effort: absent is fine, Symlink below handles IsExist + if symlinkErr := os.Symlink("/opt/weka/dist", "/opt/weka/dist/v1"); symlinkErr != nil && !os.IsExist(symlinkErr) { + return fmt.Errorf("runDriversBuilder: symlink v1: %w", symlinkErr) + } + + kernelSig, err := drivers.KernelSignature("/opt/weka/dist/drivers") + if err != nil { + return fmt.Errorf("runDriversBuilder: %w", err) + } + + res := builderResult{ + DriverBuilt: true, + Err: "", + WekaVersion: version, + KernelBuildID: kernelBuildID, + KernelSignature: kernelSig, + WekaPackNotSupported: false, + NoWekaDriversHandling: !drivers.WekaDriversHandling(cfg.ImageName), + } + if err := results.Write(res); err != nil { + return fmt.Errorf("runDriversBuilder: write results: %w", err) + } + + port := cfg.Port + if port == 0 { + port = 60002 + } + logger.Info("starting HTTP file server", "port", port) + + srv := &http.Server{ + Addr: fmt.Sprintf(":%d", port), + Handler: http.FileServer(http.Dir("/opt/weka")), + } + + serverErr := make(chan error, 1) + go func() { + serverErr <- srv.ListenAndServe() + }() + + select { + case <-ctx.Done(): + _ = srv.Shutdown(context.Background()) //nolint:errcheck // best-effort graceful shutdown on context cancellation + return nil + case err := <-serverErr: + return fmt.Errorf("runDriversBuilder: http server: %w", err) + } } diff --git a/internal/runtime/modes/drivers_dist.go b/internal/runtime/modes/drivers_dist.go index e64587da1..77235d70b 100644 --- a/internal/runtime/modes/drivers_dist.go +++ b/internal/runtime/modes/drivers_dist.go @@ -2,9 +2,16 @@ package modes import ( "context" - "fmt" + "path/filepath" + "strings" + "time" + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/runtime/cmdutil" "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/persistency" + "github.com/weka/weka-operator/internal/runtime/weka" ) func init() { @@ -12,5 +19,84 @@ func init() { } func runDriversDist(ctx context.Context, cfg *config.Config) error { - return fmt.Errorf("mode %q not yet implemented", cfg.Mode) + ctx, logger := instrumentation.CreateLogSpan(ctx, "modes.runDriversDist") + defer logger.End() + + if err := persistency.Configure(ctx, cfg); err != nil { + return err + } + if _, err := loadResources(ctx, cfg); err != nil { + return err + } + if err := network.WriteManagementIPs(ctx, cfg); err != nil { + return err + } + lock, err := runGenerationAndLock(ctx, cfg) + if err != nil { + return err + } + defer lock.Close() //nolint:errcheck // generation lock: close error on exit is not actionable + + // EnsureDrivers is intentionally skipped: drivers-dist is on the special_modes list. + if err := runAgent(ctx, cfg); err != nil { + return err + } + if err := weka.EnsureWekaVersion(ctx); err != nil { + return err + } + + if err := weka.EnsureStemContainer(ctx, "dist", cfg.Port); err != nil { + return err + } + + // Mirror Python: fatal on configure_traces failure (weka_runtime.py). + if err := weka.ConfigureTraces(ctx, cfg, "dist"); err != nil { + return err + } + + if err := weka.StartStemContainer(ctx); err != nil { + return err + } + + cleanupTracesAndStopDumper(ctx) + + // Python process exits here; "dist" container continues independently. + return nil +} + +// cleanupTracesAndStopDumper waits for supervisorctl to start inside the dist container, +// stops the trace dumper, and removes stale shard files. +// Mirrors Python cleanup_traces_and_stop_dumper() at weka_runtime.py:3072. +// All errors are non-fatal: logged and execution continues. +func cleanupTracesAndStopDumper(ctx context.Context) { + _, logger := instrumentation.CreateLogSpan(ctx, "modes.cleanupTracesAndStopDumper") + defer logger.End() + + for { + out, err := cmdutil.Output(ctx, "sh", "-c", "weka local exec --container dist supervisorctl status 2>/dev/null") + if err != nil { + logger.Warn("supervisorctl status check failed, will retry", "err", err) + } else if strings.Contains(string(out), "RUNNING") { + break + } + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } + + if err := cmdutil.Run(ctx, "sh", "-c", "weka local exec --container dist supervisorctl stop weka-trace-dumper"); err != nil { + logger.Warn("stop weka-trace-dumper failed (non-fatal)", "err", err) + } + + shards, err := filepath.Glob("/opt/weka/traces/*.shard") + if err != nil { + logger.Warn("failed to glob shard files (non-fatal)", "err", err) + } + for _, s := range shards { + if err := cmdutil.Run(ctx, "rm", "-f", s); err != nil { + logger.Warn("failed to remove shard (non-fatal)", "path", s, "err", err) + } + } } diff --git a/internal/runtime/modes/drivers_loader.go b/internal/runtime/modes/drivers_loader.go index 875d68851..a05ae297d 100644 --- a/internal/runtime/modes/drivers_loader.go +++ b/internal/runtime/modes/drivers_loader.go @@ -3,14 +3,233 @@ package modes import ( "context" "fmt" + "os" + "strings" + "time" + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/pkg/osinfo" + "github.com/weka/weka-operator/internal/runtime/agent" + "github.com/weka/weka-operator/internal/runtime/cmdutil" "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/drivers" + "github.com/weka/weka-operator/internal/runtime/results" ) func init() { register("drivers-loader", runDriversLoader) } +type loaderResult struct { + Err interface{} `json:"err"` + DriversLoaded bool `json:"drivers_loaded"` +} + func runDriversLoader(ctx context.Context, cfg *config.Config) error { - return fmt.Errorf("mode %q not yet implemented", cfg.Mode) + ctx, logger := instrumentation.CreateLogSpan(ctx, "modes.runDriversLoader") + defer logger.End() + + if err := agent.OverrideDependenciesFlag(ctx, cfg); err != nil { + return err + } + + deadline := time.Now().Add(120 * time.Second) + + if err := drivers.DisableDriverSigning(ctx, cfg.COSAllowDisableDriverSign); err != nil { + logger.Warn("DisableDriverSigning failed", "err", err) + } + + if err := drivers.SetupOverlayfsForLibModules(ctx); err != nil { + logger.Error(err, "failed to set up overlayfs") + writeLoaderResult(logger, loaderResult{ + Err: fmt.Sprintf("Failed to set up overlayfs: %v", err), + DriversLoaded: false, + }) + return nil + } + + for time.Now().Before(deadline) { + if err := loadDrivers(ctx, cfg); err != nil { + time.Sleep(5 * time.Second) + if time.Now().After(deadline) { + writeLoaderResult(logger, loaderResult{Err: err.Error(), DriversLoaded: false}) + return nil + } + logger.Warn("failed to load drivers, retrying", "err", err) + continue + } + writeLoaderResult(logger, loaderResult{Err: nil, DriversLoaded: true}) + logger.Info("drivers loaded successfully") + return nil + } + + writeLoaderResult(logger, loaderResult{Err: "Failed to load drivers within timeout", DriversLoaded: false}) + return nil +} + +// writeLoaderResult writes the loader result.json and logs (rather than swallows) any write error. +// The result is the controller's only signal of loader outcome, so a write failure must be visible. +func writeLoaderResult(logger *instrumentation.SpanLogger, res loaderResult) { + if err := results.Write(res); err != nil { + logger.Warn("failed to write loader result.json", "err", err) + } +} + +func loadDrivers(ctx context.Context, cfg *config.Config) error { + ctx, logger := instrumentation.CreateLogSpan(ctx, "modes.loadDrivers") + defer logger.End() + + // RHCOS ships kernel modules separately; copy them into the overlay first. + if _, err := os.Stat("/hostpath/lib/modules"); err == nil { + if err := cmdutil.Run(ctx, "sh", "-c", "cp -r /hostpath/lib/modules/* /lib/modules/"); err != nil { + logger.Warn("failed to copy RHCOS kernel modules (non-fatal)", "err", err) + } + } + + wekaDriversHandling := drivers.WekaDriversHandling(cfg.ImageName) + + if !wekaDriversHandling { + return loadDriversLegacy(ctx, cfg) + } + return loadDriversNew(ctx, cfg) +} + +func loadDriversLegacy(ctx context.Context, cfg *config.Config) error { + if err := os.MkdirAll("/opt/weka/dist/drivers", 0o755); err != nil { + return err + } + + nodeInfo, err := osinfo.Load() + isCOS := err == nil && nodeInfo != nil && nodeInfo.IsCos() + + // Mirror Python should_skip_uio_pci_generic() at weka_runtime.py:1418: + // return version_params.get('uio_pci_generic') is False or should_skip_uio() + // should_skip_uio() = is_google_cos() + skipUIO := drivers.ResolveVersionParams(cfg.ImageName).ShouldSkipUioPciGeneric() || isCOS + + driverFiles := []string{ + "weka_driver-wekafsgw-*.ko", + "weka_driver-wekafsio-*.ko", + "mpin_user-*.ko", + } + // igb_uio is only available on non-COS systems. + if !isCOS { + driverFiles = append(driverFiles, "igb_uio-*.ko") + } + // uio_pci_generic is skipped when version params say so OR on COS. + if !skipUIO { + driverFiles = append(driverFiles, "uio_pci_generic-*.ko") + } + + for _, df := range driverFiles { + url := fmt.Sprintf("%s/dist/v1/drivers/%s", cfg.DistService, df) + dst := fmt.Sprintf("/opt/weka/dist/drivers/%s", df) + if err := cmdutil.Run(ctx, "sh", "-c", fmt.Sprintf("curl -kfo %s %s", dst, url)); err != nil { + return fmt.Errorf("download %s: %w", df, err) + } + } + + driverPairs := []struct{ name, pattern string }{ + {"wekafsio", "weka_driver-wekafsio-*.ko"}, + {"wekafsgw", "weka_driver-wekafsgw-*.ko"}, + {"mpin_user", "mpin_user-*.ko"}, + } + // igb_uio: non-COS only (unrelated to uio_pci_generic gating). + if !isCOS { + driverPairs = append(driverPairs, + struct{ name, pattern string }{"igb_uio", "igb_uio-*.ko"}, + ) + } + // uio_pci_generic: gated by skipUIO (version params + COS). + if !skipUIO { + driverPairs = append(driverPairs, + struct{ name, pattern string }{"uio_pci_generic", "uio_pci_generic-*.ko"}, + ) + } + + for _, dp := range driverPairs { + if err := cmdutil.Run(ctx, "sh", "-c", fmt.Sprintf("lsmod | grep -w %s", dp.name)); err == nil { + continue // already loaded + } + if err := cmdutil.Run(ctx, "sh", "-c", + fmt.Sprintf("insmod /opt/weka/dist/drivers/%s", dp.pattern)); err != nil { + return fmt.Errorf("insmod %s: %w", dp.name, err) + } + } + + drivers.LoadModules(ctx, skipUIO) + return nil +} + +func loadDriversNew(ctx context.Context, cfg *config.Config) error { + version, err := drivers.GetWekaVersion() + if err != nil { + return err + } + + kernelBuildID, err := drivers.KernelBuildID(cfg.DriversBuildID, cfg.DistService) + if err != nil { + return err + } + + // When the runtime image differs from the version image, weka binaries live on a shared volume. + fromPath := "" + if cfg.TargetImageName != "" && cfg.TargetImageName != cfg.ImageName { + fromPath = "file://shared-weka-version/opt-weka" + } + + if fromPath != "" { + versionGetCmd := fmt.Sprintf( + "weka version get --without-agent --driver-only --from %s %s", + fromPath, version, + ) + err = cmdutil.Run(ctx, "sh", "-c", versionGetCmd) + if err != nil { + return fmt.Errorf("loadDriversNew: weka version get: %w", err) + } + } + + downloadArgs := buildWekaDriverArgs("download", cfg.DistService, version, kernelBuildID) + err = cmdutil.Run(ctx, "sh", "-c", downloadArgs) + if err != nil { + return fmt.Errorf("loadDriversNew: weka driver download: %w", err) + } + + // Unload any previously installed weka drivers — ignore errors if not loaded. + _ = cmdutil.Run(ctx, "rmmod", "wekafsio") //nolint:errcheck // best-effort: error expected when module is not loaded + _ = cmdutil.Run(ctx, "rmmod", "wekafsgw") //nolint:errcheck // best-effort: error expected when module is not loaded + + installArgs := buildWekaDriverArgs("install", "", version, kernelBuildID) + err = cmdutil.Run(ctx, "sh", "-c", installArgs) + if err != nil { + return fmt.Errorf("loadDriversNew: weka driver install: %w", err) + } + + nodeInfo, err := osinfo.Load() + isCOS := err == nil && nodeInfo != nil && nodeInfo.IsCos() + // Mirror Python should_skip_uio_pci_generic() at weka_runtime.py:1418. + skipUIO := drivers.ResolveVersionParams(cfg.ImageName).ShouldSkipUioPciGeneric() || isCOS + drivers.LoadModules(ctx, skipUIO) + return nil +} + +// buildWekaDriverArgs returns a shell command string for "weka driver ". +// distService is only used for "download"; empty means it is omitted. +func buildWekaDriverArgs(subcmd, distService, version, kernelBuildID string) string { + var sb strings.Builder + sb.WriteString("weka driver ") + sb.WriteString(subcmd) + if distService != "" { + sb.WriteString(" --from '") + sb.WriteString(distService) + sb.WriteString("'") + } + sb.WriteString(" --without-agent") + sb.WriteString(" --version ") + sb.WriteString(version) + if kernelBuildID != "" { + sb.WriteString(" --kernel-build-id ") + sb.WriteString(kernelBuildID) + } + return sb.String() } diff --git a/internal/runtime/modes/envoy.go b/internal/runtime/modes/envoy.go index 7ec24d0d1..640fd1f12 100644 --- a/internal/runtime/modes/envoy.go +++ b/internal/runtime/modes/envoy.go @@ -2,9 +2,13 @@ package modes import ( "context" - "fmt" + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/runtime/cmdutil" "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/persistency" + "github.com/weka/weka-operator/internal/runtime/weka" ) func init() { @@ -12,5 +16,45 @@ func init() { } func runEnvoy(ctx context.Context, cfg *config.Config) error { - return fmt.Errorf("mode %q not yet implemented", cfg.Mode) + ctx, logger := instrumentation.CreateLogSpan(ctx, "modes.runEnvoy") + defer logger.End() + + if err := persistency.Configure(ctx, cfg); err != nil { + return err + } + if _, err := loadResources(ctx, cfg); err != nil { + return err + } + if err := network.WriteManagementIPs(ctx, cfg); err != nil { + return err + } + lock, err := runGenerationAndLock(ctx, cfg) + if err != nil { + return err + } + defer lock.Close() //nolint:errcheck // generation lock: close error on exit is not actionable + + // EnsureDrivers intentionally skipped — envoy is a sidecar, not a driver container. + // agent.Configure (called inside runAgent) already appends envoy-data to conditional_mounts_ids + // and adds skip_envoy_setup=true for s3 cooperating pods (handled per cfg.Mode there). + if err := runAgent(ctx, cfg); err != nil { + return err + } + if err := weka.EnsureWekaVersion(ctx); err != nil { + return err + } + + if err := ensureEnvoyContainer(ctx); err != nil { + return err + } + + logger.Info("envoy container ready; exiting — envoy and agent continue independently") + return nil +} + +// ensureEnvoyContainer creates the envoy container if it does not already exist. +// Mirrors Python ensure_envoy_container() at weka_runtime.py. +func ensureEnvoyContainer(ctx context.Context) error { + return cmdutil.Run(ctx, "sh", "-c", + "weka local ps | grep -qw envoy || weka local setup envoy") } diff --git a/internal/runtime/modes/lifecycle.go b/internal/runtime/modes/lifecycle.go new file mode 100644 index 000000000..8de0a35ac --- /dev/null +++ b/internal/runtime/modes/lifecycle.go @@ -0,0 +1,348 @@ +// lifecycle.go provides shared mode helpers used by compute, drive, and client modes. +package modes + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "strings" + "sync/atomic" + "time" + + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/runtime/agent" + "github.com/weka/weka-operator/internal/runtime/cmdutil" + "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/daemon" + "github.com/weka/weka-operator/internal/runtime/generation" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/resources" + "github.com/weka/weka-operator/internal/runtime/shutdown" + "github.com/weka/weka-operator/internal/runtime/syslog" + "github.com/weka/weka-operator/internal/runtime/weka" +) + +// forceStopPollInterval is the poll cadence for the allow_force_stop watcher. +// Matches the shutdown-instruction poll interval in shutdown.go. +// A var (not const) so tests can lower it. +var forceStopPollInterval = 5 * time.Second + +// Test seams for the force-stop watcher; default to real implementations. +var ( + getShutdownInstructionsFn = shutdown.GetShutdownInstructions + forceStopFn = func(cfg *config.Config) error { + return cmdutil.Run(context.Background(), "sh", "-c", + fmt.Sprintf("weka local stop %s --force", cfg.Name)) + } +) + +// loadResources waits for node resources to become available, loads them, and +// populates cfg with the resource values. +func loadResources(ctx context.Context, cfg *config.Config) (*resources.NodeResources, error) { + bootID := shutdown.GetBootID() + abort := func() bool { + return shutdown.GetShutdownInstructions(cfg.PodID, bootID).AllowStop + } + res, err := resources.WaitAndLoad(ctx, abort) + if err != nil { + return nil, err + } + updateCfgFromResources(cfg, res) + return res, nil +} + +// runGenerationAndLock writes the generation file and acquires the generation lock. +// The caller must defer lock.Close(). +// +// L2 ordering note: Python order is write_generation → write_management_ips → obtain_lock +// (weka_runtime.py ~:4140-4143). Go bundles write+lock here and mode files call +// WriteManagementIPs before runGenerationAndLock, giving order: +// +// write_management_ips → write_generation → obtain_lock. +// +// Both orderings are pre-agent and functionally equivalent. Splitting runGenerationAndLock +// across 7 mode files is invasive for a low-priority ordering difference; left as-is. +func runGenerationAndLock(ctx context.Context, cfg *config.Config) (io.Closer, error) { + if err := generation.Write(ctx, cfg); err != nil { + return nil, err + } + lock, err := generation.ObtainLock(cfg.Name) + if err != nil { + return nil, err + } + return lock, nil +} + +// runAgent configures the weka agent, starts the agent supervisor, and waits until ready. +// +// L3 ordering note: Python order is configure_agent → start_syslog → override_dependencies_flag → +// ensure_drivers → start_agent. Mode files call agent.EnsureDrivers BEFORE runAgent, meaning +// driver-detection logs are not forwarded via syslog. Reordering would require splitting runAgent +// across 7 mode files (invasive for a syslog-forwarding benefit only). Left unchanged — EnsureDrivers +// uses --without-agent and does not functionally depend on agent.Configure being done first. +func runAgent(ctx context.Context, cfg *config.Config) error { + if err := agent.Configure(ctx, cfg, false); err != nil { + return err + } + if err := agent.OverrideDependenciesFlag(ctx, cfg); err != nil { + return err + } + startAgentSupervisor(ctx, cfg) + return agent.AwaitReady(ctx, cfg) +} + +// startAndVerifyContainer starts the weka container and verifies it is executing. +func startAndVerifyContainer(ctx context.Context, cfg *config.Config) error { + if err := weka.StartContainer(ctx, cfg.Name); err != nil { + return err + } + return weka.EnsureContainerExec(ctx, cfg.Name) +} + +// watchForceStop mirrors Python watch_for_force_shutdown() at weka_runtime.py:4497. +// During a graceful stop it polls for allow_force_stop and escalates to a force stop. +func watchForceStop(ctx context.Context, cfg *config.Config, bootID string) { + _, logger := instrumentation.CreateLogSpan(ctx, "modes.watchForceStop") + defer logger.End() + for { + if getShutdownInstructionsFn(cfg.PodID, bootID).AllowForceStop { + logger.Info("received allow-force-stop instruction, escalating to force stop") + if err := forceStopFn(cfg); err != nil { + logger.Warn("force stop command failed", "err", err) + } + return + } + select { + case <-ctx.Done(): + return + case <-time.After(forceStopPollInterval): + } + } +} + +// runShutdownLoop is shared by all backend/client modes that own a weka container +// (compute, drive, client, s3, nfs, smbw, data-services). +// It watches for generation mismatch or ctx cancellation, then orchestrates graceful/force stop. +// Mirrors Python shutdown() at weka_runtime.py:4524. +func runShutdownLoop(ctx context.Context, cfg *config.Config) error { + _, logger := instrumentation.CreateLogSpan(ctx, "modes.runShutdownLoop", "mode", cfg.Mode) + defer logger.End() + + bootID := shutdown.GetBootID() + + // Watch for generation mismatch; cancel watchCtx when detected. + // generationMismatch is atomic: the watcher goroutine may still be live (and may + // write it) when the main goroutine reads it after a parent-ctx cancellation, so the + // two accesses are otherwise unsynchronized. + watchCtx, watchCancel := context.WithCancel(ctx) + defer watchCancel() + var generationMismatch atomic.Bool + go func() { + t := time.NewTicker(time.Second) + defer t.Stop() + for { + select { + case <-watchCtx.Done(): + return + case <-t.C: + if generation.IsWrongGeneration(cfg) { + logger.Info("generation mismatch detected, initiating shutdown") + generationMismatch.Store(true) + watchCancel() + } + } + } + }() + + // Block until watchCtx is cancelled (generation mismatch or parent ctx done). + <-watchCtx.Done() + + logger.Warn("shutdown initiated") + + // Determine stop flag, mirroring Python weka_runtime.py:4544–4551. + // + // Compute allow_force_stop via the same I/O Python does, then delegate the flag + // choice to computeStopFlag (the pure mirror of the Python branch logic): + // - generation mismatch → force path; no instruction read. + // - data-services → non-blocking AllowForceStop check (no instruction wait). + // - instruction-gated modes → block on PollShutdownInstructions; allow_force_stop = !graceful. + wrongGeneration := generationMismatch.Load() + allowForceStop := false + switch { + case wrongGeneration: + // Force path; instruction read skipped. allowForceStop is unused by computeStopFlag here. + case cfg.Mode == "data-services": + // Non-blocking read, matching Python data-services teardown. + allowForceStop = getShutdownInstructionsFn(cfg.PodID, bootID).AllowForceStop + case modesNeedShutdownInstruction[cfg.Mode]: + // Blocks until the operator permits a stop; graceful=true → allow_stop, false → allow_force_stop. + graceful := shutdown.PollShutdownInstructions(cfg.PodID, bootID) + allowForceStop = !graceful + } + + stopFlag := computeStopFlag(cfg.Mode, allowForceStop, wrongGeneration) + forceStop := stopFlag == "--force" + + stopWatchCtx, stopWatchCancel := context.WithCancel(context.Background()) + defer stopWatchCancel() + if !forceStop { + go watchForceStop(stopWatchCtx, cfg, bootID) + } + + for isContainerRunning(cfg.Name, forceStop) { + if err := cmdutil.Run(context.Background(), "sh", "-c", + fmt.Sprintf("timeout 180 weka local stop %s %s", cfg.Name, stopFlag)); err != nil { + logger.Warn("weka local stop failed, will retry", "flag", stopFlag, "err", err) + } + time.Sleep(3 * time.Second) + } + stopWatchCancel() + + return nil +} + +// updateCfgFromResources copies NodeResources fields into cfg. +// Mirrors the global assignments in Python wait_for_resources() at weka_runtime.py:3636–3643. +func updateCfgFromResources(cfg *config.Config, res *resources.NodeResources) { + // Mirror Python: if parse_port(PORT)==0 and MODE not in ['envoy','telemetry']: PORT = data["wekaPort"] + if cfg.Port == 0 && res.WekaPort != 0 && cfg.Mode != "envoy" && cfg.Mode != "telemetry" { + cfg.Port = res.WekaPort + } + // Mirror Python: if parse_port(AGENT_PORT)==0 and MODE != 'telemetry': AGENT_PORT = data["agentPort"] + if cfg.AgentPort == 0 && res.AgentPort != 0 && cfg.Mode != "telemetry" { + cfg.AgentPort = res.AgentPort + } + if res.FailureDomain != "" { + cfg.FailureDomain = res.FailureDomain + } + if res.MachineIdentifier != "" { + cfg.MachineIdentifier = res.MachineIdentifier + } + // Mirror Python wait_for_resources (weka_runtime.py:3624-3626): + // net_devices = ",".join(data.get("netDevices", [])) + // if net_devices and should_allocate_vf_per_ionode(net_devices): + // NETWORK_DEVICE = net_devices + if len(res.NetDevices) > 0 { + netDevices := strings.Join(res.NetDevices, ",") + if network.ShouldAllocateVFPerIoNode(netDevices) { + cfg.NetworkDevice = netDevices + } + } + cfg.Drives = res.Drives +} + +// modesNeedShutdownInstruction is the set of modes that must wait for the operator +// shutdown-instruction gate before stopping. Mirrors Python logic at weka_runtime.py:4350. +// data-services is intentionally absent: it stops without an instruction gate. +var modesNeedShutdownInstruction = map[string]bool{ + "client": true, + "s3": true, + "nfs": true, + "smbw": true, + "drive": true, + "compute": true, +} + +// gracefulEligibleModes is the set of modes eligible for a graceful ("-g") stop. +// Verbatim list from weka_runtime.py:4549. Notably client is absent: it always force-stops. +var gracefulEligibleModes = map[string]bool{ + "s3": true, + "drive": true, + "compute": true, + "nfs": true, + "smbw": true, + "data-services": true, +} + +// computeStopFlag returns the weka local stop flag ("--force" or "-g"), +// mirroring the force_stop decision in Python shutdown() at weka_runtime.py:4544–4551: +// +// force_stop = False +// if allow_force_stop: force_stop = True +// if wrong_generation: force_stop = True +// if MODE not in [...]: force_stop = True # client is absent → always force +// +// It is pure: the caller resolves allowForceStop (blocking/non-blocking instruction +// read) and generationMismatch beforehand. +func computeStopFlag(mode string, allowForceStop, generationMismatch bool) string { + forceStop := allowForceStop || generationMismatch || !gracefulEligibleModes[mode] + if forceStop { + return "--force" + } + return "-g" +} + +// isContainerRunning checks weka local ps for the named container's run status. +// If noAgentAsNotRunning=true, treats an agent error as "not running". +// Mirrors Python is_container_running() at weka_runtime.py:4506. +func isContainerRunning(name string, noAgentAsNotRunning bool) bool { + out, err := cmdutil.Output(context.Background(), "weka", "local", "ps", "--json") + if err != nil { + return !noAgentAsNotRunning + } + var containers []map[string]interface{} + if err := json.Unmarshal(out, &containers); err != nil { + return !noAgentAsNotRunning + } + for _, c := range containers { + cName, ok := c["name"].(string) + if !ok || cName != name { + continue + } + status, ok := c["runStatus"].(string) + if ok && status == "Stopped" { + return false + } + return true + } + return false +} + +// startAgentSupervisor creates a Supervisor with syslog and agent processes and starts it. +func startAgentSupervisor(ctx context.Context, cfg *config.Config) { + sup := daemon.NewSupervisor() + syslog.AddToDaemon(sup, cfg) + agentCmd := agent.GetCmd(cfg) + sup.Add("agent", func() *exec.Cmd { + return exec.Command("sh", "-c", agentCmd) //nolint:gosec // agentCmd is operator-controlled, not user input + }) + go func() { _ = sup.Run(ctx) }() //nolint:errcheck // supervisor run error is logged internally; goroutine exit is non-fatal +} + +// waitForFrontendDisconnect polls /proc/wekafs/interface until the named container's +// frontend is no longer connected (up to 120s). +// Mirrors Python wait for frontend disconnect in client shutdown flow. +func waitForFrontendDisconnect(ctx context.Context, containerName string) error { + _, logger := instrumentation.CreateLogSpan(ctx, "modes.waitForFrontendDisconnect") + defer logger.End() + + deadline := time.Now().Add(120 * time.Second) + for { + data, err := os.ReadFile("/proc/wekafs/interface") + if err != nil { + return nil // driver not loaded → no frontend connected + } + connected := false + for _, line := range strings.Split(string(data), "\n") { + // Mirror Python line.startswith("Container=" + name) at weka_runtime.py:4425. + if strings.HasPrefix(line, "Container="+containerName) && strings.Contains(line, "Connected frontend") { + connected = true + break + } + } + if !connected { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("frontend %q still connected after 120s", containerName) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(5 * time.Second): + } + } +} diff --git a/internal/runtime/modes/lifecycle_test.go b/internal/runtime/modes/lifecycle_test.go new file mode 100644 index 000000000..c677b775c --- /dev/null +++ b/internal/runtime/modes/lifecycle_test.go @@ -0,0 +1,202 @@ +package modes + +import ( + "context" + "testing" + "time" + + "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/resources" + "github.com/weka/weka-operator/internal/runtime/shutdown" +) + +func TestUpdateCfgFromResources(t *testing.T) { + t.Run("vf net devices override existing device", func(t *testing.T) { + cfg := &config.Config{Mode: "compute", NetworkDevice: "eth0"} + res := &resources.NodeResources{NetDevices: []string{"vf_eth1", "vf_eth2"}} + updateCfgFromResources(cfg, res) + if cfg.NetworkDevice != "vf_eth1,vf_eth2" { + t.Errorf("NetworkDevice = %q, want %q", cfg.NetworkDevice, "vf_eth1,vf_eth2") + } + }) + + t.Run("non-vf net devices do not override", func(t *testing.T) { + cfg := &config.Config{NetworkDevice: "eth0"} + res := &resources.NodeResources{NetDevices: []string{"eth1"}} + updateCfgFromResources(cfg, res) + if cfg.NetworkDevice != "eth0" { + t.Errorf("NetworkDevice = %q, want %q", cfg.NetworkDevice, "eth0") + } + }) + + t.Run("weka port set for compute", func(t *testing.T) { + cfg := &config.Config{Mode: "compute", Port: 0} + res := &resources.NodeResources{WekaPort: 14000} + updateCfgFromResources(cfg, res) + if cfg.Port != 14000 { + t.Errorf("Port = %d, want 14000", cfg.Port) + } + }) + + t.Run("telemetry mode keeps ports zero", func(t *testing.T) { + cfg := &config.Config{Mode: "telemetry", Port: 0, AgentPort: 0} + res := &resources.NodeResources{WekaPort: 14000, AgentPort: 15000} + updateCfgFromResources(cfg, res) + if cfg.Port != 0 { + t.Errorf("Port = %d, want 0", cfg.Port) + } + if cfg.AgentPort != 0 { + t.Errorf("AgentPort = %d, want 0", cfg.AgentPort) + } + }) + + t.Run("envoy mode keeps weka port zero", func(t *testing.T) { + cfg := &config.Config{Mode: "envoy", Port: 0} + res := &resources.NodeResources{WekaPort: 14000} + updateCfgFromResources(cfg, res) + if cfg.Port != 0 { + t.Errorf("Port = %d, want 0", cfg.Port) + } + }) + + t.Run("failure domain and machine identifier override when non-empty", func(t *testing.T) { + cfg := &config.Config{Mode: "compute", FailureDomain: "old-fd", MachineIdentifier: "old-mid"} + res := &resources.NodeResources{FailureDomain: "new-fd", MachineIdentifier: "new-mid"} + updateCfgFromResources(cfg, res) + if cfg.FailureDomain != "new-fd" { + t.Errorf("FailureDomain = %q, want %q", cfg.FailureDomain, "new-fd") + } + if cfg.MachineIdentifier != "new-mid" { + t.Errorf("MachineIdentifier = %q, want %q", cfg.MachineIdentifier, "new-mid") + } + }) + + t.Run("empty failure domain and machine identifier do not override", func(t *testing.T) { + cfg := &config.Config{Mode: "compute", FailureDomain: "old-fd", MachineIdentifier: "old-mid"} + res := &resources.NodeResources{} + updateCfgFromResources(cfg, res) + if cfg.FailureDomain != "old-fd" { + t.Errorf("FailureDomain = %q, want %q", cfg.FailureDomain, "old-fd") + } + if cfg.MachineIdentifier != "old-mid" { + t.Errorf("MachineIdentifier = %q, want %q", cfg.MachineIdentifier, "old-mid") + } + }) +} + +// TestComputeStopFlag pins the Python force_stop decision at weka_runtime.py:4544–4551. +// client always force-stops; the graceful-eligible modes use "-g" only when neither +// allow_force_stop nor a generation mismatch is set. +func TestComputeStopFlag(t *testing.T) { + modes := []string{"client", "s3", "nfs", "smbw", "drive", "compute", "data-services"} + for _, mode := range modes { + for _, allowForceStop := range []bool{false, true} { + for _, genMismatch := range []bool{false, true} { + // Graceful "-g" is reachable only by graceful-eligible modes, and only when + // neither force signal is present. client is not graceful-eligible. + wantGraceful := gracefulEligibleModes[mode] && !allowForceStop && !genMismatch + want := "--force" + if wantGraceful { + want = "-g" + } + got := computeStopFlag(mode, allowForceStop, genMismatch) + if got != want { + t.Errorf("computeStopFlag(%q, allowForceStop=%v, genMismatch=%v) = %q, want %q", + mode, allowForceStop, genMismatch, got, want) + } + } + } + } + + // Explicit spot-checks of the most load-bearing cases. + if got := computeStopFlag("client", false, false); got != "--force" { + t.Errorf("client with no force signals = %q, want --force (client always force-stops)", got) + } + if got := computeStopFlag("compute", false, false); got != "-g" { + t.Errorf("compute with no force signals = %q, want -g", got) + } + if got := computeStopFlag("data-services", false, false); got != "-g" { + t.Errorf("data-services default = %q, want -g", got) + } + if got := computeStopFlag("compute", false, true); got != "--force" { + t.Errorf("compute on generation mismatch = %q, want --force", got) + } +} + +func TestWatchForceStop_Escalates(t *testing.T) { + origGet := getShutdownInstructionsFn + origForce := forceStopFn + defer func() { + getShutdownInstructionsFn = origGet + forceStopFn = origForce + }() + + getShutdownInstructionsFn = func(_, _ string) *shutdown.ShutdownInstructions { + return &shutdown.ShutdownInstructions{AllowForceStop: true} + } + forced := make(chan *config.Config, 1) + forceStopFn = func(cfg *config.Config) error { + forced <- cfg + return nil + } + + done := make(chan struct{}) + go func() { + watchForceStop(context.Background(), &config.Config{Name: "c0"}, "boot") + close(done) + }() + + select { + case cfg := <-forced: + if cfg.Name != "c0" { + t.Errorf("forceStopFn called with Name = %q, want %q", cfg.Name, "c0") + } + case <-time.After(time.Second): + t.Fatal("forceStopFn not called within 1s") + } + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("watchForceStop did not return within 1s after escalating") + } +} + +func TestWatchForceStop_ExitsOnCtxCancel(t *testing.T) { + origGet := getShutdownInstructionsFn + origForce := forceStopFn + origInterval := forceStopPollInterval + defer func() { + getShutdownInstructionsFn = origGet + forceStopFn = origForce + forceStopPollInterval = origInterval + }() + + forceStopPollInterval = time.Millisecond + getShutdownInstructionsFn = func(_, _ string) *shutdown.ShutdownInstructions { + return &shutdown.ShutdownInstructions{} // no force stop + } + var called bool + forceStopFn = func(_ *config.Config) error { + called = true + return nil + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // immediately cancelled + + done := make(chan struct{}) + go func() { + watchForceStop(ctx, &config.Config{Name: "c0"}, "boot") + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("watchForceStop did not return promptly on cancelled ctx") + } + if called { + t.Error("forceStopFn should not be called when there is no force-stop instruction") + } +} diff --git a/internal/runtime/modes/nfs.go b/internal/runtime/modes/nfs.go index 53e81a577..fcbe2160b 100644 --- a/internal/runtime/modes/nfs.go +++ b/internal/runtime/modes/nfs.go @@ -2,9 +2,13 @@ package modes import ( "context" - "fmt" + "github.com/weka/weka-operator/internal/runtime/agent" "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/persistency" + "github.com/weka/weka-operator/internal/runtime/ports" + "github.com/weka/weka-operator/internal/runtime/weka" ) func init() { @@ -12,5 +16,46 @@ func init() { } func runNFS(ctx context.Context, cfg *config.Config) error { - return fmt.Errorf("mode %q not yet implemented", cfg.Mode) + if err := persistency.Configure(ctx, cfg); err != nil { + return err + } + res, loadErr := loadResources(ctx, cfg) + if loadErr != nil { + return loadErr + } + // Mirror Python wait_for_resources() → save_weka_ports_data() at weka_runtime.py:3644. + if err := ports.SavePorts(ctx, cfg); err != nil { + return err + } + if err := network.WriteManagementIPs(ctx, cfg); err != nil { + return err + } + lock, lockErr := runGenerationAndLock(ctx, cfg) + if lockErr != nil { + return lockErr + } + defer lock.Close() //nolint:errcheck // generation lock: close error on exit is not actionable + if err := agent.EnsureDrivers(ctx, cfg); err != nil { + return err + } + if err := runAgent(ctx, cfg); err != nil { + return err + } + if err := weka.EnsureWekaVersion(ctx); err != nil { + return err + } + // EnsureWekaContainer sets allow_protocols=true for nfs. + if err := weka.EnsureWekaContainer(ctx, cfg, res); err != nil { + return err + } + if err := weka.ConfigureTraces(ctx, cfg, cfg.Name); err != nil { + return err + } + if err := startAndVerifyContainer(ctx, cfg); err != nil { + return err + } + if err := weka.WriteFeatureFlagsJSON(ctx, cfg); err != nil { + return err + } + return runShutdownLoop(ctx, cfg) } diff --git a/internal/runtime/modes/s3.go b/internal/runtime/modes/s3.go index caccea5b2..1bc774be2 100644 --- a/internal/runtime/modes/s3.go +++ b/internal/runtime/modes/s3.go @@ -2,9 +2,13 @@ package modes import ( "context" - "fmt" + "github.com/weka/weka-operator/internal/runtime/agent" "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/persistency" + "github.com/weka/weka-operator/internal/runtime/ports" + "github.com/weka/weka-operator/internal/runtime/weka" ) func init() { @@ -12,5 +16,47 @@ func init() { } func runS3(ctx context.Context, cfg *config.Config) error { - return fmt.Errorf("mode %q not yet implemented", cfg.Mode) + if err := persistency.Configure(ctx, cfg); err != nil { + return err + } + res, loadErr := loadResources(ctx, cfg) + if loadErr != nil { + return loadErr + } + // Mirror Python wait_for_resources() → save_weka_ports_data() at weka_runtime.py:3644. + if err := ports.SavePorts(ctx, cfg); err != nil { + return err + } + if err := network.WriteManagementIPs(ctx, cfg); err != nil { + return err + } + lock, lockErr := runGenerationAndLock(ctx, cfg) + if lockErr != nil { + return lockErr + } + defer lock.Close() //nolint:errcheck // generation lock: close error on exit is not actionable + if err := agent.EnsureDrivers(ctx, cfg); err != nil { + return err + } + // agent.Configure (inside runAgent) adds skip_envoy_setup=true and envoy-data mount for s3. + if err := runAgent(ctx, cfg); err != nil { + return err + } + if err := weka.EnsureWekaVersion(ctx); err != nil { + return err + } + // EnsureWekaContainer sets allow_protocols=true for s3. + if err := weka.EnsureWekaContainer(ctx, cfg, res); err != nil { + return err + } + if err := weka.ConfigureTraces(ctx, cfg, cfg.Name); err != nil { + return err + } + if err := startAndVerifyContainer(ctx, cfg); err != nil { + return err + } + if err := weka.WriteFeatureFlagsJSON(ctx, cfg); err != nil { + return err + } + return runShutdownLoop(ctx, cfg) } diff --git a/internal/runtime/modes/smbw.go b/internal/runtime/modes/smbw.go new file mode 100644 index 000000000..cf785f327 --- /dev/null +++ b/internal/runtime/modes/smbw.go @@ -0,0 +1,61 @@ +package modes + +import ( + "context" + + "github.com/weka/weka-operator/internal/runtime/agent" + "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/persistency" + "github.com/weka/weka-operator/internal/runtime/ports" + "github.com/weka/weka-operator/internal/runtime/weka" +) + +func init() { + register("smbw", runSMBW) +} + +func runSMBW(ctx context.Context, cfg *config.Config) error { + if err := persistency.Configure(ctx, cfg); err != nil { + return err + } + res, loadErr := loadResources(ctx, cfg) + if loadErr != nil { + return loadErr + } + // Mirror Python wait_for_resources() → save_weka_ports_data() at weka_runtime.py:3644. + if err := ports.SavePorts(ctx, cfg); err != nil { + return err + } + if err := network.WriteManagementIPs(ctx, cfg); err != nil { + return err + } + lock, lockErr := runGenerationAndLock(ctx, cfg) + if lockErr != nil { + return lockErr + } + defer lock.Close() //nolint:errcheck // generation lock: close error on exit is not actionable + if err := agent.EnsureDrivers(ctx, cfg); err != nil { + return err + } + if err := runAgent(ctx, cfg); err != nil { + return err + } + if err := weka.EnsureWekaVersion(ctx); err != nil { + return err + } + // EnsureWekaContainer sets allow_protocols=true for smbw. + if err := weka.EnsureWekaContainer(ctx, cfg, res); err != nil { + return err + } + if err := weka.ConfigureTraces(ctx, cfg, cfg.Name); err != nil { + return err + } + if err := startAndVerifyContainer(ctx, cfg); err != nil { + return err + } + if err := weka.WriteFeatureFlagsJSON(ctx, cfg); err != nil { + return err + } + return runShutdownLoop(ctx, cfg) +} diff --git a/internal/runtime/modes/ssdproxy.go b/internal/runtime/modes/ssdproxy.go index 090c7bd70..30995d827 100644 --- a/internal/runtime/modes/ssdproxy.go +++ b/internal/runtime/modes/ssdproxy.go @@ -3,8 +3,14 @@ package modes import ( "context" "fmt" + "os" + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/runtime/cmdutil" "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/persistency" + "github.com/weka/weka-operator/internal/runtime/weka" ) func init() { @@ -12,5 +18,74 @@ func init() { } func runSSDProxy(ctx context.Context, cfg *config.Config) error { - return fmt.Errorf("mode %q not yet implemented", cfg.Mode) + ctx, logger := instrumentation.CreateLogSpan(ctx, "modes.runSSDProxy") + defer logger.End() + + if err := persistency.Configure(ctx, cfg); err != nil { + return err + } + if _, err := loadResources(ctx, cfg); err != nil { + return err + } + if err := network.WriteManagementIPs(ctx, cfg); err != nil { + return err + } + lock, err := runGenerationAndLock(ctx, cfg) + if err != nil { + return err + } + defer lock.Close() //nolint:errcheck // generation lock: close error on exit is not actionable + + // EnsureDrivers intentionally skipped — ssdproxy is a sidecar. + if err := runAgent(ctx, cfg); err != nil { + return err + } + if err := weka.EnsureWekaVersion(ctx); err != nil { + return err + } + + if err := assertIOMMUSupported(); err != nil { + return err + } + if err := ensureSsdproxyContainer(ctx, cfg); err != nil { + return err + } + if err := weka.ForceSetWekaVersion(ctx); err != nil { + return err + } + // cfg.Mode == "ssdproxy" triggers the dedicated trace config branch in ConfigureTraces. + // Mirror Python: fatal on configure_traces failure at weka_runtime.py:2443/2469. + if err := weka.ConfigureTraces(ctx, cfg, cfg.Name); err != nil { + return err + } + + logger.Info("ssdproxy container ready; exiting — ssdproxy and agent continue independently") + return nil +} + +// assertIOMMUSupported checks that IOMMU groups are present on the host. +// Mirrors Python assert_ssdproxy_iommu_supported() at weka_runtime.py. +func assertIOMMUSupported() error { + entries, err := os.ReadDir("/sys/kernel/iommu_groups") + if err != nil { + return fmt.Errorf("IOMMU not supported: cannot read /sys/kernel/iommu_groups: %w", err) + } + if len(entries) == 0 { + return fmt.Errorf("no IOMMU groups found — IOMMU may not be enabled in BIOS or kernel cmdline") + } + return nil +} + +// ensureSsdproxyContainer creates the ssdproxy container and weka-sign-drive symlink. +// Mirrors Python ensure_ssdproxy_container() at weka_runtime.py. +func ensureSsdproxyContainer(ctx context.Context, cfg *config.Config) error { + // Mirror Python: assert MEMORY, "MEMORY is not set" at weka_runtime.py:3413-3415. + if cfg.Memory == "" { + return fmt.Errorf("ssdproxy: MEMORY is not set") + } + script := fmt.Sprintf(` +weka local ps | grep -qw ssdproxy || weka local setup ssdproxy --memory %s --base-port 13000 --enable-ssdproxy-nginx +ln -sf /opt/weka/dist/extracted/weka-sign-drive /usr/bin/weka-sign-drive +`, cfg.Memory) + return cmdutil.Run(ctx, "sh", "-c", script) } diff --git a/internal/runtime/modes/telemetry.go b/internal/runtime/modes/telemetry.go index 072f6315c..566a5c295 100644 --- a/internal/runtime/modes/telemetry.go +++ b/internal/runtime/modes/telemetry.go @@ -2,9 +2,13 @@ package modes import ( "context" - "fmt" + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/runtime/cmdutil" "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/persistency" + "github.com/weka/weka-operator/internal/runtime/weka" ) func init() { @@ -12,5 +16,49 @@ func init() { } func runTelemetry(ctx context.Context, cfg *config.Config) error { - return fmt.Errorf("mode %q not yet implemented", cfg.Mode) + ctx, logger := instrumentation.CreateLogSpan(ctx, "modes.runTelemetry") + defer logger.End() + + if err := persistency.Configure(ctx, cfg); err != nil { + return err + } + if _, err := loadResources(ctx, cfg); err != nil { + return err + } + if err := network.WriteManagementIPs(ctx, cfg); err != nil { + return err + } + lock, err := runGenerationAndLock(ctx, cfg) + if err != nil { + return err + } + defer lock.Close() //nolint:errcheck // generation lock: close error on exit is not actionable + + // EnsureDrivers intentionally skipped — telemetry is a sidecar. + if err := runAgent(ctx, cfg); err != nil { + return err + } + if err := weka.EnsureWekaVersion(ctx); err != nil { + return err + } + + if err := ensureTelemetryContainer(ctx); err != nil { + return err + } + // Mirror Python: fatal on write_telemetry_config_override failure at weka_runtime.py:3408. + // compute.go:33 already returns this error; match it here. + if err := weka.WriteTelemetryConfigOverride(ctx); err != nil { + return err + } + + logger.Info("telemetry container ready; exiting — telemetry and agent continue independently") + return nil +} + +// ensureTelemetryContainer creates the telemetry container if it does not already exist. +// --not-dependent allows it to start without waiting for other containers. +// Mirrors Python ensure_telemetry_container() at weka_runtime.py. +func ensureTelemetryContainer(ctx context.Context) error { + return cmdutil.Run(ctx, "sh", "-c", + "weka local ps | grep -qw telemetry || weka local setup telemetry --not-dependent") } diff --git a/internal/runtime/network/nics.go b/internal/runtime/network/nics.go new file mode 100644 index 000000000..4cd5d8094 --- /dev/null +++ b/internal/runtime/network/nics.go @@ -0,0 +1,471 @@ +// Package network handles management-IP discovery and network-device reconciliation. +// Mirrors write_management_ips, reconcile_net_devices, autodiscover_network_devices +// at weka_runtime.py:2217–3853. +package network + +import ( + "context" + "encoding/json" + "fmt" + "net" + "os" + "os/exec" + "strings" + "time" + + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/runtime/cmdutil" + "github.com/weka/weka-operator/internal/runtime/config" +) + +// ManagementIPs is updated by WriteManagementIPs and used when building the Weka container. +var ManagementIPs []string + +// AutodiscoverNetDevices resolves cfg.NetworkDevice from selectors or subnets when it is empty. +// Mirrors the discovery at weka_runtime.py:2333–2339. +func AutodiscoverNetDevices(ctx context.Context, cfg *config.Config) error { + if cfg.NetworkDevice != "" { + return nil + } + if len(cfg.NetworkSelectors) > 0 { + raw := mustJSONMarshal(cfg.NetworkSelectors) + devInfos, err := getDevicesBySelectors(ctx, raw) + if err != nil { + return fmt.Errorf("network: selectors discovery: %w", err) + } + var names []string + for _, d := range devInfos { + names = append(names, d.device) + } + cfg.NetworkDevice = strings.Join(names, ",") + return nil + } + if len(cfg.Subnets) > 0 { + devs, err := getDevicesBySubnets(ctx, cfg.Subnets) + if err != nil { + return fmt.Errorf("network: subnet discovery: %w", err) + } + cfg.NetworkDevice = strings.Join(devs, ",") + } + return nil +} + +// ReconcileNetDevices syncs the container's net devices to match the desired list. +// Mirrors Python reconcile_net_devices() at weka_runtime.py:2594. +func ReconcileNetDevices(ctx context.Context, containerName string, desired []string) error { + _, logger := instrumentation.CreateLogSpan(ctx, "network.ReconcileNetDevices", "container", containerName) + defer logger.End() + + current, err := getContainerNetDevices(ctx, containerName) + if err != nil { + return fmt.Errorf("network: get container net devices: %w", err) + } + + desiredSet := make(map[string]struct{}, len(desired)) + for _, d := range desired { + desiredSet[d] = struct{}{} + } + currentSet := make(map[string]struct{}, len(current)) + for _, c := range current { + currentSet[c] = struct{}{} + } + + for dev := range currentSet { + if _, ok := desiredSet[dev]; !ok { + if err := cmdutil.Run(ctx, "weka", "local", "resources", "net", "-C", containerName, "remove", dev); err != nil { + return fmt.Errorf("network: remove %s: %w", dev, err) + } + } + } + for dev := range desiredSet { + if _, ok := currentSet[dev]; !ok { + if err := cmdutil.Run(ctx, "weka", "local", "resources", "net", "-C", containerName, "add", dev); err != nil { + return fmt.Errorf("network: add %s: %w", dev, err) + } + } + } + return nil +} + +// WriteManagementIPs discovers management IPs and writes them atomically. +// Mirrors Python write_management_ips() at weka_runtime.py:3797. +func WriteManagementIPs(ctx context.Context, cfg *config.Config) error { + switch cfg.Mode { + case "drive", "compute", "s3", "nfs", "smbw", "client", "data-services": + default: + return nil + } + + _, logger := instrumentation.CreateLogSpan(ctx, "network.WriteManagementIPs") + defer logger.End() + + var ipAddresses []string + + switch { + case cfg.ManagementIP != "" && ShouldAllocateVFPerIoNode(cfg.NetworkDevice): + ipAddresses = []string{cfg.ManagementIP} + + case len(cfg.ManagementIPSelectors) > 0: + raw := mustJSONMarshal(cfg.ManagementIPSelectors) + devInfos, err := getDevicesBySelectors(ctx, raw) + if err != nil { + return fmt.Errorf("network.WriteManagementIPs selectors: %w", err) + } + for _, d := range devInfos { + ip, err := getSingleDeviceIP(ctx, d.device, cfg.IsIPv6) + if err != nil { + return err + } + ipAddresses = append(ipAddresses, ip) + } + + case cfg.NetworkDevice == "" && len(cfg.NetworkSelectors) > 0: + raw := mustJSONMarshal(cfg.NetworkSelectors) + allDevInfos, err := getDevicesBySelectors(ctx, raw) + if err != nil { + return fmt.Errorf("network.WriteManagementIPs network selectors: %w", err) + } + for _, d := range allDevInfos { + if d.rdmaOnly { + continue + } + ip, err := getSingleDeviceIP(ctx, d.device, cfg.IsIPv6) + if err != nil { + return err + } + ipAddresses = append(ipAddresses, ip) + } + if len(ipAddresses) == 0 { + return fmt.Errorf("network: no non-rdma-only devices available; configure managementIpsSelectors separately") + } + + case cfg.NetworkDevice == "" && len(cfg.Subnets) > 0: + devs, err := getDevicesBySubnets(ctx, cfg.Subnets) + if err != nil { + return err + } + for _, dev := range devs { + ip, err := getSingleDeviceIP(ctx, dev, cfg.IsIPv6) + if err != nil { + return err + } + ipAddresses = append(ipAddresses, ip) + } + + case isUDP(cfg): + device := cfg.NetworkDevice + if device == "udp" { + device = "default" + } + ip, err := getSingleDeviceIP(ctx, device, cfg.IsIPv6) + if err != nil { + return err + } + ipAddresses = []string{ip} + + case !strings.Contains(cfg.NetworkDevice, ","): + ip, err := getSingleDeviceIP(ctx, cfg.NetworkDevice, cfg.IsIPv6) + if err != nil { + return err + } + ipAddresses = []string{ip} + + default: + // Multiple NICs. + devices := strings.Split(cfg.NetworkDevice, ",") + for _, dev := range devices { + ip, err := getSingleDeviceIP(ctx, dev, cfg.IsIPv6) + if err != nil { + return err + } + ipAddresses = append(ipAddresses, ip) + } + } + + if len(ipAddresses) == 0 { + return fmt.Errorf("network: failed to discover management IPs") + } + + // Atomic write. + tmpPath := "/opt/weka/k8s-runtime/management_ips.tmp" + if err := os.MkdirAll("/opt/weka/k8s-runtime", 0o755); err != nil { + return err + } + if err := os.WriteFile(tmpPath, []byte(strings.Join(ipAddresses, "\n")), 0o644); err != nil { + return err + } + if err := os.Rename(tmpPath, "/opt/weka/k8s-runtime/management_ips"); err != nil { + return err + } + logger.Info("management IPs written", "ips", ipAddresses) + ManagementIPs = ipAddresses + return nil +} + +// ---- helpers ---------------------------------------------------------------- + +type deviceInfo struct { + device string + rdmaOnly bool + disableRDMA bool +} + +// getDevicesBySelectors filters devices from a JSON-encoded selector list. +// Mirrors Python get_devices_by_selectors() at weka_runtime.py:3750. +func getDevicesBySelectors(ctx context.Context, selectorsJSON string) ([]deviceInfo, error) { + var selectors []struct { + Min int `json:"min"` + Max int `json:"max"` + DeviceNames []string `json:"deviceNames"` + Subnet string `json:"subnet"` + RdmaOnly bool `json:"rdmaOnly"` + DisableRdma bool `json:"disableRdma"` + } + if err := json.Unmarshal([]byte(selectorsJSON), &selectors); err != nil { + return nil, fmt.Errorf("getDevicesBySelectors: parse JSON: %w", err) + } + + var devices []deviceInfo + seen := make(map[string]struct{}) + + for _, sel := range selectors { + minDev := sel.Min + maxDev := sel.Max + + if len(sel.DeviceNames) > 0 { + available := filterMissingDevices(ctx, sel.DeviceNames, sel.RdmaOnly) + if len(available) < minDev { + return nil, fmt.Errorf("not enough devices by deviceNames: want %d, got %d", minDev, len(available)) + } + if maxDev > 0 && len(available) > maxDev { + available = available[:maxDev] + } + for _, name := range available { + if _, ok := seen[name]; !ok { + seen[name] = struct{}{} + devices = append(devices, deviceInfo{device: name, rdmaOnly: sel.RdmaOnly, disableRDMA: sel.DisableRdma}) + } + } + continue + } + if sel.Subnet == "" { + return nil, fmt.Errorf("selector must have deviceNames or subnet") + } + subnetDevs, err := waitForSubnet(ctx, sel.Subnet) + if err != nil { + return nil, err + } + if len(subnetDevs) < minDev { + return nil, fmt.Errorf("not enough devices in subnet %s: want %d, got %d", sel.Subnet, minDev, len(subnetDevs)) + } + if maxDev > 0 && len(subnetDevs) > maxDev { + subnetDevs = subnetDevs[:maxDev] + } + for _, name := range subnetDevs { + if _, ok := seen[name]; !ok { + seen[name] = struct{}{} + devices = append(devices, deviceInfo{device: name, rdmaOnly: sel.RdmaOnly, disableRDMA: sel.DisableRdma}) + } + } + } + return devices, nil +} + +// getDevicesBySubnets finds interfaces whose IP is in any of the given subnets. +// Mirrors Python get_devices_by_subnets() / autodiscover_network_devices() at weka_runtime.py:3742 / 2217. +func getDevicesBySubnets(ctx context.Context, subnets []string) ([]string, error) { + var result []string + seen := make(map[string]struct{}) + for _, subnet := range subnets { + devs, err := waitForSubnet(ctx, subnet) + if err != nil { + return nil, err + } + for _, d := range devs { + if _, ok := seen[d]; !ok { + seen[d] = struct{}{} + result = append(result, d) + } + } + } + return result, nil +} + +// waitForSubnet polls ip -o addr until at least one device is in the subnet (up to 300s). +// Mirrors Python get_devices_waiting_for_all_subnets_to_have_device() at weka_runtime.py:3680 +// (5s poll interval, 300s timeout, error on timeout). +func waitForSubnet(ctx context.Context, subnetStr string) ([]string, error) { + _, logger := instrumentation.CreateLogSpan(ctx, "network.waitForSubnet") + defer logger.End() + + _, ipNet, err := net.ParseCIDR(subnetStr) + if err != nil { + return nil, fmt.Errorf("network: invalid subnet %q: %w", subnetStr, err) + } + + deadline := time.Now().Add(300 * time.Second) + for { + devs, discoverErr := autodiscoverInSubnet(ipNet) + switch { + case discoverErr != nil: + logger.Warn("autodiscover in subnet failed, will retry", "subnet", subnetStr, "err", discoverErr) + case len(devs) > 0: + return devs, nil + default: + logger.Info("no devices found for subnet, waiting", "subnet", subnetStr) + } + + if time.Now().After(deadline) { + return nil, fmt.Errorf("network: no device found for subnet %q after 300s", subnetStr) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(5 * time.Second): + } + } +} + +// filterDevicesInSubnet parses raw `ip -o addr` output and returns the names of +// interfaces whose IP falls inside ipNet. Family (inet/inet6) is inferred from +// whether ipNet.IP is an IPv4 address. Zone IDs ("%zone") and CIDR suffixes are +// stripped before parsing. +func filterDevicesInSubnet(ipAddrOutput []byte, ipNet *net.IPNet) []string { + wantFamily := "inet" + if ipNet.IP.To4() == nil { + wantFamily = "inet6" + } + var devices []string + for _, line := range strings.Split(string(ipAddrOutput), "\n") { + parts := strings.Fields(line) + if len(parts) < 4 { + continue + } + devName := parts[1] + family := parts[2] + ipWithCIDR := parts[3] + + if family != wantFamily { + continue + } + ipStr := strings.Split(strings.Split(ipWithCIDR, "/")[0], "%")[0] + ip := net.ParseIP(ipStr) + if ip == nil { + continue + } + if ipNet.Contains(ip) { + devices = append(devices, devName) + } + } + return devices +} + +// autodiscoverInSubnet runs ip -o addr and returns interfaces with IPs in subnet. +// Mirrors Python autodiscover_network_devices() at weka_runtime.py:2217. +func autodiscoverInSubnet(ipNet *net.IPNet) ([]string, error) { + out, err := exec.Command("ip", "-o", "addr").Output() //nolint:gosec // ip with fixed args, no user input + if err != nil { + return nil, err + } + return filterDevicesInSubnet(out, ipNet), nil +} + +// getSingleDeviceIP gets the primary IP of a network interface. +// Mirrors Python get_single_device_ip() at weka_runtime.py:3647. +func getSingleDeviceIP(ctx context.Context, device string, isIPv6 bool) (string, error) { + var script string + if device == "" || device == "default" { + if isIPv6 { + script = "ip -6 addr show $(ip -6 route show default | awk '{print $5}' | head -n1) | grep 'inet6 ' | grep global | awk '{print $2}' | cut -d/ -f1" + } else { + script = "ip route show default | grep src | awk '/default/ {print $9}' | head -n1" + } + } else { + if isIPv6 { + script = fmt.Sprintf("ip -6 addr show dev %s | grep -E 'inet6 (fd|2)' | head -n1 | awk '{print $2}' | cut -d/ -f1", device) + } else { + script = fmt.Sprintf("ip addr show dev %s | grep 'inet ' | head -n1 | awk '{print $2}' | cut -d/ -f1", device) + } + } + + out, err := cmdutil.Output(ctx, "sh", "-c", script) + if err != nil { + return "", fmt.Errorf("getSingleDeviceIP(%s): %w", device, err) + } + ip := strings.TrimSpace(string(out)) + + // Fallback for default IPv4 device. + if ip == "" && (device == "" || device == "default") && !isIPv6 { + fallback := "ip -4 addr show dev $(ip route show default | awk '{print $5}') | grep inet | awk '{print $2}' | cut -d/ -f1" + out, err = cmdutil.Output(ctx, "sh", "-c", fallback) + if err == nil { + ip = strings.TrimSpace(string(out)) + } + } + if ip == "" { + return "", fmt.Errorf("getSingleDeviceIP(%s): empty result", device) + } + return ip, nil +} + +// filterMissingDevices removes devices that have no IP (or no interface for rdmaOnly). +// Mirrors Python filter_out_missing_devices() at weka_runtime.py:3720. +func filterMissingDevices(ctx context.Context, names []string, rdmaOnly bool) []string { + var available []string + for _, name := range names { + if rdmaOnly { + // Just check if the interface exists. + if err := cmdutil.Run(ctx, "ip", "link", "show", "dev", name); err == nil { + available = append(available, name) + } + } else { + ip, err := getSingleDeviceIP(ctx, name, false) + if err == nil && ip != "" { + available = append(available, name) + } + } + } + return available +} + +// getContainerNetDevices reads the current net_devices from weka local resources. +func getContainerNetDevices(ctx context.Context, name string) ([]string, error) { + out, err := cmdutil.Output(ctx, "weka", "local", "resources", "-C", name, "--json") + if err != nil { + return nil, err + } + var res struct { + NetDevices []struct { + Device string `json:"device"` + } `json:"net_devices"` + } + if err := json.Unmarshal(out, &res); err != nil { + return nil, err + } + result := make([]string, len(res.NetDevices)) + for i, d := range res.NetDevices { + result[i] = d.Device + } + return result, nil +} + +// ShouldAllocateVFPerIoNode reports whether the given network device string uses +// NVIDIA VF-per-IOnode topology. Mirrors Python should_allocate_vf_per_ionode() +// at weka_runtime.py:2305 ("vf_" in network_device). +func ShouldAllocateVFPerIoNode(networkDevice string) bool { + return strings.Contains(networkDevice, "vf_") +} + +// isUDP mirrors Python is_udp(). +func isUDP(cfg *config.Config) bool { + return cfg.UDPMode || strings.EqualFold(cfg.NetworkDevice, "udp") +} + +// mustJSONMarshal marshals v or panics. Only used with compile-time-known string slices. +func mustJSONMarshal(v interface{}) string { + b, err := json.Marshal(v) + if err != nil { + panic(err) + } + return string(b) +} diff --git a/internal/runtime/network/nics_test.go b/internal/runtime/network/nics_test.go new file mode 100644 index 000000000..acc19c6cf --- /dev/null +++ b/internal/runtime/network/nics_test.go @@ -0,0 +1,206 @@ +package network + +import ( + "net" + "testing" + + "github.com/weka/weka-operator/internal/runtime/config" +) + +func TestShouldAllocateVFPerIoNode(t *testing.T) { + tests := []struct { + device string + want bool + }{ + {"vf_eth0", true}, + {"eth0,vf_eth1", true}, + {"eth0", false}, + {"", false}, + {"udp", false}, + } + for _, tt := range tests { + if got := ShouldAllocateVFPerIoNode(tt.device); got != tt.want { + t.Errorf("ShouldAllocateVFPerIoNode(%q) = %v, want %v", tt.device, got, tt.want) + } + } +} + +// realIPAddrOutput is a verbatim fixture from a live node. +// Field layout: parts[0]=index, parts[1]=device, parts[2]=inet/inet6, parts[3]=addr/CIDR. +var realIPAddrOutput = []byte( + "1: lo inet 127.0.0.1/8 scope host lo\\ \n" + + " valid_lft forever preferred_lft forever\n" + + "1: lo inet6 ::1/128 scope host \\ \n" + + " valid_lft forever preferred_lft forever\n" + + "2: enp80s0f0 inet 172.31.5.61/21 metric 100 brd 172.31.7.255 scope global dynamic enp80s0f0\\ \n" + + " valid_lft 9949sec preferred_lft 9949sec\n" + + "2: enp80s0f0 inet6 fe80::1/64 scope link\\ \n" + + " valid_lft forever preferred_lft forever\n" + + "4: enp99s0f0np0 inet 10.100.5.61/16 brd 10.100.255.255 scope global enp99s0f0np0\\ \n" + + " valid_lft forever preferred_lft forever\n" + + "4: enp99s0f0np0 inet6 fe80::2/64 scope link\\ \n" + + " valid_lft forever preferred_lft forever\n" + + "5: ib0 inet 10.2.5.61/16 brd 10.2.255.255 scope global ib0\\ \n" + + " valid_lft forever preferred_lft forever\n" + + "5: ib0 inet6 fe80::3/64 scope link\\ \n" + + " valid_lft forever preferred_lft forever\n", +) + +func mustParseCIDR(s string) *net.IPNet { + _, ipNet, err := net.ParseCIDR(s) + if err != nil { + panic(err) + } + return ipNet +} + +func TestFilterDevicesInSubnet(t *testing.T) { + tests := []struct { + name string + input []byte + subnet string + want []string + }{ + { + name: "10.100.0.0/16 matches enp99s0f0np0", + input: realIPAddrOutput, + subnet: "10.100.0.0/16", + want: []string{"enp99s0f0np0"}, + }, + { + name: "10.2.0.0/16 matches ib0", + input: realIPAddrOutput, + subnet: "10.2.0.0/16", + want: []string{"ib0"}, + }, + { + name: "172.31.0.0/21 matches enp80s0f0", + input: realIPAddrOutput, + subnet: "172.31.0.0/21", + want: []string{"enp80s0f0"}, + }, + { + // IPv4 target: all inet6 lines must be excluded by family filter. + name: "IPv4 subnet excludes all inet6 lines", + input: realIPAddrOutput, + subnet: "10.100.0.0/16", + want: []string{"enp99s0f0np0"}, // no inet6 entries even though enp99s0f0np0 has one + }, + { + // lo (127.0.0.1) must be excluded when target subnet doesn't contain it. + name: "lo excluded by CIDR mismatch", + input: realIPAddrOutput, + subnet: "10.2.0.0/16", + want: []string{"ib0"}, // lo is not in 10.2.0.0/16 + }, + { + // Subnet that no address belongs to returns nil. + name: "no match returns nil", + input: realIPAddrOutput, + subnet: "192.168.0.0/24", + want: nil, + }, + { + // Short/garbage lines (< 4 fields) must be skipped silently. + name: "short lines skipped", + input: []byte("1: lo\n2: eth0 inet\n"), + subnet: "10.0.0.0/8", + want: nil, + }, + { + // Empty input produces nil. + name: "empty input", + input: []byte(""), + subnet: "10.0.0.0/8", + want: nil, + }, + { + // IPv6 case: target fe80::/64, expect enp80s0f0 (has fe80::1/64). + name: "IPv6 fe80::/64 matches link-local on enp80s0f0", + input: realIPAddrOutput, + subnet: "fe80::/64", + want: []string{"enp80s0f0", "enp99s0f0np0", "ib0"}, + }, + { + // Zone ID in address ("%eth0") must be stripped before parsing. + name: "zone ID stripped from IPv6 address", + input: []byte("3: eth1 inet6 fe80::1%eth1/64 scope link\\ \n"), + subnet: "fe80::/64", + want: []string{"eth1"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ipNet := mustParseCIDR(tt.subnet) + got := filterDevicesInSubnet(tt.input, ipNet) + + if len(got) != len(tt.want) { + t.Fatalf("filterDevicesInSubnet(%q): got %v, want %v", tt.subnet, got, tt.want) + } + for i := range got { + if got[i] != tt.want[i] { + t.Errorf("filterDevicesInSubnet(%q)[%d] = %q, want %q", tt.subnet, i, got[i], tt.want[i]) + } + } + }) + } +} + +func TestIsUDP(t *testing.T) { + tests := []struct { + name string + cfg *config.Config + want bool + }{ + { + name: "UDPMode true", + cfg: &config.Config{UDPMode: true}, + want: true, + }, + { + name: "NetworkDevice=udp (lowercase)", + cfg: &config.Config{NetworkDevice: "udp"}, + want: true, + }, + { + name: "NetworkDevice=UDP (uppercase)", + cfg: &config.Config{NetworkDevice: "UDP"}, + want: true, + }, + { + name: "NetworkDevice=Udp (mixed case)", + cfg: &config.Config{NetworkDevice: "Udp"}, + want: true, + }, + { + name: "both UDPMode and NetworkDevice=udp", + cfg: &config.Config{UDPMode: true, NetworkDevice: "udp"}, + want: true, + }, + { + name: "neither UDPMode nor udp device", + cfg: &config.Config{NetworkDevice: "eth0"}, + want: false, + }, + { + name: "empty config", + cfg: &config.Config{}, + want: false, + }, + { + name: "NetworkDevice contains udp but is not exactly udp", + cfg: &config.Config{NetworkDevice: "eth0,udp"}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isUDP(tt.cfg) + if got != tt.want { + t.Errorf("isUDP(%+v) = %v, want %v", tt.cfg, got, tt.want) + } + }) + } +} diff --git a/internal/runtime/persistency/persistency.go b/internal/runtime/persistency/persistency.go new file mode 100644 index 000000000..fae8a2732 --- /dev/null +++ b/internal/runtime/persistency/persistency.go @@ -0,0 +1,95 @@ +// Package persistency sets up persistent storage bind-mounts for the Weka pod runtime. +// It mirrors configure_persistency() at weka_runtime.py:2835. +package persistency + +import ( + "fmt" + "os" + + "context" + + "github.com/weka/weka-operator/internal/runtime/cmdutil" + "github.com/weka/weka-operator/internal/runtime/config" +) + +const ( + persistencyConfiguredPath = "/opt/weka/k8s-runtime/persistency-configured" + wekaK8sRuntimeDir = "/opt/weka/k8s-runtime" +) + +// Configure sets up persistent storage bind-mounts. +// Mirrors Python configure_persistency() at weka_runtime.py:2835. +func Configure(ctx context.Context, cfg *config.Config) error { + persistenceDir := "/host-binds/opt-weka" + if cfg.WekaPersistenceMode == "global" { + persistenceDir = fmt.Sprintf("/opt/weka-global-persistence/containers/%s", cfg.WekaContainerID) + } + + script := fmt.Sprintf(` +if [ -d /host-binds/opt-weka ]; then + mkdir -p /opt/weka-preinstalled + mount -o bind /opt/weka /opt/weka-preinstalled + mkdir -p %s/dist/drivers + mount -o bind %s/dist/drivers /opt/weka-preinstalled/dist/drivers + mount -o bind %s /opt/weka + mkdir -p /opt/weka/dist + mount -o bind /opt/weka-preinstalled/dist /opt/weka/dist + mount -o bind %s/dist/drivers /opt/weka/dist/drivers +fi + +if [ -d /host-binds/boot-level ]; then + BOOT_DIR=/host-binds/boot-level/$(cat /proc/sys/kernel/random/boot_id)/cleanup + mkdir -p $BOOT_DIR + mkdir -p /opt/weka/external-mounts/cleanup + mount -o bind $BOOT_DIR /opt/weka/external-mounts/cleanup +fi + +if [ -d /host-binds/ssdproxy ]; then + mkdir -p /opt/weka/external-mounts/ssdproxy + mount -o bind /host-binds/ssdproxy /opt/weka/external-mounts/ssdproxy +fi + +if [ -d /host-binds/shared ]; then + mkdir -p /host-binds/shared/local-sockets + mkdir -p /opt/weka/external-mounts/local-sockets + mount -o bind /host-binds/shared/local-sockets /opt/weka/external-mounts/local-sockets +fi + +if [ -f /var/run/secrets/weka-operator/wekahome-cacert/cert.pem ]; then + rm -rf /opt/weka/k8s-runtime/vars/wh-cacert + mkdir -p /opt/weka/k8s-runtime/vars/wh-cacert/ + cp /var/run/secrets/weka-operator/wekahome-cacert/cert.pem /opt/weka/k8s-runtime/vars/wh-cacert/cert.pem + chmod 400 /opt/weka/k8s-runtime/vars/wh-cacert/cert.pem +fi + +if [ -d /host-binds/shared-configs ]; then + mkdir -p /opt/weka/external-mounts/shared_boot_level + mount -o bind /host-binds/shared-configs /opt/weka/external-mounts/shared_boot_level + ENVOY_DIR=/opt/weka/envoy + EXT_ENVOY_DIR=/host-binds/shared-configs/envoy + mkdir -p $ENVOY_DIR + mkdir -p $EXT_ENVOY_DIR + mount -o bind $EXT_ENVOY_DIR $ENVOY_DIR + mkdir -p /opt/weka/wtracer + mkdir -p /host-binds/shared-configs/audit-traces + mount -o bind /host-binds/shared-configs/audit-traces /opt/weka/wtracer +fi + +mkdir -p %s +touch %s +`, + persistenceDir, persistenceDir, persistenceDir, persistenceDir, + wekaK8sRuntimeDir, persistencyConfiguredPath, + ) + + if err := cmdutil.Run(ctx, "sh", "-c", script); err != nil { + return fmt.Errorf("configure_persistency: %w", err) + } + return nil +} + +// IsConfigured reports whether persistency has been configured. +func IsConfigured() bool { + _, err := os.Stat(persistencyConfiguredPath) + return err == nil +} diff --git a/internal/runtime/ports/ports.go b/internal/runtime/ports/ports.go new file mode 100644 index 000000000..61a6947fd --- /dev/null +++ b/internal/runtime/ports/ports.go @@ -0,0 +1,178 @@ +// Package ports allocates weka container port ranges for client mode. +// Mirrors ensure_client_ports, get_free_subrange_in_port_range at weka_runtime.py:3527–3552. +package ports + +import ( + "bufio" + "context" + "fmt" + "os" + "strconv" + "strings" + + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/runtime/config" +) + +const ( + subrangeSize = 100 // WEKA_CONTAINER_PORT_SUBRANGE + maxPort = 65535 +) + +// SavePorts writes the current ports in cfg to the runtime vars files. +// Called after port allocation (client) and after loading resources.json (compute/drive/etc). +// Mirrors Python save_weka_ports_data() at weka_runtime.py:3554. +func SavePorts(_ context.Context, cfg *config.Config) error { + return savePorts(cfg) +} + +// AllocateClientPorts finds free ports and writes them to runtime files. +// Mirrors Python ensure_client_ports() at weka_runtime.py:3527. +// No-op when cfg.Port != 0 (ports already assigned via resources). +func AllocateClientPorts(ctx context.Context, cfg *config.Config) error { + _, logger := instrumentation.CreateLogSpan(ctx, "ports.AllocateClientPorts") + defer logger.End() + + if cfg.Port != 0 && cfg.AgentPort != 0 { + // Already have ports from environment; just persist them. + return savePorts(cfg) + } + + // Mirror Python assert base_port > 0, "BASE_PORT is not set" at weka_runtime.py:3537. + base := cfg.BasePort + if base == 0 { + return fmt.Errorf("ports: BASE_PORT is not set") + } + portRange := cfg.PortRange + // Mirror Python: max_port = base_port + port_range if port_range > 0 else MAX_PORT (weka_runtime.py:3538). + // MAX_PORT = 65535. + top := maxPort + if portRange > 0 { + top = base + portRange + if top > maxPort { + top = maxPort + } + } + + inUse, err := readInUsePorts() + if err != nil { + return fmt.Errorf("ports: reading in-use ports: %w", err) + } + + if cfg.AgentPort == 0 { + agentPort, err := findFreePort(base, top, inUse, nil) + if err != nil { + return fmt.Errorf("ports: find agent port: %w", err) + } + cfg.AgentPort = agentPort + } + + if cfg.Port == 0 { + p, err := getFreeSubrange(base, top, subrangeSize, inUse, []int{cfg.AgentPort}) + if err != nil { + return fmt.Errorf("ports: find port subrange: %w", err) + } + cfg.Port = p + } + + return savePorts(cfg) +} + +// savePorts writes the port vars files. +// Mirrors Python save_weka_ports_data() at weka_runtime.py:3554-3557 which writes ONLY +// vars/port and vars/agent_port — no weka-ports-data.json (that file is never written by Python). +func savePorts(cfg *config.Config) error { + if err := os.MkdirAll("/opt/weka/k8s-runtime/vars", 0o755); err != nil { + return err + } + if err := os.WriteFile("/opt/weka/k8s-runtime/vars/port", []byte(strconv.Itoa(cfg.Port)), 0o644); err != nil { + return err + } + return os.WriteFile("/opt/weka/k8s-runtime/vars/agent_port", []byte(strconv.Itoa(cfg.AgentPort)), 0o644) +} + +// readInUsePorts parses /proc/net/tcp, tcp6, udp, udp6 and returns the set of in-use ports. +func readInUsePorts() (map[int]struct{}, error) { + inUse := make(map[int]struct{}) + files := []string{"/proc/net/tcp", "/proc/net/tcp6", "/proc/net/udp", "/proc/net/udp6"} + for _, path := range files { + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, err + } + scanner := bufio.NewScanner(f) + scanner.Scan() // skip header + for scanner.Scan() { + cols := strings.Fields(scanner.Text()) + if len(cols) < 2 { + continue + } + // Column 1: "IP:PORT" (hex) + addrParts := strings.SplitN(cols[1], ":", 2) + if len(addrParts) != 2 { + continue + } + port, err := strconv.ParseInt(addrParts[1], 16, 32) + if err != nil || port == 0 { + continue + } + inUse[int(port)] = struct{}{} + } + _ = f.Close() //nolint:errcheck // read-only file, close error is not actionable + } + return inUse, nil +} + +// findFreePort finds a single free port in [base, top) not in inUse or exclude. +func findFreePort(base, top int, inUse map[int]struct{}, exclude []int) (int, error) { + excludeSet := make(map[int]struct{}, len(exclude)) + for _, p := range exclude { + excludeSet[p] = struct{}{} + } + for p := base; p < top; p++ { + if _, used := inUse[p]; used { + continue + } + if _, excl := excludeSet[p]; excl { + continue + } + return p, nil + } + return 0, fmt.Errorf("no free port in [%d, %d)", base, top) +} + +// getFreeSubrange finds the first window of `size` consecutive free ports in [base, top). +// Mirrors Python get_free_subrange_in_port_range() at weka_runtime.py:3470. +func getFreeSubrange(base, top, size int, inUse map[int]struct{}, exclude []int) (int, error) { + excludeSet := make(map[int]struct{}, len(exclude)) + for _, p := range exclude { + excludeSet[p] = struct{}{} + } + + for start := base; start <= top-size; start++ { + // Skip if start itself is excluded. + if _, ex := excludeSet[start]; ex { + continue + } + allFree := true + for p := start; p < start+size; p++ { + if _, used := inUse[p]; used { + allFree = false + start = p // jump past the blocked port + break + } + if _, ex := excludeSet[p]; ex { + allFree = false + start = p + break + } + } + if allFree { + return start, nil + } + } + return 0, fmt.Errorf("no free %d-port subrange in [%d, %d)", size, base, top) +} diff --git a/internal/runtime/ports/ports_test.go b/internal/runtime/ports/ports_test.go new file mode 100644 index 000000000..90cf43ea9 --- /dev/null +++ b/internal/runtime/ports/ports_test.go @@ -0,0 +1,190 @@ +package ports + +import ( + "testing" +) + +// ---- getFreeSubrange tests ---- + +func TestGetFreeSubrange(t *testing.T) { + tests := []struct { + name string + base int + top int + size int + inUse map[int]struct{} + exclude []int + want int + wantErr bool + }{ + { + name: "empty inUse returns base", + base: 1000, + top: 1200, + size: 10, + inUse: map[int]struct{}{}, + exclude: nil, + want: 1000, + }, + { + name: "used port mid-window forces jump past it", + base: 1000, + top: 1200, + size: 10, + inUse: map[int]struct{}{1005: {}}, + exclude: nil, + // start=1000: scans to 1005 (used) → jump start=1005, loop increments to 1006 + want: 1006, + }, + { + name: "excluded port at start is skipped", + base: 1000, + top: 1200, + size: 10, + inUse: map[int]struct{}{}, + exclude: []int{1000}, + // 1000 excluded at start check → start=1001 + want: 1001, + }, + { + name: "excluded port mid-window causes jump", + base: 1000, + top: 1200, + size: 10, + inUse: map[int]struct{}{}, + exclude: []int{1007}, + // start=1000: scans to 1007 (excluded) → jump to 1007, loop increments to 1008 + want: 1008, + }, + { + name: "agentPort as exclude skips that window", + base: 1000, + top: 1200, + size: 5, + inUse: map[int]struct{}{}, + exclude: []int{1003}, // agentPort in the middle of first window + // 1000 is not excluded; scans 1000..1004, hits 1003 (excluded) → jump to 1003, incr to 1004 + // then 1004 start: check 1004..1008 — all free → return 1004 + want: 1004, + }, + { + name: "no window fits returns error", + base: 1000, + top: 1010, + size: 20, + inUse: map[int]struct{}{}, + exclude: nil, + wantErr: true, + }, + { + name: "window exactly at top-size boundary", + base: 1000, + top: 1010, + size: 10, + inUse: map[int]struct{}{}, + exclude: nil, + // start <= top-size = 1000, exactly one window [1000,1010) + want: 1000, + }, + { + name: "window starting one before the boundary", + base: 995, + top: 1010, + size: 10, + inUse: map[int]struct{}{}, + exclude: nil, + // many windows fit; first free is 995 + want: 995, + }, + { + name: "all ports in range used returns error", + base: 1000, + top: 1005, + size: 3, + inUse: map[int]struct{}{ + 1000: {}, 1001: {}, 1002: {}, 1003: {}, 1004: {}, + }, + exclude: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getFreeSubrange(tt.base, tt.top, tt.size, tt.inUse, tt.exclude) + if (err != nil) != tt.wantErr { + t.Fatalf("getFreeSubrange() error = %v, wantErr %v", err, tt.wantErr) + } + if !tt.wantErr && got != tt.want { + t.Errorf("getFreeSubrange() = %d, want %d", got, tt.want) + } + }) + } +} + +// ---- findFreePort tests ---- + +func TestFindFreePort(t *testing.T) { + tests := []struct { + name string + base int + top int + inUse map[int]struct{} + exclude []int + want int + wantErr bool + }{ + { + name: "first free port returned", + base: 2000, + top: 2100, + inUse: map[int]struct{}{}, + exclude: nil, + want: 2000, + }, + { + name: "all ports used returns error", + base: 2000, + top: 2003, + inUse: map[int]struct{}{2000: {}, 2001: {}, 2002: {}}, + want: 0, + wantErr: true, + }, + { + name: "excluded port skipped", + base: 3000, + top: 3100, + inUse: map[int]struct{}{}, + exclude: []int{3000, 3001}, + want: 3002, + }, + { + name: "base equals top returns error (empty range)", + base: 5000, + top: 5000, + inUse: map[int]struct{}{}, + exclude: nil, + wantErr: true, + }, + { + name: "used and excluded combination", + base: 4000, + top: 4010, + inUse: map[int]struct{}{4000: {}, 4001: {}}, + exclude: []int{4002}, + want: 4003, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := findFreePort(tt.base, tt.top, tt.inUse, tt.exclude) + if (err != nil) != tt.wantErr { + t.Fatalf("findFreePort() error = %v, wantErr %v", err, tt.wantErr) + } + if !tt.wantErr && got != tt.want { + t.Errorf("findFreePort() = %d, want %d", got, tt.want) + } + }) + } +} diff --git a/internal/runtime/resources/resources.go b/internal/runtime/resources/resources.go new file mode 100644 index 000000000..c30703deb --- /dev/null +++ b/internal/runtime/resources/resources.go @@ -0,0 +1,89 @@ +// Package resources waits for and loads the k8s-runtime resources.json written by the operator. +// Mirrors wait_for_resources() at weka_runtime.py:3575. +package resources + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/weka/go-weka-observability/instrumentation" +) + +const resourcesPath = "/opt/weka/k8s-runtime/resources.json" + +// retryInterval is the poll/retry cadence; a var (not const) so tests can lower it. +var retryInterval = 3 * time.Second + +// NodeResources is the JSON structure written by the operator controller. +type NodeResources struct { + WekaPort int `json:"wekaPort"` + AgentPort int `json:"agentPort"` + FailureDomain string `json:"failureDomain"` + Drives []string `json:"drives"` + NetDevices []string `json:"netDevices"` + MachineIdentifier string `json:"machineIdentifier,omitempty"` +} + +// WaitAndLoad polls until /opt/weka/k8s-runtime/resources.json appears, then parses it. +// shouldAbort, if non-nil, is called after each phase-1 sleep; if it returns true the wait is +// aborted immediately. Mirrors Python wait_for_resources() at weka_runtime.py:3586–3621. +func WaitAndLoad(ctx context.Context, shouldAbort func() bool) (*NodeResources, error) { + _, logger := instrumentation.CreateLogSpan(ctx, "resources.WaitAndLoad") + defer logger.End() + + // Phase 1: wait for file to appear. + for { + if _, err := os.Stat(resourcesPath); err == nil { + break + } + logger.Info("waiting for /opt/weka/k8s-runtime/resources.json") + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(retryInterval): + } + if shouldAbort != nil && shouldAbort() { + return nil, fmt.Errorf("resources: shutdown requested while waiting for %s", resourcesPath) + } + } + + // Phase 2: try up to 10 times to read valid JSON. + const maxRetries = 10 + for attempt := 0; attempt < maxRetries; attempt++ { + content, err := os.ReadFile(resourcesPath) + if err != nil { + logger.Warn("error reading resources.json", "err", err, "attempt", attempt+1) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(retryInterval): + continue + } + } + if len(content) == 0 { + logger.Warn("resources.json is empty, waiting for content...", "attempt", attempt+1) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(retryInterval): + continue + } + } + var res NodeResources + if err := json.Unmarshal(content, &res); err != nil { + logger.Warn("invalid JSON in resources.json", "err", err, "attempt", attempt+1) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(retryInterval): + continue + } + } + logger.Info("loaded resources.json", "resources", res) + return &res, nil + } + return nil, fmt.Errorf("resources: failed to read valid JSON from %s after %d attempts", resourcesPath, maxRetries) +} diff --git a/internal/runtime/resources/resources_test.go b/internal/runtime/resources/resources_test.go new file mode 100644 index 000000000..a171f5203 --- /dev/null +++ b/internal/runtime/resources/resources_test.go @@ -0,0 +1,39 @@ +package resources + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +func TestWaitAndLoad_AbortsOnShutdown(t *testing.T) { + orig := retryInterval + retryInterval = time.Millisecond + defer func() { retryInterval = orig }() + + // resourcesPath won't exist in the test environment, so phase 1 loops, + // sleeps retryInterval, then shouldAbort returns true → aborts with error. + _, err := WaitAndLoad(context.Background(), func() bool { return true }) + if err == nil { + t.Fatal("expected error when shutdown requested, got nil") + } + if !strings.Contains(err.Error(), "shutdown") { + t.Errorf("error = %q, want it to mention shutdown", err.Error()) + } +} + +func TestWaitAndLoad_CtxCancel(t *testing.T) { + orig := retryInterval + retryInterval = time.Millisecond + defer func() { retryInterval = orig }() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // immediately cancelled + + _, err := WaitAndLoad(ctx, func() bool { return false }) + if !errors.Is(err, context.Canceled) { + t.Errorf("err = %v, want context.Canceled", err) + } +} diff --git a/internal/runtime/shutdown/shutdown.go b/internal/runtime/shutdown/shutdown.go new file mode 100644 index 000000000..c477076f3 --- /dev/null +++ b/internal/runtime/shutdown/shutdown.go @@ -0,0 +1,151 @@ +// Package shutdown reads and polls shutdown instructions written by the operator controller. +// Mirrors get_shutdown_instructions, wait_for_shutdown_instruction, and drive-shutdown phase +// at weka_runtime.py:2794–4591. +package shutdown + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/runtime/wekadrive" +) + +// Test seams for WaitForDriveRelease; default to the real implementation and cadence. +var ( + findWekaPartitionsFn = wekadrive.FindWekaPartitions + driveReleasePollInterval = 300 * time.Millisecond +) + +// ShutdownInstructions holds the controller-written instructions for this pod. +type ShutdownInstructions struct { + AllowStop bool `json:"allow_stop"` + AllowForceStop bool `json:"allow_force_stop"` +} + +// GetBootID reads the kernel boot_id. +func GetBootID() string { + content, err := os.ReadFile("/proc/sys/kernel/random/boot_id") + if err != nil { + fmt.Fprintf(os.Stderr, "shutdown: failed to read boot_id: %v\n", err) + return "" + } + return strings.TrimSpace(string(content)) +} + +// GetShutdownInstructions reads the shutdown instructions file for the given pod+boot pair. +// Mirrors Python get_shutdown_instructions() at weka_runtime.py:2794. +// A missing or unparsable file is treated as "no instructions" (an empty struct), matching +// Python's best-effort behavior, so there is no error to return to the caller. +func GetShutdownInstructions(podID, bootID string) *ShutdownInstructions { + ret := &ShutdownInstructions{} + + if podID != "" { + path := fmt.Sprintf("/host-binds/shared/instructions/%s/%s/shutdown_instructions.json", podID, bootID) + if _, err := os.Stat(path); err == nil { + data, err := os.ReadFile(path) + if err == nil { + if jsonErr := json.Unmarshal(data, ret); jsonErr != nil { + fmt.Fprintf(os.Stderr, "shutdown: failed to parse instructions file %s: %v\n", path, jsonErr) + ret = &ShutdownInstructions{} + } + } + } + } + + if _, err := os.Stat("/tmp/.allow-force-stop"); err == nil { + ret.AllowForceStop = true + } + if _, err := os.Stat("/tmp/.allow-stop"); err == nil { + ret.AllowStop = true + } + return ret +} + +// PollShutdownInstructions blocks until the operator permits a stop. +// Returns graceful=true for allow_stop, graceful=false for allow_force_stop. +// Mirrors Python wait_for_shutdown_instruction() at weka_runtime.py:4474. +func PollShutdownInstructions(podID, bootID string) (graceful bool) { + iteration := 0 + for { + iteration++ + instructions := GetShutdownInstructions(podID, bootID) + if instructions.AllowForceStop { + return false + } + if instructions.AllowStop { + return true + } + if iteration%6 == 1 { + fmt.Printf("shutdown: waiting for shutdown instruction (iteration %d, elapsed ~%ds)\n", + iteration, iteration*5) + } + time.Sleep(5 * time.Second) + } +} + +// WaitForDriveRelease polls until all requested drive serials have returned to the kernel +// (i.e. are visible as Weka partitions again after Weka stops owning them). +// Mirrors the drive-release polling loop at weka_runtime.py:4565–4590: +// +// find_weka_drives() every 0.3s for up to 60s; break when every requested serial IS present; +// on timeout logging.error and continue (non-fatal). +// +// On timeout this function logs an error and returns nil — it never returns a hard error. +func WaitForDriveRelease(ctx context.Context, requestedSerials []string, timeout time.Duration) error { + _, logger := instrumentation.CreateLogSpan(ctx, "shutdown.WaitForDriveRelease") + defer logger.End() + + if len(requestedSerials) == 0 { + return nil + } + + requested := make(map[string]struct{}, len(requestedSerials)) + for _, s := range requestedSerials { + requested[s] = struct{}{} + } + + deadline := time.Now().Add(timeout) + for { + drives, err := findWekaPartitionsFn(ctx) + if err != nil { + logger.Warn("FindWekaPartitions error while waiting for drive release", "err", err) + } else { + // Success: every requested serial is present in the kernel scan. + // Mirrors Python: if set(requested_serials) <= found_serials: break + allFound := true + for serial := range requested { + found := false + for _, d := range drives { + if d.SerialId == serial { + found = true + break + } + } + if !found { + allFound = false + break + } + } + if allFound { + logger.Info("all requested drives returned to kernel") + return nil + } + } + + if time.Now().After(deadline) { + // Non-fatal on timeout — mirrors Python logging.error + continue (weka_runtime.py:4588-4590). + logger.Error(nil, "shutdown: drives did not return to kernel after timeout; continuing teardown", "timeout", timeout) + return nil + } + select { + case <-ctx.Done(): + return nil // treat ctx cancel as non-fatal too, matching Python's non-fatal timeout + case <-time.After(driveReleasePollInterval): + } + } +} diff --git a/internal/runtime/shutdown/shutdown_test.go b/internal/runtime/shutdown/shutdown_test.go new file mode 100644 index 000000000..c4f30fdc2 --- /dev/null +++ b/internal/runtime/shutdown/shutdown_test.go @@ -0,0 +1,77 @@ +package shutdown + +import ( + "context" + "testing" + "time" + + "github.com/weka/weka-operator/internal/pkg/domain" +) + +// drivesFromSerials builds a fake FindWekaPartitions result from serial ids. +func drivesFromSerials(serials ...string) []domain.DriveInfo { + out := make([]domain.DriveInfo, 0, len(serials)) + for _, s := range serials { + out = append(out, domain.DriveInfo{SerialId: s}) + } + return out +} + +// TestWaitForDriveRelease covers the three behaviors of the drive-release loop: +// immediate return on empty input, waiting until ALL requested serials reappear, +// and non-fatal return on timeout. Mirrors weka_runtime.py:4565–4590. +func TestWaitForDriveRelease(t *testing.T) { + origFind := findWekaPartitionsFn + origInterval := driveReleasePollInterval + defer func() { + findWekaPartitionsFn = origFind + driveReleasePollInterval = origInterval + }() + driveReleasePollInterval = time.Millisecond + + t.Run("empty serials returns immediately", func(t *testing.T) { + called := false + findWekaPartitionsFn = func(_ context.Context) ([]domain.DriveInfo, error) { + called = true + return nil, nil + } + if err := WaitForDriveRelease(context.Background(), nil, time.Second); err != nil { + t.Fatalf("WaitForDriveRelease(nil) = %v, want nil", err) + } + if called { + t.Error("findWekaPartitionsFn should not be called for empty requested serials") + } + }) + + t.Run("returns nil only once all requested serials are present", func(t *testing.T) { + // First poll: only one of two serials present. Second poll: both present. + poll := 0 + findWekaPartitionsFn = func(_ context.Context) ([]domain.DriveInfo, error) { + poll++ + if poll == 1 { + return drivesFromSerials("serial-a"), nil + } + return drivesFromSerials("serial-a", "serial-b"), nil + } + if err := WaitForDriveRelease(context.Background(), []string{"serial-a", "serial-b"}, time.Second); err != nil { + t.Fatalf("WaitForDriveRelease = %v, want nil", err) + } + if poll < 2 { + t.Errorf("expected to wait for at least 2 polls until all serials present, got %d", poll) + } + }) + + t.Run("non-fatal on timeout when serials never reappear", func(t *testing.T) { + findWekaPartitionsFn = func(_ context.Context) ([]domain.DriveInfo, error) { + return drivesFromSerials("serial-a"), nil // serial-b never returns + } + start := time.Now() + err := WaitForDriveRelease(context.Background(), []string{"serial-a", "serial-b"}, 20*time.Millisecond) + if err != nil { + t.Fatalf("WaitForDriveRelease on timeout = %v, want nil (non-fatal)", err) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("WaitForDriveRelease took %v, expected to return shortly after the 20ms timeout", elapsed) + } + }) +} diff --git a/internal/runtime/syslog/syslog.go b/internal/runtime/syslog/syslog.go new file mode 100644 index 000000000..938c66a9d --- /dev/null +++ b/internal/runtime/syslog/syslog.go @@ -0,0 +1,44 @@ +// Package syslog adds a syslog daemon to the process supervisor. +// Mirrors start_syslog() at weka_runtime.py:3995. +package syslog + +import ( + "os" + "os/exec" + + "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/daemon" +) + +// AddToDaemon registers the appropriate syslog daemon with the supervisor. +// Mirrors Python start_syslog() at weka_runtime.py:3995. +// +// Syslog selection rules (matching Python use_go_syslog()): +// - SyslogPackage == "auto": use go-syslog if /usr/sbin/go-syslog exists, else syslog-ng +// - SyslogPackage == "go-syslog": always use go-syslog +// - SyslogPackage == "syslog-ng": always use syslog-ng +func AddToDaemon(sup *daemon.Supervisor, cfg *config.Config) { + cmd, args := chooseSyslog(cfg.SyslogPackage) + sup.Add("syslog", func() *exec.Cmd { + return exec.Command(cmd, args...) //nolint:gosec // path is a known binary + }) +} + +func chooseSyslog(pkg string) (cmd string, args []string) { + if useGoSyslog(pkg) { + return "/usr/sbin/go-syslog", nil + } + return "/usr/sbin/syslog-ng", []string{"-F", "-f", "/etc/syslog-ng/syslog-ng.conf", "--pidfile", "/var/run/syslog-ng.pid"} +} + +func useGoSyslog(pkg string) bool { + switch pkg { + case "go-syslog": + return true + case "syslog-ng": + return false + default: // "auto" or empty + _, err := os.Stat("/usr/sbin/go-syslog") + return err == nil + } +} diff --git a/internal/runtime/weka/ensure.go b/internal/runtime/weka/ensure.go new file mode 100644 index 000000000..c911c8ab0 --- /dev/null +++ b/internal/runtime/weka/ensure.go @@ -0,0 +1,641 @@ +// Package weka — ensure.go implements Weka container lifecycle management. +// Mirrors ensure_weka_container, create_container, handle_existing_container, +// should_recreate_client_container, write_feature_flags_json, write_telemetry_config_override +// at weka_runtime.py:2312–3408. +package weka + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "syscall" + "time" + + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/runtime/cmdutil" + "github.com/weka/weka-operator/internal/runtime/config" + "github.com/weka/weka-operator/internal/runtime/cpuaffinity" + "github.com/weka/weka-operator/internal/runtime/network" + "github.com/weka/weka-operator/internal/runtime/resources" +) + +// modeCoresFlag maps mode → weka local resources cores flag. +var modeCoresFlag = map[string]string{ + "compute": "--only-compute-cores", + "drive": "--only-drives-cores", + "client": "--only-frontend-cores", + "s3": "--only-frontend-cores", + "nfs": "--only-frontend-cores", + "smbw": "--only-frontend-cores", + "data-services": "--only-dataserv-cores", +} + +// EnsureWekaContainer ensures the named Weka container exists and is configured. +// Mirrors Python ensure_weka_container() at weka_runtime.py:2644. +func EnsureWekaContainer(ctx context.Context, cfg *config.Config, res *resources.NodeResources) error { + _, logger := instrumentation.CreateLogSpan(ctx, "weka.EnsureWekaContainer", "name", cfg.Name) + defer logger.End() + + resourcesDir := fmt.Sprintf("/opt/weka/data/%s/container", cfg.Name) + if err := os.MkdirAll(resourcesDir, 0o755); err != nil { + return fmt.Errorf("EnsureWekaContainer mkdir: %w", err) + } + + containers, err := getContainers(ctx) + if err != nil { + return err + } + + if len(containers) == 0 { + logger.Info("no pre-existing containers, creating") + if createErr := createContainer(ctx, cfg); createErr != nil { + return createErr + } + } else { + var found map[string]interface{} + for _, c := range containers { + name, ok := c["name"].(string) + if ok && name == cfg.Name { + found = c + break + } + } + if found == nil { + names := make([]string, 0, len(containers)) + for _, c := range containers { + n, ok := c["name"].(string) + if ok && n != "" { + names = append(names, n) + } + } + return fmt.Errorf("EnsureWekaContainer: container with name %q not found; existing: %v", cfg.Name, names) + } + if handleErr := handleExistingContainer(ctx, cfg, found, resourcesDir); handleErr != nil { + return handleErr + } + } + + // Get number of cores for this container. + numCores := numCoresForMode(cfg) + var fullCores []string + var localRes map[string]interface{} + fullCores, err = cpuaffinity.FindFullCores(ctx, cfg, numCores) + if err != nil { + return fmt.Errorf("EnsureWekaContainer: find full cores: %w", err) + } + + localRes, err = getWekaLocalResources(ctx, cfg.Name) + if err != nil { + if cfg.Mode == "client" && strings.Contains(err.Error(), "resources.json.staging: No such file or directory") { + logger.Warn("client container corrupted state, recreating", "err", err) + if stopErr := cmdutil.Run(ctx, "weka", "local", "stop", "--force"); stopErr != nil { + logger.Warn("weka local stop --force failed during recreate (continuing)", "err", stopErr) + } + if rmErr := cmdutil.Run(ctx, "weka", "local", "rm", cfg.Name, "--force"); rmErr != nil { + logger.Warn("weka local rm --force failed during recreate (continuing)", "err", rmErr) + } + if err2 := createContainer(ctx, cfg); err2 != nil { + return err2 + } + localRes, err = getWekaLocalResources(ctx, cfg.Name) + if err != nil { + return err + } + } else { + return err + } + } + + if cfg.Mode == "client" && shouldRecreateClientContainer(cfg, localRes) { + logger.Info("recreating client container") + if stopErr := cmdutil.Run(ctx, "weka", "local", "stop", "--force"); stopErr != nil { + logger.Warn("weka local stop --force failed during recreate (continuing)", "err", stopErr) + } + if rmErr := cmdutil.Run(ctx, "weka", "local", "rm", cfg.Name, "--force"); rmErr != nil { + logger.Warn("weka local rm --force failed during recreate (continuing)", "err", rmErr) + } + if createErr := createContainer(ctx, cfg); createErr != nil { + return createErr + } + localRes, err = getWekaLocalResources(ctx, cfg.Name) + if err != nil { + return err + } + } + + // Reconfigure core count if needed. + if coresFlag, ok := modeCoresFlag[cfg.Mode]; ok { + nodes, nodesOK := localRes["nodes"].(map[string]interface{}) + if !nodesOK || len(nodes) != numCores+1 { + coreIDs := fullCores + if len(coreIDs) > numCores { + coreIDs = coreIDs[:numCores] + } + args := []string{"local", "resources", "cores", strconv.Itoa(numCores), + "-C", cfg.Name, coresFlag, "--core-ids", strings.Join(coreIDs, ",")} + if coresErr := cmdutil.Run(ctx, "weka", args...); coresErr != nil { + return fmt.Errorf("EnsureWekaContainer: reconfigure cores: %w", coresErr) + } + } + } + + // Re-fetch after potential core change. + localRes, err = getWekaLocalResources(ctx, cfg.Name) + if err != nil { + return err + } + + // Patch resource fields. + if cfg.Mode == "s3" || cfg.Mode == "nfs" || cfg.Mode == "smbw" { + localRes["allow_protocols"] = true + } + localRes["reserve_1g_hugepages"] = false + localRes["excluded_drivers"] = []string{"igb_uio"} + if cfg.Memory != "" { + if memBytes, memErr := convertToBytes(cfg.Memory); memErr == nil { + localRes["memory"] = memBytes + } else { + logger.Warn("failed to parse memory value, skipping", "memory", cfg.Memory, "err", memErr) + } + } + localRes["auto_discovery_enabled"] = false + localRes["ips"] = network.ManagementIPs + + dpdk := cfg.DPDKBaseMemMB + if dpdk == 0 { + dpdk = 64 + } + localRes["dpdk_base_memory_mb"] = dpdk + localRes["auto_remove_timeout"] = cfg.AutoRemoveTimeout + + // Join IPs / backend endpoints. + if len(cfg.JoinIPs) > 0 { + var endpoints []map[string]interface{} + for _, joinIP := range cfg.JoinIPs { + parts := strings.SplitN(joinIP, ":", 2) + if len(parts) != 2 { + continue + } + port, perr := strconv.Atoi(parts[1]) + if perr != nil { + logger.Warn("invalid join IP port, using 0", "joinIP", joinIP, "err", perr) + } + endpoints = append(endpoints, map[string]interface{}{"ip": parts[0], "port": port}) + } + localRes["backend_endpoints"] = endpoints + } + + // Binding restriction. + if cfg.Features.SupportsBindingToNotAllInterfaces { + localRes["restrict_listen"] = !cfg.BindManagementAll + } + + // NVIDIA VF single IP. + localRes["nvidia_vf_single_ip"] = cfg.NvidiaVFSingleIP + + // Net gateway. + if cfg.NetGateway != "" && !isUDP(cfg) { + netDevs, ok := localRes["net_devices"].([]interface{}) + if ok && len(netDevs) == 1 { + if devMap, ok := netDevs[0].(map[string]interface{}); ok { + devMap["gateway"] = cfg.NetGateway + } + } + } + + // Assign core IDs to nodes. + nodes, nodesOK := localRes["nodes"].(map[string]interface{}) + coresCursor := 0 + if nodesOK { + for _, nodeVal := range nodes { + node, ok := nodeVal.(map[string]interface{}) + if !ok { + continue + } + roles, rolesOK := node["roles"].([]interface{}) + isManagement := false + if rolesOK { + for _, r := range roles { + s, ok := r.(string) + if ok && s == "MANAGEMENT" { + isManagement = true + break + } + } + } + if isManagement { + continue + } + if cfg.CPUPolicy == "shared" { + node["dedicate_core"] = false + node["dedicated_mode"] = "NONE" + } else { + node["dedicate_core"] = true + } + if coresCursor < len(fullCores) { + coreID, coreIDErr := strconv.Atoi(fullCores[coresCursor]) + if coreIDErr != nil { + logger.Warn("invalid core ID, skipping", "coreID", fullCores[coresCursor], "err", coreIDErr) + } else { + node["core_id"] = coreID + } + coresCursor++ + } + } + } + + // Atomic write of patched resources. + resourceGen := fmt.Sprintf("%x", time.Now().UnixNano()) + fileName := fmt.Sprintf("weka-resources.%s.json", resourceGen) + resourceFile := filepath.Join(resourcesDir, fileName) + data, err := json.Marshal(localRes) + if err != nil { + return fmt.Errorf("EnsureWekaContainer: marshal resources: %w", err) + } + if err := os.WriteFile(resourceFile, data, 0o644); err != nil { + return fmt.Errorf("EnsureWekaContainer: write resources: %w", err) + } + if err := linkResourcesFile(ctx, fileName, resourcesDir); err != nil { + return err + } + + // Reconcile net devices. + desired := desiredNetDevices(cfg) + if err := network.ReconcileNetDevices(ctx, cfg.Name, desired); err != nil { + return fmt.Errorf("EnsureWekaContainer: reconcile net devices: %w", err) + } + + return nil +} + +// EnsureWekaVersion sets the active Weka version if not already set. +// Mirrors Python ensure_weka_version() at weka_runtime.py:2913. +func EnsureWekaVersion(ctx context.Context) error { + return cmdutil.Run(ctx, "sh", "-c", "weka version | grep '*' || weka version set $(weka version)") +} + +// ForceSetWekaVersion unconditionally pins the active Weka version. +// Used by ssdproxy mode (force_set=True) after creating the proxy container. +func ForceSetWekaVersion(ctx context.Context) error { + return cmdutil.Run(ctx, "sh", "-c", "weka version set $(weka version)") +} + +// WriteFeatureFlagsJSON atomically writes feature flags to /opt/weka/k8s-runtime/feature_flags.json. +// Mirrors Python write_feature_flags_json() at weka_runtime.py:1401. +func WriteFeatureFlagsJSON(ctx context.Context, cfg *config.Config) error { + data, err := json.Marshal(cfg.Features) + if err != nil { + return err + } + const tmp = "/opt/weka/k8s-runtime/feature_flags.json.tmp" + const dst = "/opt/weka/k8s-runtime/feature_flags.json" + if err := os.MkdirAll("/opt/weka/k8s-runtime", 0o755); err != nil { + return err + } + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return err + } + return os.Rename(tmp, dst) +} + +// WriteTelemetryConfigOverride writes the telemetry audit-traces config override atomically. +// Mirrors Python write_telemetry_config_override() at weka_runtime.py:3344. +func WriteTelemetryConfigOverride(ctx context.Context) error { + const auditDir = "/opt/weka/external-mounts/shared_boot_level/audit-traces" + const configPath = auditDir + "/override.config.json" + + if _, err := os.Stat(auditDir); os.IsNotExist(err) { + return nil + } + + // Get filesystem stats to compute minimumFreeSpace. + var stat syscall.Statfs_t + minimumFreeSpace := int64(5368709120) // fallback ~5GiB + if err := syscall.Statfs(auditDir, &stat); err == nil { + // Python uses f_blocks * f_frsize (weka_runtime.py:3360); Bsize is f_bsize which may differ + // on some NFS/btrfs mounts. statfsFragmentSize returns Frsize on Linux (f_frsize). + total := int64(stat.Blocks) * statfsFragmentSize(&stat) + minimumFreeSpace = total * 20 / 100 + } + + const tracesRetentionSize = 10 * 1024 * 1024 * 1024 // 10 GiB + + configOverride := map[string]interface{}{ + "global": map[string]interface{}{ + "dumping": map[string]interface{}{ + "histogramRetentionSize": 134217728, // 128 MiB + "maxHistograms": 30000, + "minimumFreeSpace": minimumFreeSpace, + "tracesRetentionSize": tracesRetentionSize, + }, + }, + } + + newContent, err := json.Marshal(configOverride) + if err != nil { + return err + } + + // Idempotent check. + if existing, err := os.ReadFile(configPath); err == nil { + if bytes.Equal(existing, newContent) { + return nil + } + } + + tmpPath := fmt.Sprintf("%s/.config.json.tmp.%d", auditDir, os.Getpid()) + if err := os.WriteFile(tmpPath, newContent, 0o644); err != nil { + return fmt.Errorf("WriteTelemetryConfigOverride write tmp: %w", err) + } + if err := os.Rename(tmpPath, configPath); err != nil { + _ = os.Remove(tmpPath) //nolint:errcheck // best-effort cleanup of orphaned temp file + return fmt.Errorf("WriteTelemetryConfigOverride rename: %w", err) + } + return nil +} + +// StartContainer starts the named container. +// Mirrors Python start_weka_container() at weka_runtime.py:2827. +func StartContainer(ctx context.Context, name string) error { + return cmdutil.Run(ctx, "weka", "local", "start", name) +} + +// ---- unexported helpers ---------------------------------------------------------------- + +// getContainers runs "weka local ps --json" and returns the parsed array. +func getContainers(ctx context.Context) ([]map[string]interface{}, error) { + out, err := cmdutil.Output(ctx, "weka", "local", "ps", "--json") + if err != nil { + return nil, fmt.Errorf("getContainers: %w", err) + } + var result []map[string]interface{} + if err := json.Unmarshal(out, &result); err != nil { + return nil, fmt.Errorf("getContainers: parse JSON: %w", err) + } + return result, nil +} + +// getWekaLocalResources runs "weka local resources -C name --json" and returns the parsed map. +func getWekaLocalResources(ctx context.Context, name string) (map[string]interface{}, error) { + out, err := cmdutil.Output(ctx, "weka", "local", "resources", "-C", name, "--json") + if err != nil { + return nil, fmt.Errorf("getWekaLocalResources(%s): %w", name, err) + } + var result map[string]interface{} + if err := json.Unmarshal(out, &result); err != nil { + return nil, fmt.Errorf("getWekaLocalResources(%s): parse JSON: %w", name, err) + } + return result, nil +} + +// shouldRecreateClientContainer returns true when the client container must be recreated. +// Mirrors Python should_recreate_client_container() at weka_runtime.py:2503. +// +// DELIBERATE DEVIATION from Python (weka_runtime.py:2503-2508): +// Python unconditionally checks `restricted_client is not True`, which always triggers +// recreation on 4.2.7.64 images (they never set restricted_client=True) causing an +// infinite recreate loop. Go instead computes the expected value from the image name +// (restricted_client should be True for all images except 4.2.7.64) and only recreates +// when the actual value differs from that expectation. Do not revert to Python's logic. +func shouldRecreateClientContainer(cfg *config.Config, res map[string]interface{}) bool { + // base_port: zero on missing/wrong type → mismatch with any real port → recreate. + basePort, basePortOK := res["base_port"].(float64) + if !basePortOK || int(basePort) != cfg.Port { + return true + } + expectedRestricted := !strings.Contains(cfg.ImageName, "4.2.7.64") + // restricted_client: false on missing/wrong type is the safe default for the comparison. + restricted, restrictedOK := res["restricted_client"].(bool) + if !restrictedOK { + restricted = false + } + return restricted != expectedRestricted +} + +// handleExistingContainer handles a container that already exists. +// Mirrors Python handle_existing_container() at weka_runtime.py:2571. +func handleExistingContainer(ctx context.Context, cfg *config.Config, container map[string]interface{}, resourcesDir string) error { + running, runningOK := container["isRunning"].(bool) + if runningOK && running { + return nil + } + status, statusOK := container["runStatus"].(string) + if statusOK && status == "Unknown" { + return checkResourcesJSON(ctx, cfg, resourcesDir) + } + return nil +} + +// checkResourcesJSON handles empty resources file by restoring from a backup or recreating. +// Mirrors Python check_resources_json() at weka_runtime.py:2538. +func checkResourcesJSON(ctx context.Context, cfg *config.Config, resourcesDir string) error { + _, logger := instrumentation.CreateLogSpan(ctx, "weka.checkResourcesJSON") + defer logger.End() + + resourcesFile := filepath.Join(resourcesDir, "resources.json") + info, err := os.Stat(resourcesFile) + if err != nil { + return fmt.Errorf("checkResourcesJSON: %w", err) + } + if info.Size() > 0 { + return nil // not empty, nothing to do + } + + // Find older non-empty weka-resources.*.json files. + entries, err := os.ReadDir(resourcesDir) + if err != nil { + return err + } + var candidates []string + for _, e := range entries { + if strings.HasPrefix(e.Name(), "weka-resources.") && strings.HasSuffix(e.Name(), ".json") { + full := filepath.Join(resourcesDir, e.Name()) + if fi, err := os.Stat(full); err == nil && fi.Size() > 0 { + candidates = append(candidates, e.Name()) + } + } + } + if len(candidates) == 0 { + // Recreate container. + if stopErr := cmdutil.Run(ctx, "weka", "local", "stop", "--force"); stopErr != nil { + logger.Warn("weka local stop --force failed during recreate (continuing)", "err", stopErr) + } + if rmErr := cmdutil.Run(ctx, "weka", "local", "rm", "--all", "--force"); rmErr != nil { + logger.Warn("weka local rm --all --force failed during recreate (continuing)", "err", rmErr) + } + return createContainer(ctx, cfg) + } + // Link the most-recently-modified candidate. + var latest string + var latestMod time.Time + for _, name := range candidates { + if fi, err := os.Stat(filepath.Join(resourcesDir, name)); err == nil { + if fi.ModTime().After(latestMod) { + latestMod = fi.ModTime() + latest = name + } + } + } + return linkResourcesFile(ctx, latest, resourcesDir) +} + +// linkResourcesFile creates the standard symlinks for a resource file. +// Mirrors Python link_resources_file() at weka_runtime.py:2581. +func linkResourcesFile(_ context.Context, fileName, resourcesDir string) error { + script := fmt.Sprintf(` +ln -sf %s %s/resources.json +ln -sf %s %s/resources.json.stable +ln -sf %s %s/resources.json.staging +`, fileName, resourcesDir, + fileName, resourcesDir, + fileName, resourcesDir) + cmd := fmt.Sprintf("cd %s && %s", resourcesDir, script) + return cmdutil.Run(context.Background(), "sh", "-c", cmd) +} + +// createContainer builds and runs the "weka local setup container" command. +// Mirrors Python create_container() at weka_runtime.py:2312. +func createContainer(ctx context.Context, cfg *config.Config) error { + _, logger := instrumentation.CreateLogSpan(ctx, "weka.createContainer", "name", cfg.Name) + defer logger.End() + + numCores := numCoresForMode(cfg) + fullCores, err := cpuaffinity.FindFullCores(ctx, cfg, numCores) + if err != nil { + return fmt.Errorf("createContainer: find cores: %w", err) + } + coreStr := strings.Join(fullCores, ",") + modeFlag := modeCoresFlag[cfg.Mode] + + // Join secret. + joinSecretFlag := "" + joinSecretCmd := "" + if _, err := os.Stat("/var/run/secrets/weka-operator/operator-user/join-secret"); err == nil { + joinSecretFlag = "--join-secret" + if cfg.Mode == "client" { + joinSecretFlag = "--join-token" + } + joinSecretCmd = "$(cat /var/run/secrets/weka-operator/operator-user/join-secret)" + } + + // Network flags. + var netStr string + switch { + case network.ShouldAllocateVFPerIoNode(cfg.NetworkDevice): + devices := make([]string, 0) + for _, dev := range strings.Split(cfg.NetworkDevice, ",") { + bare := strings.TrimPrefix(dev, "vf_") + devices = append(devices, "--net "+bare) + } + netStr = strings.Join(devices, " ") + " --management-ips " + strings.Join(network.ManagementIPs, ",") + default: + // UDP mode and bare-metal both start with "--net udp"; + // bare-metal reconcile adds NICs later via ReconcileNetDevices. + netStr = "--net udp" + } + + // Build command parts. + parts := []string{ + "weka", "local", "setup", "container", + "--name", cfg.Name, + "--no-start", "--disable", + "--core-ids", coreStr, + "--cores", strconv.Itoa(numCores), + } + if modeFlag != "" { + parts = append(parts, strings.Fields(modeFlag)...) + } + parts = append(parts, strings.Fields(netStr)...) + parts = append(parts, "--base-port", strconv.Itoa(cfg.Port)) + + if joinSecretCmd != "" { + parts = append(parts, joinSecretFlag, joinSecretCmd) + } + if len(cfg.JoinIPs) > 0 { + parts = append(parts, "--join-ips", strings.Join(cfg.JoinIPs, ",")) + } + if cfg.Mode == "client" { + parts = append(parts, "--client") + if !strings.Contains(cfg.ImageName, "4.2.7.64") { + parts = append(parts, "--restricted") + } + } + if cfg.FailureDomain != "" { + parts = append(parts, "--failure-domain", cfg.FailureDomain) + } + if cfg.Mode == "data-services" { + parts = append(parts, "--allow-mix-setting") + } + + // Run via shell to allow $(cat ...) expansion. + cmdStr := strings.Join(parts, " ") + logger.Info("creating container", "cmd", cmdStr) + if err := cmdutil.Run(ctx, "sh", "-c", cmdStr); err != nil { + return fmt.Errorf("createContainer: %w", err) + } + + // For bare-metal (non-VF, non-UDP): reconcile net devices after creation. + if !network.ShouldAllocateVFPerIoNode(cfg.NetworkDevice) && !isUDP(cfg) { + desired := desiredNetDevices(cfg) + if err := network.ReconcileNetDevices(ctx, cfg.Name, desired); err != nil { + return fmt.Errorf("createContainer: reconcile net devices: %w", err) + } + } + + return nil +} + +// desiredNetDevices computes the list of net devices the container should have. +func desiredNetDevices(cfg *config.Config) []string { + if cfg.NetworkDevice == "" { + return nil + } + return strings.Split(cfg.NetworkDevice, ",") +} + +// numCoresForMode returns the number of cores for the current config. +// Python: NUM_CORES = int(os.environ.get("CORES", 0)) and the list is per-role. +func numCoresForMode(cfg *config.Config) int { + if len(cfg.Cores) > 0 { + return cfg.Cores[0] + } + return 0 +} + +// isUDP mirrors Python is_udp(). +func isUDP(cfg *config.Config) bool { + return cfg.UDPMode || strings.EqualFold(cfg.NetworkDevice, "udp") +} + +// convertToBytes parses a human-readable size string like "1GiB", "512MiB" into bytes. +// Mirrors Python convert_to_bytes() at weka_runtime.py:2511. +func convertToBytes(memory string) (int64, error) { + upper := strings.ToUpper(strings.TrimSpace(memory)) + re := regexp.MustCompile(`^(\d+)([KMGTPE]I?B?)$`) + matches := re.FindStringSubmatch(upper) + if len(matches) != 3 { + return 0, fmt.Errorf("invalid size: %q", memory) + } + size, parseErr := strconv.ParseInt(matches[1], 10, 64) + if parseErr != nil { + return 0, fmt.Errorf("convertToBytes: parse size %q: %w", matches[1], parseErr) + } + multipliers := map[string]int64{ + "B": 1, + "KB": 1e3, "MB": 1e6, "GB": 1e9, "TB": 1e12, "PB": 1e15, "EB": 1e18, + "KIB": 1 << 10, "MIB": 1 << 20, "GIB": 1 << 30, + "TIB": 1 << 40, "PIB": 1 << 50, + } + unit := matches[2] + mult, ok := multipliers[unit] + if !ok { + return 0, fmt.Errorf("unknown unit: %q", unit) + } + return size * mult, nil +} diff --git a/internal/runtime/weka/local.go b/internal/runtime/weka/local.go index 18a583d8e..9a063afd0 100644 --- a/internal/runtime/weka/local.go +++ b/internal/runtime/weka/local.go @@ -35,7 +35,7 @@ func StartStemContainer(_ context.Context) error { } // EnsureContainerExec polls until the named container accepts exec commands. -// Polls every 1s with a 300s total timeout. +// Polls every 2s with a 300s total timeout, matching Python asyncio.sleep(2) at weka_runtime.py:3055. // Mirrors Python ensure_container_exec() at weka_runtime.py:3055. func EnsureContainerExec(ctx context.Context, name string) error { deadline := time.Now().Add(300 * time.Second) @@ -50,7 +50,7 @@ func EnsureContainerExec(ctx context.Context, name string) error { select { case <-ctx.Done(): return ctx.Err() - case <-time.After(1 * time.Second): + case <-time.After(2 * time.Second): } } } @@ -68,20 +68,20 @@ func ConfigureTraces(ctx context.Context, cfg *config.Config, name string) error } const ( - oldFullLocation = "/data/reserved_space/dumper_config.json.override" - legacyPartialLoc = "/data/reserved_space/dumper_config_overrides.json" - newPartialLoc = "/traces/config_overrides.json" - stagingPath = "/opt/weka/k8s-scripts/dumper_config.json.override" + oldFullLocation = "/data/reserved_space/dumper_config.json.override" + legacyPartialLoc = "/data/reserved_space/dumper_config_overrides.json" + newPartialLoc = "/traces/config_overrides.json" + stagingPath = "/opt/weka/k8s-scripts/dumper_config.json.override" ) switch mode { case "override": data := map[string]interface{}{ - "enabled": true, + "enabled": true, "ensure_free_space_bytes": cfg.EnsureFreeSpaceGB * 1024 * 1024 * 1024, - "retention_bytes": cfg.MaxTraceCapacityGB * 1024 * 1024 * 1024, - "retention_type": "BYTES", - "version": 1, + "retention_bytes": cfg.MaxTraceCapacityGB * 1024 * 1024 * 1024, + "retention_type": "BYTES", + "version": 1, "freeze_period": map[string]interface{}{ "start_time": "0001-01-01T00:00:00+00:00", "end_time": "0001-01-01T00:00:00+00:00", @@ -93,8 +93,8 @@ func ConfigureTraces(ctx context.Context, cfg *config.Config, name string) error case "partial-override": data := map[string]interface{}{ "ensure_free_space_bytes": cfg.EnsureFreeSpaceGB * 1024 * 1024 * 1024, - "retention_bytes": cfg.MaxTraceCapacityGB * 1024 * 1024 * 1024, - "retention_type": "BYTES", + "retention_bytes": cfg.MaxTraceCapacityGB * 1024 * 1024 * 1024, + "retention_type": "BYTES", } dest := legacyPartialLoc if cfg.Features.TracesOverrideInSlashTraces { @@ -121,7 +121,7 @@ func ConfigureTraces(ctx context.Context, cfg *config.Config, name string) error ensureFreeBytes = cfg.EnsureFreeSpaceGB * 1024 * 1024 * 1024 } ssdCfg := map[string]interface{}{ - "enabled": true, + "enabled": true, "ensure_free_space_bytes": ensureFreeBytes, "freeze_period": map[string]interface{}{ "comment": "", diff --git a/internal/runtime/weka/statfs_linux.go b/internal/runtime/weka/statfs_linux.go new file mode 100644 index 000000000..2daab47ac --- /dev/null +++ b/internal/runtime/weka/statfs_linux.go @@ -0,0 +1,13 @@ +//go:build linux + +package weka + +import "syscall" + +// statfsFragmentSize returns the fragment size (f_frsize) from a Statfs_t. +// On Linux, Statfs_t.Frsize corresponds to statvfs f_frsize, which Python uses +// in weka_runtime.py:3360 as `stat.f_frsize`. This differs from Bsize (f_bsize) +// on some NFS/btrfs mounts. +func statfsFragmentSize(stat *syscall.Statfs_t) int64 { + return int64(stat.Frsize) +} diff --git a/internal/runtime/weka/statfs_other.go b/internal/runtime/weka/statfs_other.go new file mode 100644 index 000000000..9de206952 --- /dev/null +++ b/internal/runtime/weka/statfs_other.go @@ -0,0 +1,12 @@ +//go:build !linux + +package weka + +import "syscall" + +// statfsFragmentSize returns the block size from a Statfs_t for non-Linux platforms. +// On Darwin/other, Statfs_t does not have Frsize; fall back to Bsize. +// This path is compile-only; the runtime binary only runs on Linux. +func statfsFragmentSize(stat *syscall.Statfs_t) int64 { + return int64(stat.Bsize) +} diff --git a/internal/runtime/wekadrive/discover.go b/internal/runtime/wekadrive/discover.go index 115b0f25e..6a1566ac2 100644 --- a/internal/runtime/wekadrive/discover.go +++ b/internal/runtime/wekadrive/discover.go @@ -48,12 +48,7 @@ func FindWekaPartitions(ctx context.Context) ([]domain.DriveInfo, error) { signature = "" } - isSigned := signature != "" && signature != unsignedDriveSignature - wekaGUID := "" - if isSigned && len(signature) == 32 { - wekaGUID = fmt.Sprintf("%s-%s-%s-%s-%s", - signature[0:8], signature[8:12], signature[12:16], signature[16:20], signature[20:32]) - } + isSigned, wekaGUID := driveSignatureInfo(signature) // Resolve partition block device to its parent disk pciDevPath, err := filepath.EvalSymlinks(fmt.Sprintf("/sys/class/block/%s", partName)) @@ -80,6 +75,18 @@ func FindWekaPartitions(ctx context.Context) ([]domain.DriveInfo, error) { return drives, nil } +// driveSignatureInfo interprets a raw drive signature string and returns whether the drive is +// signed and (for valid 32-hex-char signatures) the Weka GUID in dashed 8-4-4-4-12 UUID form. +// An empty string or the unsignedDriveSignature constant means the drive is unsigned. +func driveSignatureInfo(signature string) (isSigned bool, wekaGUID string) { + isSigned = signature != "" && signature != unsignedDriveSignature + if isSigned && len(signature) == 32 { + wekaGUID = fmt.Sprintf("%s-%s-%s-%s-%s", + signature[0:8], signature[8:12], signature[12:16], signature[16:20], signature[20:32]) + } + return isSigned, wekaGUID +} + // collectPartNames collects unique partition names from /dev/disk/by-path/ and /dev/disk/by-id/. func collectPartNames(_ context.Context) ([]string, error) { var partNames []string diff --git a/internal/runtime/wekadrive/discover_test.go b/internal/runtime/wekadrive/discover_test.go new file mode 100644 index 000000000..fe0fec950 --- /dev/null +++ b/internal/runtime/wekadrive/discover_test.go @@ -0,0 +1,61 @@ +package wekadrive + +import "testing" + +func TestDriveSignatureInfo(t *testing.T) { + tests := []struct { + name string + signature string + wantSigned bool + wantGUID string + }{ + { + name: "empty string — unsigned, no GUID", + signature: "", + wantSigned: false, + wantGUID: "", + }, + { + name: "unsignedDriveSignature constant — unsigned, no GUID", + signature: unsignedDriveSignature, + wantSigned: false, + wantGUID: "", + }, + { + name: "valid 32-hex-char — signed, correctly dashed 8-4-4-4-12 GUID", + signature: "90f0090f90f0090f90f0090f90f0090e", // differs from unsigned by last char + wantSigned: true, + wantGUID: "90f0090f-90f0-090f-90f0-090f90f0090e", + }, + { + name: "another valid 32-hex signature", + signature: "aabbccdd11223344aabbccdd11223344", + wantSigned: true, + wantGUID: "aabbccdd-1122-3344-aabb-ccdd11223344", + }, + { + name: "signed but wrong length (30 chars) — signed, empty GUID", + signature: "90f0090f90f0090f90f0090f90f009", // 30 chars, not unsigned + wantSigned: true, + wantGUID: "", + }, + { + name: "signed but wrong length (34 chars) — signed, empty GUID", + signature: "90f0090f90f0090f90f0090f90f0090f00", // 34 chars + wantSigned: true, + wantGUID: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotSigned, gotGUID := driveSignatureInfo(tt.signature) + if gotSigned != tt.wantSigned { + t.Errorf("driveSignatureInfo(%q) isSigned = %v; want %v", tt.signature, gotSigned, tt.wantSigned) + } + if gotGUID != tt.wantGUID { + t.Errorf("driveSignatureInfo(%q) wekaGUID = %q; want %q", tt.signature, gotGUID, tt.wantGUID) + } + }) + } +} diff --git a/internal/runtime/wekadrive/ensure.go b/internal/runtime/wekadrive/ensure.go new file mode 100644 index 000000000..4686a2070 --- /dev/null +++ b/internal/runtime/wekadrive/ensure.go @@ -0,0 +1,91 @@ +// ensure.go implements drive verification for the wekadrive package. +// Mirrors ensure_drives, assert_vfio_pci_loaded_if_required, has_iommu_groups +// at weka_runtime.py:3874–3909. +package wekadrive + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/weka/go-weka-observability/instrumentation" + "github.com/weka/weka-operator/internal/pkg/osinfo" + "github.com/weka/weka-operator/internal/runtime/cmdutil" + "github.com/weka/weka-operator/internal/runtime/config" +) + +// EnsureDrives validates VFIO-PCI is loaded if required, then matches requested drives +// against the system and writes the result to /opt/weka/k8s-runtime/drives.json. +// Mirrors Python ensure_drives() at weka_runtime.py:3890. +func EnsureDrives(ctx context.Context, cfg *config.Config) error { + _, logger := instrumentation.CreateLogSpan(ctx, "wekadrive.EnsureDrives") + defer logger.End() + + if err := assertVFIOPCILoaded(ctx, cfg); err != nil { + return err + } + + sysDrives, err := FindWekaPartitions(ctx) + if err != nil { + return fmt.Errorf("EnsureDrives: find partitions: %w", err) + } + + reqSet := make(map[string]struct{}, len(cfg.Drives)) + for _, s := range cfg.Drives { + reqSet[s] = struct{}{} + } + + // Filter to drives whose serial is in the requested set. + var matched []interface{} + for _, d := range sysDrives { + if _, ok := reqSet[d.SerialId]; ok { + matched = append(matched, d) + } + } + + logger.Info("drive reconciliation", "sys_drives", len(sysDrives), "requested", len(cfg.Drives), "matched", len(matched)) + + err = os.MkdirAll("/opt/weka/k8s-runtime", 0o755) + if err != nil { + return err + } + var data []byte + data, err = json.Marshal(matched) + if err != nil { + return err + } + return os.WriteFile("/opt/weka/k8s-runtime/drives.json", data, 0o644) +} + +// assertVFIOPCILoaded checks that vfio_pci is loaded when IOMMU groups are present or on COS. +// Mirrors Python assert_vfio_pci_loaded_if_required() at weka_runtime.py:3874. +func assertVFIOPCILoaded(ctx context.Context, cfg *config.Config) error { + _, logger := instrumentation.CreateLogSpan(ctx, "wekadrive.assertVFIOPCILoaded") + defer logger.End() + + nodeInfo, err := osinfo.Load() + isCOS := err == nil && nodeInfo.IsCos() + + hasIOMMU, iommuErr := hasIOMMUGroups(ctx) + if iommuErr != nil { + logger.Warn("failed to detect IOMMU groups, assuming none", "err", iommuErr) + } + + if isCOS || hasIOMMU { + if err := cmdutil.Run(ctx, "sh", "-c", "lsmod | grep -w vfio_pci"); err != nil { + return fmt.Errorf("vfio_pci module is required for drives but is not loaded: %w", err) + } + } + return nil +} + +// hasIOMMUGroups checks whether /sys/kernel/iommu_groups/ is non-empty. +// Mirrors Python has_iommu_groups() at weka_runtime.py:3855. +func hasIOMMUGroups(_ context.Context) (bool, error) { + entries, err := os.ReadDir("/sys/kernel/iommu_groups") + if err != nil { + return false, nil // directory missing or inaccessible — treat as no IOMMU + } + return len(entries) > 0, nil +} diff --git a/internal/runtime/wekadrive/sign.go b/internal/runtime/wekadrive/sign.go index 7ea649212..43bd64b19 100644 --- a/internal/runtime/wekadrive/sign.go +++ b/internal/runtime/wekadrive/sign.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "os" + "os/exec" "strings" "github.com/weka/go-weka-observability/instrumentation" @@ -70,6 +71,72 @@ type signDriveWekaInfo struct { IsProxy bool `json:"is_proxy"` } +// parseSignDriveListJSON is a thin helper that unmarshals raw JSON into a signDriveListOutput. +// It is used by both the production callers and tests. +func parseSignDriveListJSON(data []byte, out *signDriveListOutput) error { + return json.Unmarshal(data, out) +} + +// filterClusterGUIDDrives returns a serial→path map from a parsed list, keeping only devices +// that have a non-empty WekaInfo.ClusterGUID. Devices with empty serial, empty path, or nil +// WekaInfo are skipped. hardware.Path takes priority over the top-level path field. +func filterClusterGUIDDrives(parsed signDriveListOutput) map[string]string { + result := make(map[string]string, len(parsed.Devices)) + for _, dev := range parsed.Devices { + // M9 review: plan stated Python reads top-level device['serial'], but the actual + // Python code at weka_runtime.py:513-515 reads hardware.get('serial_number') — + // identical to dev.Hardware.SerialNumber here. No change needed; already correct. + serial := dev.Hardware.SerialNumber + path := dev.Hardware.Path + if path == "" { + path = dev.Path + } + if serial == "" || path == "" { + continue + } + if dev.WekaInfo == nil || dev.WekaInfo.ClusterGUID == "" { + continue + } + result[serial] = path + } + return result +} + +// extractProxyDrives returns SharedDriveInfo for every proxy-signed drive in a parsed list. +// A drive qualifies when: +// - status == "weka_formatted" +// - WekaInfo != nil AND (clusterGUID matches proxySignedGUID, or equals "proxy guid", or IsProxy) +// - PhysicalUUID is non-empty +// - SizeBytes > 0 +func extractProxyDrives(parsed signDriveListOutput) []domain.SharedDriveInfo { + var drives []domain.SharedDriveInfo + for _, dev := range parsed.Devices { + if dev.Status != "weka_formatted" { + continue + } + if dev.WekaInfo == nil { + continue + } + clusterGUID := strings.ToLower(dev.WekaInfo.ClusterGUID) + if clusterGUID != proxySignedGUID && clusterGUID != "proxy guid" && !dev.WekaInfo.IsProxy { + continue + } + if dev.PhysicalUUID == "" { + continue + } + if dev.Hardware.SizeBytes <= 0 { + continue + } + drives = append(drives, domain.SharedDriveInfo{ + PhysicalUUID: dev.PhysicalUUID, + Serial: dev.Hardware.SerialNumber, + CapacityGiB: int(dev.Hardware.SizeBytes / (1024 * 1024 * 1024)), + Type: iuSizeToDriveType(dev.Hardware.IuSize), + }) + } + return drives +} + // GetDrivesWithClusterGUID runs `weka-sign-drive list -j` and returns a map of serial → path // for drives that have a cluster_guid (i.e. are claimed by a Weka cluster). // If useProxySocket is true and the socket file exists, the proxy socket is used. @@ -93,25 +160,11 @@ func GetDrivesWithClusterGUID(ctx context.Context, useProxySocket bool) (map[str } var parsed signDriveListOutput - if jsonErr := json.Unmarshal(out, &parsed); jsonErr != nil { + if jsonErr := parseSignDriveListJSON(out, &parsed); jsonErr != nil { return nil, fmt.Errorf("weka-sign-drive list: JSON parse: %w", jsonErr) } - result := make(map[string]string, len(parsed.Devices)) - for _, dev := range parsed.Devices { - serial := dev.Hardware.SerialNumber - path := dev.Hardware.Path - if path == "" { - path = dev.Path - } - if serial == "" || path == "" { - continue - } - if dev.WekaInfo == nil || dev.WekaInfo.ClusterGUID == "" { - continue - } - result[serial] = path - } + result := filterClusterGUIDDrives(parsed) logger.Info("done", "count", len(result)) return result, nil } @@ -147,40 +200,25 @@ func ListAllProxyDrives(ctx context.Context) ([]domain.SharedDriveInfo, error) { out = out[jsonStart:] var parsed signDriveListOutput - if jsonErr := json.Unmarshal(out, &parsed); jsonErr != nil { + if jsonErr := parseSignDriveListJSON(out, &parsed); jsonErr != nil { return nil, fmt.Errorf("weka-sign-drive list: JSON parse: %w", jsonErr) } - var drives []domain.SharedDriveInfo - for _, dev := range parsed.Devices { - if dev.Status != "weka_formatted" { - continue - } - if dev.WekaInfo == nil { - continue - } - clusterGUID := strings.ToLower(dev.WekaInfo.ClusterGUID) - if clusterGUID != proxySignedGUID && clusterGUID != "proxy guid" && !dev.WekaInfo.IsProxy { - continue - } - if dev.PhysicalUUID == "" { - continue - } - if dev.Hardware.SizeBytes <= 0 { - logger.Warn("skipping drive with zero size_bytes", "path", dev.Path) - continue - } - drives = append(drives, domain.SharedDriveInfo{ - PhysicalUUID: dev.PhysicalUUID, - Serial: dev.Hardware.SerialNumber, - CapacityGiB: int(dev.Hardware.SizeBytes / (1024 * 1024 * 1024)), - Type: iuSizeToDriveType(dev.Hardware.IuSize), - }) - } + drives := extractProxyDrives(parsed) logger.Info("done", "count", len(drives)) return drives, nil } +// runWithStderr runs a command and returns stdout, stderr, and any error. +// Used when callers need to inspect stderr independently of the error value. +func runWithStderr(ctx context.Context, name string, args ...string) (stdout, stderr []byte, err error) { + var stderrBuf bytes.Buffer + cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // args are controlled by internal callers + cmd.Stderr = &stderrBuf + stdout, err = cmd.Output() + return stdout, stderrBuf.Bytes(), err +} + // SignBatch signs paths in a single batch invocation of weka-sign-drive. // Falls back to per-device signing if the batch fails. // Returns the list of successfully signed paths. @@ -252,8 +290,22 @@ func SignBatchProxy(ctx context.Context, paths []string, opts *SignOptions) ([]d for _, p := range paths { perArgs := append([]string{"sign", "proxy"}, flags...) perArgs = append(perArgs, "--", p) - if _, perErr := cmdutil.Output(ctx, "/weka-sign-drive", perArgs...); perErr != nil { - logger.Error(perErr, "failed to sign device for proxy", "path", p) + _, perStderr, perErr := runWithStderr(ctx, "/weka-sign-drive", perArgs...) + if perErr != nil { + // Python sign_device_path_for_proxy (weka_runtime.py:559-581): if stderr contains + // "already a Weka partition" the drive is already proxy-signed — not an error. + // Read existing drive metadata and include the drive in the result set. + if strings.Contains(string(perStderr), "already a Weka partition") { + logger.Info("device already proxy-signed, reading existing metadata", "path", p) + info, infoErr := GetProxyDriveInfo(ctx, p) + if infoErr != nil { + logger.Warn("failed to get proxy drive info for already-signed device", "path", p, "err", infoErr) + continue + } + infos = append(infos, info) + continue + } + logger.Error(perErr, "failed to sign device for proxy", "path", p, "stderr", string(perStderr)) continue } info, infoErr := GetProxyDriveInfo(ctx, p) diff --git a/internal/runtime/wekadrive/sign_test.go b/internal/runtime/wekadrive/sign_test.go new file mode 100644 index 000000000..7bd434cb7 --- /dev/null +++ b/internal/runtime/wekadrive/sign_test.go @@ -0,0 +1,515 @@ +package wekadrive + +import ( + "reflect" + "testing" + + "github.com/weka/weka-operator/internal/pkg/domain" +) + +// --------------------------------------------------------------------------- +// buildSignFlags +// --------------------------------------------------------------------------- + +func TestBuildSignFlags(t *testing.T) { + tests := []struct { + name string + opts *SignOptions + want []string + }{ + { + name: "nil opts returns nil", + opts: nil, + want: nil, + }, + { + name: "all false returns empty (non-nil) via no appends", + opts: &SignOptions{}, + want: nil, + }, + { + name: "AllowEraseWekaPartitions only", + opts: &SignOptions{AllowEraseWekaPartitions: true}, + want: []string{"--allow-erase-weka-partitions"}, + }, + { + name: "AllowEraseNonWekaPartitions only", + opts: &SignOptions{AllowEraseNonWekaPartitions: true}, + want: []string{"--allow-erase-non-weka-partitions"}, + }, + { + name: "AllowNonEmptyDevice only", + opts: &SignOptions{AllowNonEmptyDevice: true}, + want: []string{"--allow-non-empty-device"}, + }, + { + name: "SkipTrimFormat only", + opts: &SignOptions{SkipTrimFormat: true}, + want: []string{"--skip-trim-format"}, + }, + { + name: "all four true — declared order", + opts: &SignOptions{ + AllowEraseWekaPartitions: true, + AllowEraseNonWekaPartitions: true, + AllowNonEmptyDevice: true, + SkipTrimFormat: true, + }, + want: []string{ + "--allow-erase-weka-partitions", + "--allow-erase-non-weka-partitions", + "--allow-non-empty-device", + "--skip-trim-format", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildSignFlags(tt.opts) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("buildSignFlags(%+v) = %v; want %v", tt.opts, got, tt.want) + } + }) + } +} + +// --------------------------------------------------------------------------- +// iuSizeToDriveType +// --------------------------------------------------------------------------- + +func TestIuSizeToDriveType(t *testing.T) { + tests := []struct { + name string + iuSize int + want string + }{ + {name: "zero -> TLC", iuSize: 0, want: "TLC"}, + {name: "4096 -> TLC", iuSize: 4096, want: "TLC"}, + {name: "16383 boundary-1 -> TLC", iuSize: 16383, want: "TLC"}, + {name: "16384 boundary -> QLC", iuSize: 16384, want: "QLC"}, + {name: "32768 above -> QLC", iuSize: 32768, want: "QLC"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := iuSizeToDriveType(tt.iuSize) + if got != tt.want { + t.Errorf("iuSizeToDriveType(%d) = %q; want %q", tt.iuSize, got, tt.want) + } + }) + } +} + +// --------------------------------------------------------------------------- +// filterClusterGUIDDrives +// --------------------------------------------------------------------------- + +// realSignDriveListJSON is representative of actual `weka-sign-drive list -j` output: +// 6 weka_formatted drives with cluster_guid=null, is_proxy=false, hardware has no "path" +// key (only top-level "path"), plus 2 excluded drives with weka_info=null. +const realSignDriveListJSON = `{ + "devices": [ + { + "path": "/dev/nvme0n1", + "status": "weka_formatted", + "physical_uuid": "55943511-a49e-4dec-ba9a-20d084917627", + "weka_info": { + "cluster_guid": null, + "is_proxy": false + }, + "hardware": { + "serial_number": "22184A1936FE", + "size_bytes": 7681501126656, + "iu_size": 4096 + } + }, + { + "path": "/dev/nvme1n1", + "status": "weka_formatted", + "physical_uuid": "66a12345-b59f-4dec-bb9a-31e095028738", + "weka_info": { + "cluster_guid": null, + "is_proxy": false + }, + "hardware": { + "serial_number": "22184A1936FF", + "size_bytes": 7681501126656, + "iu_size": 4096 + } + }, + { + "path": "/dev/nvme2n1", + "status": "weka_formatted", + "physical_uuid": "77b23456-c60g-4dec-cc9a-42f106139849", + "weka_info": { + "cluster_guid": null, + "is_proxy": false + }, + "hardware": { + "serial_number": "22184A1937AA", + "size_bytes": 7681501126656, + "iu_size": 4096 + } + }, + { + "path": "/dev/nvme3n1", + "status": "weka_formatted", + "physical_uuid": "88c34567-d71h-4dec-dd9a-53g21724095a", + "weka_info": { + "cluster_guid": null, + "is_proxy": false + }, + "hardware": { + "serial_number": "22184A1937BB", + "size_bytes": 7681501126656, + "iu_size": 4096 + } + }, + { + "path": "/dev/nvme4n1", + "status": "weka_formatted", + "physical_uuid": "99d45678-e82i-4dec-ee9a-64h3283840ab", + "weka_info": { + "cluster_guid": null, + "is_proxy": false + }, + "hardware": { + "serial_number": "22184A1937CC", + "size_bytes": 7681501126656, + "iu_size": 4096 + } + }, + { + "path": "/dev/nvme5n1", + "status": "weka_formatted", + "physical_uuid": "aae56789-f93j-4dec-ff9a-75i439495bc", + "weka_info": { + "cluster_guid": null, + "is_proxy": false + }, + "hardware": { + "serial_number": "22184A1937DD", + "size_bytes": 7681501126656, + "iu_size": 4096 + } + }, + { + "path": "/dev/nvme6n1", + "status": "excluded", + "physical_uuid": "", + "weka_info": null, + "hardware": { + "serial_number": "22184A1937EE", + "size_bytes": 7681501126656, + "iu_size": 4096 + } + }, + { + "path": "/dev/nvme7n1", + "status": "excluded", + "physical_uuid": "", + "weka_info": null, + "hardware": { + "serial_number": "22184A1937FF", + "size_bytes": 7681501126656, + "iu_size": 4096 + } + } + ] +}` + +func mustParseSignDriveList(t *testing.T, jsonStr string) signDriveListOutput { + t.Helper() + var out signDriveListOutput + if err := parseSignDriveListJSON([]byte(jsonStr), &out); err != nil { + t.Fatalf("JSON parse failed: %v", err) + } + return out +} + +func TestFilterClusterGUIDDrives(t *testing.T) { + t.Run("real fixture — no cluster_guid — empty map", func(t *testing.T) { + parsed := mustParseSignDriveList(t, realSignDriveListJSON) + got := filterClusterGUIDDrives(parsed) + if len(got) != 0 { + t.Errorf("expected empty map, got %v", got) + } + // must not panic on weka_info=null devices (the excluded ones) + }) + + t.Run("device with cluster_guid — top-level path used as fallback", func(t *testing.T) { + // hardware has no "path" key; only top-level path should be used. + const json = `{ + "devices": [ + { + "path": "/dev/nvmeXn1", + "status": "weka_formatted", + "physical_uuid": "some-uuid", + "weka_info": { "cluster_guid": "some-guid-123", "is_proxy": false }, + "hardware": { "serial_number": "SER1", "size_bytes": 1000000000, "iu_size": 4096 } + } + ] + }` + parsed := mustParseSignDriveList(t, json) + got := filterClusterGUIDDrives(parsed) + want := map[string]string{"SER1": "/dev/nvmeXn1"} + if !reflect.DeepEqual(got, want) { + t.Errorf("got %v; want %v", got, want) + } + }) + + t.Run("device with cluster_guid — hardware path takes priority over top-level path", func(t *testing.T) { + const json = `{ + "devices": [ + { + "path": "/dev/nvmeXn1", + "status": "weka_formatted", + "physical_uuid": "some-uuid", + "weka_info": { "cluster_guid": "some-guid-123", "is_proxy": false }, + "hardware": { "serial_number": "SER2", "path": "/dev/hwpath", "size_bytes": 1000000000, "iu_size": 4096 } + } + ] + }` + parsed := mustParseSignDriveList(t, json) + got := filterClusterGUIDDrives(parsed) + want := map[string]string{"SER2": "/dev/hwpath"} + if !reflect.DeepEqual(got, want) { + t.Errorf("got %v; want %v", got, want) + } + }) + + t.Run("empty serial — skipped", func(t *testing.T) { + const json = `{ + "devices": [ + { + "path": "/dev/nvmeXn1", + "status": "weka_formatted", + "physical_uuid": "some-uuid", + "weka_info": { "cluster_guid": "some-guid-123", "is_proxy": false }, + "hardware": { "serial_number": "", "size_bytes": 1000000000, "iu_size": 4096 } + } + ] + }` + parsed := mustParseSignDriveList(t, json) + got := filterClusterGUIDDrives(parsed) + if len(got) != 0 { + t.Errorf("expected empty map for missing serial, got %v", got) + } + }) + + t.Run("weka_info nil — skipped", func(t *testing.T) { + const json = `{ + "devices": [ + { + "path": "/dev/nvmeXn1", + "status": "excluded", + "physical_uuid": "", + "weka_info": null, + "hardware": { "serial_number": "SER3", "size_bytes": 1000000000, "iu_size": 4096 } + } + ] + }` + parsed := mustParseSignDriveList(t, json) + got := filterClusterGUIDDrives(parsed) + if len(got) != 0 { + t.Errorf("expected empty map for nil weka_info, got %v", got) + } + }) +} + +// --------------------------------------------------------------------------- +// extractProxyDrives +// --------------------------------------------------------------------------- + +func TestExtractProxyDrives(t *testing.T) { + t.Run("real fixture — no proxy drives — empty slice", func(t *testing.T) { + parsed := mustParseSignDriveList(t, realSignDriveListJSON) + got := extractProxyDrives(parsed) + if len(got) != 0 { + t.Errorf("expected empty slice, got %v", got) + } + // must not panic on weka_info=null devices + }) + + t.Run("is_proxy=true TLC (iu_size=4096)", func(t *testing.T) { + const json = `{ + "devices": [ + { + "path": "/dev/nvmeXn1", + "status": "weka_formatted", + "physical_uuid": "uuid-1", + "weka_info": { "cluster_guid": "some-guid", "is_proxy": true }, + "hardware": { + "serial_number": "SERP1", + "size_bytes": 17179869184, + "iu_size": 4096 + } + } + ] + }` + // 17179869184 = 16 * 1024 * 1024 * 1024 = 16 GiB + parsed := mustParseSignDriveList(t, json) + got := extractProxyDrives(parsed) + if len(got) != 1 { + t.Fatalf("expected 1 drive, got %d: %v", len(got), got) + } + want := domain.SharedDriveInfo{ + PhysicalUUID: "uuid-1", + Serial: "SERP1", + CapacityGiB: 16, + Type: "TLC", + } + if got[0] != want { + t.Errorf("got %+v; want %+v", got[0], want) + } + }) + + t.Run("is_proxy=true QLC (iu_size=16384)", func(t *testing.T) { + const json = `{ + "devices": [ + { + "path": "/dev/nvmeQn1", + "status": "weka_formatted", + "physical_uuid": "uuid-qlc", + "weka_info": { "cluster_guid": "some-guid", "is_proxy": true }, + "hardware": { + "serial_number": "SERQLC", + "size_bytes": 17179869184, + "iu_size": 16384 + } + } + ] + }` + parsed := mustParseSignDriveList(t, json) + got := extractProxyDrives(parsed) + if len(got) != 1 { + t.Fatalf("expected 1 drive, got %d: %v", len(got), got) + } + if got[0].Type != "QLC" { + t.Errorf("got Type=%q; want %q", got[0].Type, "QLC") + } + }) + + t.Run("proxy GUID sentinel", func(t *testing.T) { + const json = `{ + "devices": [ + { + "path": "/dev/nvmePn1", + "status": "weka_formatted", + "physical_uuid": "uuid-sentinel", + "weka_info": { "cluster_guid": "026938d8-a8a2-4ad4-a316-2f23358a1e7a", "is_proxy": false }, + "hardware": { + "serial_number": "SERSENTINEL", + "size_bytes": 17179869184, + "iu_size": 4096 + } + } + ] + }` + parsed := mustParseSignDriveList(t, json) + got := extractProxyDrives(parsed) + if len(got) != 1 { + t.Fatalf("expected 1 drive for proxySignedGUID sentinel, got %d: %v", len(got), got) + } + }) + + t.Run("proxy guid string sentinel", func(t *testing.T) { + const json = `{ + "devices": [ + { + "path": "/dev/nvmePGn1", + "status": "weka_formatted", + "physical_uuid": "uuid-proxyguid", + "weka_info": { "cluster_guid": "proxy guid", "is_proxy": false }, + "hardware": { + "serial_number": "SERPG", + "size_bytes": 17179869184, + "iu_size": 4096 + } + } + ] + }` + parsed := mustParseSignDriveList(t, json) + got := extractProxyDrives(parsed) + if len(got) != 1 { + t.Fatalf("expected 1 drive for 'proxy guid' sentinel, got %d: %v", len(got), got) + } + }) + + t.Run("status not weka_formatted — skipped", func(t *testing.T) { + const json = `{ + "devices": [ + { + "path": "/dev/nvmeXn1", + "status": "excluded", + "physical_uuid": "uuid-excl", + "weka_info": { "cluster_guid": "some-guid", "is_proxy": true }, + "hardware": { "serial_number": "SEREXCL", "size_bytes": 17179869184, "iu_size": 4096 } + } + ] + }` + parsed := mustParseSignDriveList(t, json) + got := extractProxyDrives(parsed) + if len(got) != 0 { + t.Errorf("expected empty for non-weka_formatted status, got %v", got) + } + }) + + t.Run("weka_info nil — skipped", func(t *testing.T) { + const json = `{ + "devices": [ + { + "path": "/dev/nvmeXn1", + "status": "weka_formatted", + "physical_uuid": "uuid-nil", + "weka_info": null, + "hardware": { "serial_number": "SERNILWI", "size_bytes": 17179869184, "iu_size": 4096 } + } + ] + }` + parsed := mustParseSignDriveList(t, json) + got := extractProxyDrives(parsed) + if len(got) != 0 { + t.Errorf("expected empty for nil weka_info, got %v", got) + } + }) + + t.Run("empty physical_uuid — skipped", func(t *testing.T) { + const json = `{ + "devices": [ + { + "path": "/dev/nvmeXn1", + "status": "weka_formatted", + "physical_uuid": "", + "weka_info": { "cluster_guid": "some-guid", "is_proxy": true }, + "hardware": { "serial_number": "SERNOUUID", "size_bytes": 17179869184, "iu_size": 4096 } + } + ] + }` + parsed := mustParseSignDriveList(t, json) + got := extractProxyDrives(parsed) + if len(got) != 0 { + t.Errorf("expected empty for empty physical_uuid, got %v", got) + } + }) + + t.Run("zero size_bytes — skipped", func(t *testing.T) { + const json = `{ + "devices": [ + { + "path": "/dev/nvmeXn1", + "status": "weka_formatted", + "physical_uuid": "uuid-zero", + "weka_info": { "cluster_guid": "some-guid", "is_proxy": true }, + "hardware": { "serial_number": "SERZERO", "size_bytes": 0, "iu_size": 4096 } + } + ] + }` + parsed := mustParseSignDriveList(t, json) + got := extractProxyDrives(parsed) + if len(got) != 0 { + t.Errorf("expected empty for zero size_bytes, got %v", got) + } + }) +}