fix: detect compute-blocked drive growth, split per-cause warnings, fix infeasibility binding

cmd/weka-capacity/autofulldrives_test.go

@@ -996,3 +996,40 @@ func mapKeys(m map[string]any) []string {
 	sort.Strings(out)
 	return out
 }
+
+// TestAutoFullDrivesNodeRows_ComputeDeletingNote covers the NOTE for the compute-blocked deferral. The
+// contract for the column is that any row with used < avail explains the gap, and this cause reaches it on
+// both paths: a create that never happens, and a growth held at the container's current size. Without the
+// case the row shows a bare gap while WARNINGS names the node.
+func TestAutoFullDrivesNodeRows_ComputeDeletingNote(t *testing.T) {
+	nodeInv := []capacityplanner.NodeCapacity{
+		// create deferred: no container of ours yet, nothing planned.
+		{NodeName: "n-blk-create", FDValue: "n-blk-create", DriveCapacitiesGiB: []int{3840, 3840},
+			HasDeletingComputeContainer: true},
+		// growth deferred: holds 1 of its 3 drives, no Grow entry written this pass.
+		{NodeName: "n-blk-grow", FDValue: "n-blk-grow", OwnDriveCapacitiesGiB: []int{3840},
+			DriveCapacitiesGiB: []int{3840, 3840}, HasDeletingComputeContainer: true},
+	}
+	existing := []capacityplanner.ExistingContainer{
+		{Name: "c-blk-grow", Node: "n-blk-grow", TlcGiB: 3840, NumCores: 1, NumDrives: 1},
+	}
+	plan := &capacityplanner.CapacityPlan{}
+
+	byNode := map[string]autoFullDrivesNodeRow{}
+	for _, r := range autoFullDrivesNodeRows(nodeInv, existing, plan) {
+		byNode[r.Node] = r
+	}
+
+	for _, node := range []string{"n-blk-create", "n-blk-grow"} {
+		row, ok := byNode[node]
+		if !ok {
+			t.Fatalf("%s: missing row", node)
+		}
+		if row.DrivesUsed >= row.DrivesAvail {
+			t.Fatalf("%s: fixture must leave a gap to explain, got used=%d avail=%d", node, row.DrivesUsed, row.DrivesAvail)
+		}
+		if !strings.Contains(row.Note, "compute container being deleted") {
+			t.Errorf("%s: Note = %q, want it to name the deleting compute container", node, row.Note)
+		}
+	}
+}

cmd/weka-capacity/plan.go

@@ -695,6 +695,11 @@ func autoFullDrivesNodeRows(nodeInv []capacityplanner.NodeCapacity, existing []c
 				row.Note = note(capacityplanner.WarningKindTransient, "drive container being deleted")
 			case existingByNode[n.NodeName].Unscheduled:
 				row.Note = note(capacityplanner.WarningKindTransient, "pod has not been scheduled yet")
+			// Not gated on nodeStateNotPlanned: this defers a growth as well as a create, and a grown-but-
+			// deferred node still renders its existing (smaller) container rather than "not planned". Ordered
+			// after Unscheduled to match the walk, which resolves that skip first.
+			case n.HasDeletingComputeContainer:
+				row.Note = note(capacityplanner.WarningKindTransient, "compute container being deleted")
 			case row.State != nodeStateNotPlanned && hasFleetWarning(plan.Warnings, capacityplanner.WarningKindDrivesStranded):
 				row.Note = "drives held back by the numDrives pin — see WARNINGS"
 			}

internal/capacityplanner/autofulldrives.go

@@ -17,6 +17,9 @@ type autoNode struct {
 	nc NodeCapacity
 	// existing is this cluster's drive container on the node, or nil when there is none yet.
 	existing *ExistingContainer
+	// ownCompute is whether this cluster also runs a compute container here. Only the growth-hazard
+	// diagnostic needs it: its remedy names a container to delete, so it may not be offered otherwise.
+	ownCompute bool
 	// free is the node's unallocated drives; all is own+free — both descending, which is what makes a
 	// numDrives pin take the largest drives.
 	free []int
@@ -25,20 +28,29 @@ type autoNode struct {
 
 // autoFullDrivesNodes indexes the inventory into name-sorted autoNodes, so Create/Grow/Warnings ordering is
 // deterministic and a fleet always plans the same way twice.
-func autoFullDrivesNodes(existingDrives []ExistingContainer, inventory []NodeCapacity) []autoNode {
+func autoFullDrivesNodes(
+	existingDrives []ExistingContainer, existingCompute []ExistingComputeContainer, inventory []NodeCapacity,
+) []autoNode {
 	byNode := make(map[string]*ExistingContainer, len(existingDrives))
 	for i := range existingDrives {
 		if existingDrives[i].Node != "" {
 			byNode[existingDrives[i].Node] = &existingDrives[i]
 		}
 	}
+	computeByNode := make(map[string]bool, len(existingCompute))
+	for _, ec := range existingCompute {
+		if ec.Node != "" {
+			computeByNode[ec.Node] = true
+		}
+	}
 	nodes := make([]autoNode, 0, len(inventory))
 	for i := range inventory {
 		nc := inventory[i]
 		nodes = append(nodes, autoNode{
-			nc:       nc,
-			existing: byNode[nc.NodeName],
-			free:     SortDriveCapacitiesDesc(nc.DriveCapacitiesGiB),
+			nc:         nc,
+			existing:   byNode[nc.NodeName],
+			ownCompute: computeByNode[nc.NodeName],
+			free:       SortDriveCapacitiesDesc(nc.DriveCapacitiesGiB),
 			all: SortDriveCapacitiesDesc(
 				append(append([]int(nil), nc.OwnDriveCapacitiesGiB...), nc.DriveCapacitiesGiB...)),
 		})
@@ -59,7 +71,7 @@ func PlanAutoFullDrives(
 	computeNodes map[string]bool,
 	cons *CapacityConstraints,
 ) CapacityPlan {
-	nodes := autoFullDrivesNodes(existingDrives, inventory)
+	nodes := autoFullDrivesNodes(existingDrives, existingCompute, inventory)
 	plan, totals, remaining := planAutoFullDrivesDrives(desired, nodes, cons)
 
 	// Set above the feasibility gate so an infeasible plan can still report the core demand its claimed drives
@@ -109,10 +121,13 @@ func planAutoFullDrivesDrives(
 	// same condition (a cordoned node, an unscheduled pod) commonly hits several nodes in one pass.
 	var stranded []strandedNode
 	var failures []autoFitFailure
-	var ineligible []string  // "h1-2-a (cordoned)"
-	var ineligibleDrives int // free signed drives on those nodes, for formatIneligibleWarning's total
-	var deferred []string    // nodes whose existing container's pod is unscheduled
-	var deleting []string    // nodes with HasDeletingDriveContainer
+	var ineligible []string                     // "h1-2-a (cordoned)"
+	var ineligibleDrives int                    // free signed drives on those nodes, for formatIneligibleWarning's total
+	ineligibleReasons := map[string]bool{}      // distinct IneligibleReason values seen, for the NodeIneligible Cause
+	var deferred []string                       // nodes whose existing container's pod is unscheduled
+	var deleting []string                       // nodes with HasDeletingDriveContainer
+	var computeBlocked []string                 // nodes whose fit failed because HasDeletingComputeContainer holds what it needs
+	computeBlockedBindings := map[string]bool{} // distinct fit.binding values among them, for the warning's wording
 
 	// Called on both exits. The infeasibility check below returns from inside the walk, and a condition
 	// already collected is still true of the plan that return carries — the CLI's per-node NOTE column points
@@ -122,10 +137,24 @@ func planAutoFullDrivesDrives(
 			plan.Warnings = append(plan.Warnings, formatStrandedWarning(stranded, desired.NumDrives))
 		}
 		if len(ineligible) > 0 {
-			plan.Warnings = append(plan.Warnings, formatIneligibleWarning(ineligible, ineligibleDrives))
+			reasons := make([]string, 0, len(ineligibleReasons))
+			for r := range ineligibleReasons {
+				reasons = append(reasons, r)
+			}
+			sort.Strings(reasons)
+			plan.Warnings = append(plan.Warnings, formatIneligibleWarning(ineligible, ineligibleDrives, reasons))
 		}
-		if len(deferred) > 0 || len(deleting) > 0 {
-			plan.Warnings = append(plan.Warnings, formatPlacementDeferredWarning(deferred, deleting))
+		if len(deferred) > 0 || len(deleting) > 0 || len(computeBlocked) > 0 {
+			// Name the blocked dimension only when every node agrees on it, the same rule autoNodeFitInfeasible
+			// uses for Binding — one node's cause must not stand for the rest.
+			binding := ""
+			if len(computeBlockedBindings) == 1 {
+				for b := range computeBlockedBindings {
+					binding = b
+				}
+			}
+			plan.Warnings = append(plan.Warnings,
+				formatPlacementDeferredWarning(deferred, deleting, computeBlocked, binding)...)
 		}
 	}
 
@@ -155,6 +184,7 @@ func planAutoFullDrivesDrives(
 				totals.tlcGiBAvailable += sumInts(drives)
 				ineligible = append(ineligible, fmt.Sprintf("%s (%s)", name, n.nc.IneligibleReason))
 				ineligibleDrives += len(drives)
+				ineligibleReasons[n.nc.IneligibleReason] = true
 				continue
 			}
 			// The one per-node skip that is a skip rather than an infeasibility: it clears itself, so failing
@@ -186,17 +216,49 @@ func planAutoFullDrivesDrives(
 		to := autoFootprint{cores: max(cur.cores, np.cores), drives: max(cur.drives, np.numDrives())}
 		newTlcGiB := max(curTlcGiB, np.tlcGiB())
 
-		// An unscheduled container's cores charge at their frozen value here, matching that its Grow entry
-		// never gets written below — the ratcheted `to.cores` is what growth would apply, not what is
-		// actually taken while the pod sits unscheduled.
+		unscheduled := n.existing != nil && n.existing.Unscheduled
+
+		// The fit runs before the totals because what this node charges depends on whether the walk will skip
+		// it below, and only the fit can tell. Never for an unscheduled node: that one is skipped either way,
+		// and fitting it could only manufacture an infeasibility.
+		var fit autoFitResult
+		if !unscheduled {
+			fit = autoNodeFit(&n.nc, cur, to, cons)
+		}
+		// !unscheduled because that node never ran a fit: its zero-valued result reads as a failure, which
+		// would pull it into the compute-blocked charging convention instead of the unscheduled one.
+		blockedByDeletingCompute := !unscheduled && !fit.ok && n.nc.HasDeletingComputeContainer
+
+		// A node that either skip below leaves alone charges the cores it is actually running: its Grow entry
+		// never gets written, so the ratcheted `to.cores` is what growth would apply, not what is taken.
+		// Charging the target sizes compute against cores that do not exist, which can flip the plan
+		// infeasible — and an infeasible plan applies nothing, including the growth the other nodes earned.
 		chargedCores := to.cores
-		if n.existing != nil && n.existing.Unscheduled {
+		if unscheduled || blockedByDeletingCompute {
 			chargedCores = cur.cores
 		}
 
-		totals.drivesTaken += to.drives
+		// Drives and TLC freeze for a compute-blocked node only; an unscheduled one keeps charging its planned
+		// figure (TestPlanAutoFullDrives_UnscheduledDriveContainer_ComputeCountsPlannedNotFrozenCapacity pins
+		// that). The difference is causal, not cosmetic: tlcGiBTaken sizes compute hugepages, so charging
+		// capacity this pass will not create raises compute demand on the very node whose growth compute is
+		// already blocking. An unscheduled pod is only waiting on the scheduler, and pre-sizing compute for the
+		// drives it will bring costs nothing.
+		chargedDrives, chargedTlcGiB := to.drives, newTlcGiB
+		if blockedByDeletingCompute {
+			// cur.drives is the count the container holds, but ExistingContainer.TlcGiB is structurally 0 on an
+			// auto-full-drives container — the mode is defined by driveCapacity and containerCapacity both being
+			// unset, which is all DriveContainerCapacities reads. Its capacity comes from the node's own-drive
+			// split instead, the same fallback the CLI's NODES table uses, and is 0 on the create path.
+			chargedDrives, chargedTlcGiB = cur.drives, 0
+			if n.existing != nil {
+				chargedTlcGiB = sumInts(n.nc.OwnDriveCapacitiesGiB)
+			}
+		}
+
+		totals.drivesTaken += chargedDrives
 		totals.drivesAvailable += len(drives)
-		totals.tlcGiBTaken += newTlcGiB
+		totals.tlcGiBTaken += chargedTlcGiB
 		totals.tlcGiBAvailable += sumInts(drives)
 		totals.driveCoresTaken += chargedCores
 
@@ -208,21 +270,30 @@ func planAutoFullDrivesDrives(
 		}
 
 		// An unscheduled pod holds no node resources to grow into, and raising its spec would only make it
-		// harder to schedule. Skipped after the totals so the fleet accounting still reflects its drives, and
-		// before the fit so it cannot manufacture an infeasibility.
-		if n.existing != nil && n.existing.Unscheduled {
+		// harder to schedule. Skipped after the totals so the fleet accounting still reflects its drives.
+		if unscheduled {
 			deferred = append(deferred, name)
 			continue
 		}
 
-		fit := autoNodeFit(&n.nc, cur, to, cons)
 		if !fit.ok {
-			kind := "create"
+			// A deleting compute container on this node still holds the hugepages the fit needs, but that
+			// clears itself once the deletion lands — and failing the plan here is exactly what would stop
+			// compute from ever being re-planned, which is the capacity weka needs before it will let the
+			// deactivation through (see PlanAutoFullDrives's infeasibility gate). Deferred, not infeasible,
+			// same as the deleting-drive-container skip above.
+			if blockedByDeletingCompute {
+				computeBlocked = append(computeBlocked, name)
+				computeBlockedBindings[fit.binding] = true
+				continue
+			}
+			kind := fitKindCreate
 			if n.existing != nil {
-				kind = "growth"
+				kind = fitKindGrowth
 			}
 			failures = append(failures, autoFitFailure{
 				node: name, kind: kind, numDrives: to.drives, toCores: to.cores, fit: fit,
+				ownCompute: n.ownCompute,
 			})
 			continue
 		}

internal/capacityplanner/autofulldrives_compute.go

@@ -139,24 +139,25 @@ func planComputeAutoFullDrives(in *autoComputeInput, plan *CapacityPlan) {
 	// Prefer new containers, top up with in-place growth: the least growth that lets the rest fit on free
 	// nodes. probe reports whether a given growTake is coverable, deriving the layout for the target it
 	// leaves; it has no side effects, so it is safe to call for candidates that are never committed.
-	probe := func(growTake int) (computeProbe, string) {
+	probe := func(growTake int) (computeProbe, string, string) {
 		// Growth alone closes it, so there is nothing to derive — and deriving against a zero target would
 		// resurrect the phantom-container case guarded above. growTake==0 is excluded: a zero deficit there
 		// still owes the derivation its floor containers.
 		if deficit-growTake <= 0 && growTake > 0 {
-			return computeProbe{growthAlone: true}, ""
+			return computeProbe{growthAlone: true}, "", ""
 		}
 		// specCount is hard 0: a pinned computeContainers means the cluster is not in this mode at all.
-		count, cores, infeasible, warnings := deriveComputeLayout(
+		count, cores, infeasible, binding, warnings := deriveComputeLayout(
 			0, in.desired.ComputeCores, deficit-growTake,
 			floor, in.cons.MaxCoresPerContainer, coreHeadroom, nodeHugepagesMiB, hugepagesFor,
 		)
-		return computeProbe{count: count, cores: cores, warnings: warnings}, infeasible
+		return computeProbe{count: count, cores: cores, warnings: warnings}, infeasible, binding
 	}
 
 	// unaidedReason is the growTake==0 attempt's own explanation (usually the binding resource, often compute
-	// hugepages) for why new containers alone do not fit, and leads the infeasibility below.
-	best, unaidedReason := probe(0)
+	// hugepages) for why new containers alone do not fit, and leads the infeasibility below; unaidedBinding is
+	// its structured classification, carried through so the caller never re-parses the English reason.
+	best, unaidedReason, unaidedBinding := probe(0)
 	growTake := 0
 	if unaidedReason != "" {
 		// The search space is [1, hi]: hi is where growth alone would close the deficit, or all the growth
@@ -172,7 +173,7 @@ func planComputeAutoFullDrives(in *autoComputeInput, plan *CapacityPlan) {
 			// len(kept)*MaxCoresPerContainer and every probe re-derives the whole layout.
 			for lo := 1; lo <= hi; {
 				mid := lo + (hi-lo)/2
-				if p, infeasible := probe(mid); infeasible == "" {
+				if p, infeasible, _ := probe(mid); infeasible == "" {
 					best, growTake, found = p, mid, true
 					hi = mid - 1 // a smaller growTake may also cover it — keep the least
 				} else {
@@ -185,7 +186,7 @@ func planComputeAutoFullDrives(in *autoComputeInput, plan *CapacityPlan) {
 			// container count — rises as it shrinks. Coverability can therefore hold on an interval and fail
 			// above it, which a binary search would walk away from. Scan.
 			for g := 1; g <= hi; g++ {
-				if p, infeasible := probe(g); infeasible == "" {
+				if p, infeasible, _ := probe(g); infeasible == "" {
 					best, growTake, found = p, g, true
 					break
 				}
@@ -199,10 +200,14 @@ func planComputeAutoFullDrives(in *autoComputeInput, plan *CapacityPlan) {
 					"; growing the %d existing compute container(s) in place offers only %d more core(s), "+
 						"which does not close the %d-core shortfall", len(kept), growTotal, deficit)
 			}
+			reason += drainingComputeClause(in)
 			setInfeasible(plan, &InfeasibilityReport{
-				Reason: reason,
-				Pool:   "compute",
-				Fixes:  fixesAutoFullDrivesCompute(in.cons),
+				Reason:  reason,
+				Pool:    "compute",
+				Binding: unaidedBinding,
+				// ShortfallGiB stays 0: the deficit here is in cores or MiB-hugepages, never GiB, and
+				// converting either into GiB would invent a number this report never measured.
+				Fixes: fixesAutoFullDrivesCompute(in.cons),
 			})
 			return
 		}
@@ -283,17 +288,43 @@ func autoKeptCompute(in *autoComputeInput) (kept []autoComputeEntry, pinned map[
 	return kept, pinned, keptCores
 }
 
-// autoComputeGrowHeadroom is how many extra data cores ec's node can absorb in place, bounded by
-// MaxCoresPerContainer and the node's remaining CPU/hugepages/memory. All bounds are deltas: the container's
-// current footprint is already charged against remaining. The hugepages bound has no closed form, so the
-// candidate size walks down until the delta fits.
+// drainingComputeClause names nodes whose compute headroom is depressed by a compute container of this
+// cluster that is pending deletion, for appending to a shortfall report. Wording only: the planner cannot
+// tell whether that reservation returning would actually close the gap, so the shortfall stays infeasible —
+// but which of "wait" or "intervene" is right turns on this fact, and nothing else in the report carries it.
+func drainingComputeClause(in *autoComputeInput) string {
+	var nodes []string
+	for node, nc := range in.remaining {
+		if nc.HasDeletingComputeContainer {
+			nodes = append(nodes, node)
+		}
+	}
+	if len(nodes) == 0 {
+		return ""
+	}
+	sort.Strings(nodes)
+	return fmt.Sprintf(
+		"; a compute container of this cluster is still being deleted on %s, so this shortfall may clear on "+
+			"its own once that deletion lands", listNodes(nodes))
+}
+
+// autoComputeGrowHeadroom is how many extra data cores ec's node can absorb in place, bounded by a pinned
+// computeCores, MaxCoresPerContainer and the node's remaining CPU/hugepages/memory. All bounds are deltas:
+// the container's current footprint is already charged against remaining. The hugepages bound has no closed
+// form, so the candidate size walks down until the delta fits.
 func autoComputeGrowHeadroom(ec *ExistingComputeContainer, nc *NodeCapacity, keptCount int, in *autoComputeInput) int {
 	// includeBase=false throughout: the management core and memory base are already reserved by the running
 	// container, so only the per-core increments are charged.
 	maxCores := ec.NumCores + physicalCPUToDataCores(nc, 0, in.cons, false)
 	if in.cons.MaxCoresPerContainer > 0 {
 		maxCores = min(maxCores, in.cons.MaxCoresPerContainer)
 	}
+	// A pinned computeCores is every compute container's exact size, which deriveComputeLayout honors for new
+	// containers; growth stops there too, so a shortfall new containers cannot place is reported rather than
+	// absorbed by an oversized survivor.
+	if in.desired.ComputeCores > 0 {
+		maxCores = min(maxCores, in.desired.ComputeCores)
+	}
 	if in.cons.MemoryPerCoreMiB > 0 {
 		maxCores = min(maxCores, ec.NumCores+nc.AvailableMemoryMiB/in.cons.MemoryPerCoreMiB)
 	}
@@ -442,8 +473,8 @@ func autoPlaceNewCompute(
 		setInfeasible(plan, &InfeasibilityReport{
 			Reason: fmt.Sprintf(
 				"compute: cannot place %d new compute container(s) to cover the %d-core shortfall — "+
-					"only %d free fitting compute node(s) (each holds up to %d cores + %d MiB hugepages)",
-				count, shortfall, len(candidates), cores, perContainerHP),
+					"only %d free fitting compute node(s) (each holds up to %d cores + %d MiB hugepages)%s",
+				count, shortfall, len(candidates), cores, perContainerHP, drainingComputeClause(in)),
 			Pool:    "compute",
 			Binding: "cores",
 			Fixes:   fixesAutoFullDrivesCompute(in.cons),