From 52fcfacf5ef551a2445b30ebacf97876079b39d5 Mon Sep 17 00:00:00 2001 From: Syed <40798652+thelostorbital@users.noreply.github.com> Date: Wed, 9 Sep 2026 08:16:10 +0530 Subject: [PATCH] feat(cleanup): select and record nightly test-wipe deletions Add internal/isolation/cleanup: an I/O-free selector, planner, canonical record, and executor for WF-TEST-01 step T7. Selection admits only the closed three-kind capability set with D-157 ownership proof, refuses any ambiguous, unsupported, permanent, or cross-project target by failing the whole inventory, orders instances before disks before run firewall rules with retained-disk semantics, and seals every deletion so an adapter cannot be called with a plain name. The first run after bootstrap and any run whose harness state is not open and usable plan and record but delete nothing. Every deletion is recorded before it is issued and re-runs converge from observed state. Claude-Session: https://claude.ai/code/session_01MQYGGYsYAzMU3MiM2SKhEZ --- internal/isolation/cleanup/execute.go | 154 +++++++ internal/isolation/cleanup/helpers_test.go | 254 ++++++++++++ internal/isolation/cleanup/inventory.go | 201 +++++++++ internal/isolation/cleanup/plan.go | 149 +++++++ internal/isolation/cleanup/plan_test.go | 244 +++++++++++ internal/isolation/cleanup/record.go | 208 ++++++++++ internal/isolation/cleanup/selection.go | 406 +++++++++++++++++++ internal/isolation/cleanup/selection_test.go | 246 +++++++++++ 8 files changed, 1862 insertions(+) create mode 100644 internal/isolation/cleanup/execute.go create mode 100644 internal/isolation/cleanup/helpers_test.go create mode 100644 internal/isolation/cleanup/inventory.go create mode 100644 internal/isolation/cleanup/plan.go create mode 100644 internal/isolation/cleanup/plan_test.go create mode 100644 internal/isolation/cleanup/record.go create mode 100644 internal/isolation/cleanup/selection.go create mode 100644 internal/isolation/cleanup/selection_test.go diff --git a/internal/isolation/cleanup/execute.go b/internal/isolation/cleanup/execute.go new file mode 100644 index 0000000..de2e795 --- /dev/null +++ b/internal/isolation/cleanup/execute.go @@ -0,0 +1,154 @@ +// Copyright 2026 CtrlBoard.dev +// SPDX-License-Identifier: Apache-2.0 + +package cleanup + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/thelostorbital/ctrldb/internal/isolation" + "github.com/thelostorbital/ctrldb/internal/redact" +) + +var ErrWipeExecution = errors.New("test wipe execution failed") + +// Journal is the minimal durable-record surface the wipe needs. Every method +// is create-only; the M1-05 control store adapter implements it without +// exposing deletion. Until that adapter lands, tests use an in-memory value. +type Journal interface { + RecordPlan(ctx context.Context, record WipeRecordV1) error + RecordDeletion(ctx context.Context, record DeletionRecordV1) error +} + +// Deleter executes one sealed deletion. The provider adapter must refuse an +// unsealed deletion and must not expose any other delete surface. +type Deleter interface { + Delete(ctx context.Context, deletion Deletion) error +} + +// DeletionOutcome is the typed, redacted result of one attempted deletion. +type DeletionOutcome struct { + Sequence int + Capability isolation.CleanupCapability + Identity isolation.ResourceIdentity + Attempt uint32 + Status DeletionStatus + Failure redact.Text +} + +// WipeResult is the typed, redacted outcome of one execution. Counts and +// identities are safe to log; failures are sanitized at construction. +type WipeResult struct { + Mode string + ProjectID string + RunNumber uint64 + Disposition Disposition + InventoryRevision string + RecordIntegrity string + Planned int + Deleted int + Failed int + Retained int + Deferred int + Protected int + StartedAt time.Time + EndedAt time.Time + Deletions []DeletionOutcome +} + +// Execute records the plan, then issues each sealed deletion in order, +// recording it before and after the provider call. It stops at the first +// failure so the next run converges from freshly observed state. +func Execute(ctx context.Context, plan WipePlan, journal Journal, deleter Deleter, clock func() time.Time) (WipeResult, error) { + if ctx == nil || journal == nil || deleter == nil || clock == nil { + return WipeResult{}, inputError("execution", "requires context, journal, deleter, and clock") + } + record, err := plan.Record() + if err != nil { + return WipeResult{}, err + } + startedAt := clock().UTC() + if err := validateNow(startedAt); err != nil { + return WipeResult{}, err + } + result := WipeResult{ + Mode: plan.Mode, ProjectID: plan.ProjectID, RunNumber: plan.RunNumber, Disposition: plan.Disposition, + InventoryRevision: plan.Selection.InventoryRevision, RecordIntegrity: record.IntegritySHA256, + Planned: len(plan.Selection.Deletions), Retained: len(plan.Selection.Retained), + Deferred: len(plan.Selection.Deferred), Protected: len(plan.Selection.Protected), StartedAt: startedAt, + } + if err := journal.RecordPlan(ctx, record); err != nil { + result.EndedAt = clock().UTC() + return result, fmt.Errorf("%w: plan record was not persisted", ErrWipeExecution) + } + if plan.Disposition != DispositionDelete { + result.EndedAt = clock().UTC() + return result, nil + } + for _, deletion := range plan.Selection.Deletions { + outcome, err := executeDeletion(ctx, record.IntegritySHA256, deletion, journal, deleter, clock) + result.Deletions = append(result.Deletions, outcome) + if err != nil { + result.Failed++ + result.EndedAt = clock().UTC() + return result, err + } + result.Deleted++ + } + result.EndedAt = clock().UTC() + return result, nil +} + +func executeDeletion(ctx context.Context, planIntegrity string, deletion Deletion, journal Journal, deleter Deleter, clock func() time.Time) (DeletionOutcome, error) { + outcome := DeletionOutcome{ + Sequence: deletion.Sequence, Capability: deletion.Capability, Identity: deletion.Identity, + Attempt: deletion.Attempt, Status: DeletionFailed, Failure: redact.Sanitize(""), + } + if !deletion.Sealed() { + outcome.Failure = redact.Sanitize("deletion is not sealed") + return outcome, fmt.Errorf("%w: unsealed deletion", ErrWipeExecution) + } + if ctx.Err() != nil { + outcome.Failure = redact.Sanitize("execution canceled before issue") + return outcome, fmt.Errorf("%w: canceled", ErrWipeExecution) + } + issuedAt := clock().UTC() + issued, err := sealDeletionRecord(DeletionRecordV1{ + PlanIntegrity: planIntegrity, Deletion: plannedDeletion(deletion), Status: DeletionIssued, IssuedAt: issuedAt, Failure: redact.Sanitize(""), + }) + if err != nil { + return outcome, err + } + if err := journal.RecordDeletion(ctx, issued); err != nil { + outcome.Failure = redact.Sanitize("deletion intent was not persisted") + return outcome, fmt.Errorf("%w: deletion intent was not persisted", ErrWipeExecution) + } + deleteErr := deleter.Delete(ctx, deletion) + completedAt := clock().UTC() + final := issued + final.CompletedAt = &completedAt + if deleteErr != nil { + final.Status = DeletionFailed + final.Failure = redact.Sanitize(deleteErr.Error()) + outcome.Failure = final.Failure + } else { + final.Status = DeletionDeleted + outcome.Status = DeletionDeleted + } + final, err = sealDeletionRecord(final) + if err != nil { + return outcome, err + } + if err := journal.RecordDeletion(ctx, final); err != nil { + outcome.Status = DeletionFailed + outcome.Failure = redact.Sanitize("deletion outcome was not persisted") + return outcome, fmt.Errorf("%w: deletion outcome was not persisted", ErrWipeExecution) + } + if deleteErr != nil { + return outcome, fmt.Errorf("%w: deletion %d failed", ErrWipeExecution, deletion.Sequence) + } + return outcome, nil +} diff --git a/internal/isolation/cleanup/helpers_test.go b/internal/isolation/cleanup/helpers_test.go new file mode 100644 index 0000000..7d5e99b --- /dev/null +++ b/internal/isolation/cleanup/helpers_test.go @@ -0,0 +1,254 @@ +// Copyright 2026 CtrlBoard.dev +// SPDX-License-Identifier: Apache-2.0 + +package cleanup_test + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + "github.com/thelostorbital/ctrldb/internal/config" + "github.com/thelostorbital/ctrldb/internal/isolation" + "github.com/thelostorbital/ctrldb/internal/isolation/cleanup" +) + +const ( + testProject = "example-test-project" + testZone = "us-central1-a" + testRunID = "run-0a1b" + otherRunID = "run-9z8y" + testLifetime = 6 * time.Hour +) + +var ( + testNow = time.Date(2026, 9, 9, 20, 0, 0, 0, time.UTC) + testRevision = strings.Repeat("d", 64) +) + +func identity(t *testing.T, kind isolation.ResourceKind, scope isolation.ResourceScope, location, name string) isolation.ResourceIdentity { + t.Helper() + value := isolation.ResourceIdentity{ + Project: testProject, Service: isolation.ComputeServiceName, Kind: kind, Scope: scope, Location: location, Name: name, + } + key, err := isolation.CanonicalTargetKey(value) + if err != nil { + t.Fatalf("CanonicalTargetKey(%q) error = %v", name, err) + } + value.CanonicalKey = key + return value +} + +func instanceIdentity(t *testing.T, name string) isolation.ResourceIdentity { + t.Helper() + return identity(t, isolation.ComputeInstanceKind, isolation.ResourceScopeZone, testZone, name) +} + +func diskIdentity(t *testing.T, name string) isolation.ResourceIdentity { + t.Helper() + return identity(t, isolation.ComputeDiskKind, isolation.ResourceScopeZone, testZone, name) +} + +func firewallIdentity(t *testing.T, name string) isolation.ResourceIdentity { + t.Helper() + return identity(t, isolation.ComputeFirewallKind, isolation.ResourceScopeGlobal, "global", name) +} + +func runLabels(runID string) map[string]string { + return map[string]string{ + config.LabelManagedBy: config.LabelManagedByValue, config.LabelEnvironment: config.TestEnvironmentLabel, + config.LabelPurpose: config.TestResourcePurposeLabel, isolation.LabelRunID: runID, + } +} + +func testPolicy() cleanup.WipePolicy { + return cleanup.WipePolicy{ProjectID: testProject, MaxLifetime: testLifetime} +} + +func expiredAt() time.Time { return testNow.Add(-testLifetime - time.Hour) } +func freshAt() time.Time { return testNow.Add(-time.Hour) } + +func lifetimeRecord(t *testing.T, runID string, createdAt, expiresAt time.Time) cleanup.LifetimeRecord { + t.Helper() + contract := isolation.RunLifetimeContract{ + ProjectID: testProject, RunID: runID, + Plan: isolation.PlanIdentity{ID: "plan-0123456789abcdef", Hash: strings.Repeat("b", 64)}, + OperationID: "op-0123456789abcdef", RecordID: "lifetime-0123456789abcdef", RecordGeneration: 3, + CreatedAt: createdAt, ExpiresAt: expiresAt, RevocationWorkflowID: isolation.TestHarnessRevocationWorkflowID, + } + fingerprint, err := isolation.RunLifetimeContractFingerprint(contract) + if err != nil { + t.Fatalf("RunLifetimeContractFingerprint() error = %v", err) + } + return cleanup.LifetimeRecord{Contract: contract, Expected: isolation.OwnershipRecordExpectation{ + RecordID: contract.RecordID, RecordGeneration: contract.RecordGeneration, Revision: fingerprint, + ObservedAt: testNow.Add(-time.Minute), ValidUntil: testNow.Add(time.Minute), + }} +} + +func firewallDescription(t *testing.T, record cleanup.LifetimeRecord) string { + t.Helper() + description, err := isolation.RunFirewallDescription(record.Expected.Revision) + if err != nil { + t.Fatalf("RunFirewallDescription() error = %v", err) + } + return description +} + +func runFirewallName(t *testing.T, runID string, purpose isolation.FirewallPurpose) string { + t.Helper() + name, err := isolation.RunFirewallRuleName(runID, purpose) + if err != nil { + t.Fatalf("RunFirewallRuleName() error = %v", err) + } + return name +} + +// fullInventory contains one expired run (instance, attached data disk, two +// firewall rules), one fresh run, one foreign instance, and both permanent +// harness firewall rules. +func fullInventory(t *testing.T) (cleanup.Inventory, []cleanup.LifetimeRecord) { + t.Helper() + expiredRecord := lifetimeRecord(t, testRunID, expiredAt(), expiredAt().Add(testLifetime)) + freshRecord := lifetimeRecord(t, otherRunID, freshAt(), freshAt().Add(testLifetime)) + inventory := cleanup.Inventory{ + ProjectID: testProject, Revision: testRevision, ObservedAt: testNow.Add(-time.Minute), ValidUntil: testNow.Add(time.Minute), Exhaustive: true, + Instances: []cleanup.InstanceObservation{ + {Identity: instanceIdentity(t, "ctrldb-test-run-0a1b-node"), Labels: runLabels(testRunID), CreatedAt: expiredAt(), AttachedDisks: []string{"ctrldb-test-run-0a1b-node", "ctrldb-test-run-0a1b-data"}}, + {Identity: instanceIdentity(t, "ctrldb-test-run-9z8y-node"), Labels: runLabels(otherRunID), CreatedAt: freshAt()}, + {Identity: instanceIdentity(t, "billing-web-1"), Labels: map[string]string{"team": "web"}, CreatedAt: expiredAt()}, + }, + Disks: []cleanup.DiskObservation{ + {Identity: diskIdentity(t, "ctrldb-test-run-0a1b-node"), Labels: runLabels(testRunID), CreatedAt: expiredAt(), AttachedInstances: []string{"ctrldb-test-run-0a1b-node"}}, + {Identity: diskIdentity(t, "ctrldb-test-run-0a1b-data"), Labels: runLabels(testRunID), CreatedAt: expiredAt(), AttachedInstances: []string{"ctrldb-test-run-0a1b-node"}}, + {Identity: diskIdentity(t, "ctrldb-test-run-9z8y-node"), Labels: runLabels(otherRunID), CreatedAt: freshAt(), AttachedInstances: []string{"ctrldb-test-run-9z8y-node"}}, + }, + Firewalls: []cleanup.FirewallObservation{ + {Identity: firewallIdentity(t, isolation.TestIAPSSHFirewallName), Description: "ctrldb:permanent", CreatedAt: expiredAt()}, + {Identity: firewallIdentity(t, isolation.TestInternalFirewallName), Description: "ctrldb:permanent", CreatedAt: expiredAt()}, + {Identity: firewallIdentity(t, runFirewallName(t, testRunID, isolation.FirewallPurposeInternalMongo)), Description: firewallDescription(t, expiredRecord), CreatedAt: expiredAt()}, + {Identity: firewallIdentity(t, runFirewallName(t, testRunID, isolation.FirewallPurposeIAPSSH)), Description: firewallDescription(t, expiredRecord), CreatedAt: expiredAt()}, + {Identity: firewallIdentity(t, runFirewallName(t, otherRunID, isolation.FirewallPurposeIAPSSH)), Description: firewallDescription(t, freshRecord), CreatedAt: freshAt()}, + {Identity: firewallIdentity(t, "default-allow-internal"), Description: "", CreatedAt: expiredAt()}, + }, + } + return inventory, []cleanup.LifetimeRecord{expiredRecord, freshRecord} +} + +func harnessSeed() isolation.HarnessStateSeed { + approvedAt := testNow.Add(-24 * time.Hour) + return isolation.HarnessStateSeed{ + ProjectID: testProject, Environment: "disposable-test", EnvironmentClass: "disposable", + ManifestHash: strings.Repeat("a", 64), + ApprovedPlan: isolation.PlanIdentity{ID: "plan-0123456789abcdef", Hash: strings.Repeat("b", 64)}, + OperationID: "op-0123456789abcdef", BootstrapEnvelopeHash: strings.Repeat("c", 64), + ControlRecordGeneration: 1, + Resources: isolation.HarnessResourceFingerprints{ + Network: strings.Repeat("1", 64), RoleBindings: strings.Repeat("2", 64), + ServiceAccounts: strings.Repeat("3", 64), WipeJob: strings.Repeat("4", 64), + Scheduler: strings.Repeat("5", 64), Image: strings.Repeat("6", 64), + }, + CleanupCapabilities: isolation.InitialCleanupCapabilities(), + BootstrapSteps: []string{"k1-audit-bootstrap", "t7-nightly-wipe", "t8-isolation-gate"}, + RollbackSteps: []string{"t7-nightly-wipe"}, + ApprovedAt: approvedAt, ApprovalValidUntil: approvedAt.Add(48 * time.Hour), + } +} + +func pendingState(t *testing.T) isolation.HarnessStateV1 { + t.Helper() + state, err := isolation.NewPendingHarnessStateV1(harnessSeed()) + if err != nil { + t.Fatalf("NewPendingHarnessStateV1() error = %v", err) + } + return state +} + +func openState(t *testing.T) isolation.HarnessStateV1 { + t.Helper() + openedAt := testNow.Add(-12 * time.Hour) + state, err := pendingState(t).OpenAfterT8(isolation.T8Evidence{ + Revision: strings.Repeat("e", 64), ObservedAt: openedAt, ValidUntil: openedAt.Add(30 * time.Minute), + }, openedAt) + if err != nil { + t.Fatalf("OpenAfterT8() error = %v", err) + } + return state +} + +func driftedState(t *testing.T) isolation.HarnessStateV1 { + t.Helper() + state, err := openState(t).MarkTestsUnusable(isolation.HarnessDriftEvidence{Revision: strings.Repeat("f", 64), DetectedAt: testNow.Add(-6 * time.Hour)}) + if err != nil { + t.Fatalf("MarkTestsUnusable() error = %v", err) + } + return state +} + +func planInput(t *testing.T, state isolation.HarnessStateV1, history cleanup.History) cleanup.PlanInput { + t.Helper() + inventory, records := fullInventory(t) + return cleanup.PlanInput{Policy: testPolicy(), State: state, Inventory: inventory, LifetimeRecords: records, History: history, Now: testNow} +} + +type memoryJournal struct { + mu sync.Mutex + plans []cleanup.WipeRecordV1 + deletions []cleanup.DeletionRecordV1 + failPlan bool +} + +func (journal *memoryJournal) RecordPlan(_ context.Context, record cleanup.WipeRecordV1) error { + journal.mu.Lock() + defer journal.mu.Unlock() + if journal.failPlan { + return context.DeadlineExceeded + } + journal.plans = append(journal.plans, record) + return nil +} + +func (journal *memoryJournal) RecordDeletion(_ context.Context, record cleanup.DeletionRecordV1) error { + journal.mu.Lock() + defer journal.mu.Unlock() + journal.deletions = append(journal.deletions, record) + return nil +} + +type scriptedDeleter struct { + t *testing.T + failAt int + calls []cleanup.Deletion +} + +func (deleter *scriptedDeleter) Delete(_ context.Context, deletion cleanup.Deletion) error { + deleter.t.Helper() + if !deletion.Sealed() { + deleter.t.Fatal("deleter received an unsealed deletion") + } + deleter.calls = append(deleter.calls, deletion) + if deleter.failAt != 0 && deletion.Sequence == deleter.failAt { + return context.DeadlineExceeded + } + return nil +} + +type forbiddenDeleter struct{ t *testing.T } + +func (deleter forbiddenDeleter) Delete(context.Context, cleanup.Deletion) error { + deleter.t.Helper() + deleter.t.Fatal("deleter must not be called") + return nil +} + +func fixedClock() time.Time { return testNow.Add(time.Second) } + +func deletionNames(deletions []cleanup.Deletion) []string { + names := make([]string, len(deletions)) + for index, deletion := range deletions { + names[index] = string(deletion.Capability) + ":" + deletion.Identity.Name + } + return names +} diff --git a/internal/isolation/cleanup/inventory.go b/internal/isolation/cleanup/inventory.go new file mode 100644 index 0000000..1fc4423 --- /dev/null +++ b/internal/isolation/cleanup/inventory.go @@ -0,0 +1,201 @@ +// Copyright 2026 CtrlBoard.dev +// SPDX-License-Identifier: Apache-2.0 + +// Package cleanup selects and orders WF-TEST-01 nightly-wipe deletions from an +// exhaustive typed inventory. It performs no process execution, provider I/O, +// or durable I/O: a provider adapter supplies the inventory, a control-store +// adapter supplies durable records, and the package returns sealed deletions +// which only that same adapter family may execute. +package cleanup + +import ( + "errors" + "fmt" + "regexp" + "sort" + "time" + + "github.com/thelostorbital/ctrldb/internal/isolation" +) + +const ( + // ModeTestWipe is the only cleanup mode implemented by M1-08. + ModeTestWipe = "test-wipe" + // MaxInventoryLifetime bounds how old an inventory may be at the mutation + // boundary. It equals the isolation package's pre-mutation proof lifetime + // because every firewall proof is re-validated against the same clock. + MaxInventoryLifetime = isolation.MaxPreMutationProofLifetime + // MaxWipeLifetime bounds the configured age threshold so a corrupted policy + // cannot postpone cleanup indefinitely or make every resource expired. + MaxWipeLifetime = 24 * time.Hour + // MinWipeLifetime rejects a threshold so small that a resource created by + // an in-flight approved operation would be selected immediately. + MinWipeLifetime = 15 * time.Minute +) + +var ( + ErrInvalidWipeInput = errors.New("invalid test wipe input") + ErrInventoryRefused = errors.New("test wipe inventory refused") +) + +var ( + projectIDPattern = regexp.MustCompile(`^[a-z][a-z0-9-]{4,28}[a-z0-9]$`) + sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`) +) + +// WipePolicy is the trusted boundary supplied from fixed job configuration. +// It carries no provider defaults; a missing value fails closed. +type WipePolicy struct { + ProjectID string + MaxLifetime time.Duration +} + +// InstanceObservation is one complete Compute instance observation. +type InstanceObservation struct { + Identity isolation.ResourceIdentity + Labels map[string]string + CreatedAt time.Time + AttachedDisks []string +} + +// DiskObservation is one complete Compute disk observation. AttachedInstances +// lists the exact instance names currently using the disk. +type DiskObservation struct { + Identity isolation.ResourceIdentity + Labels map[string]string + CreatedAt time.Time + AttachedInstances []string +} + +// FirewallObservation is one complete classic Compute firewall observation. +// Classic firewalls have no ordinary labels; ownership comes from the exact +// description and the durable lifetime record. +type FirewallObservation struct { + Identity isolation.ResourceIdentity + Description string + CreatedAt time.Time +} + +// UnsupportedObservation is any test-namespace resource of a kind outside the +// closed cleanup capability set. Its presence refuses the whole inventory. +type UnsupportedObservation struct { + Identity isolation.ResourceIdentity +} + +// Inventory is the exhaustive observation of the configured project at one +// instant. A non-exhaustive or stale inventory selects nothing. +type Inventory struct { + ProjectID string + Revision string + ObservedAt time.Time + ValidUntil time.Time + Exhaustive bool + Instances []InstanceObservation + Disks []DiskObservation + Firewalls []FirewallObservation + Unsupported []UnsupportedObservation +} + +// LifetimeRecord pairs one durable run lifetime record with the fresh +// generation observation of the object which holds it. +type LifetimeRecord struct { + Contract isolation.RunLifetimeContract + Expected isolation.OwnershipRecordExpectation +} + +func validatePolicy(policy WipePolicy) error { + if !projectIDPattern.MatchString(policy.ProjectID) { + return inputError("policy.projectID", "must be an explicit canonical project ID") + } + if policy.MaxLifetime < MinWipeLifetime || policy.MaxLifetime > MaxWipeLifetime { + return inputError("policy.maxLifetime", "must be between 15 minutes and 24 hours") + } + return nil +} + +func validateNow(now time.Time) error { + if now.IsZero() { + return inputError("now", "must be an explicit mutation-boundary time") + } + if _, offset := now.Zone(); offset != 0 { + return inputError("now", "must use UTC") + } + return nil +} + +func validateInventoryWindow(policy WipePolicy, inventory Inventory, now time.Time) error { + if inventory.ProjectID != policy.ProjectID { + return refusal("inventory.projectID", "does not name the configured project") + } + if !inventory.Exhaustive { + return refusal("inventory", "is not exhaustive") + } + if !sha256Pattern.MatchString(inventory.Revision) { + return refusal("inventory.revision", "must be a SHA-256 content revision") + } + if err := validateUTCTimestamp(inventory.ObservedAt); err != nil || validateUTCTimestamp(inventory.ValidUntil) != nil { + return refusal("inventory.window", "must be a complete UTC window") + } + if !inventory.ValidUntil.After(inventory.ObservedAt) || inventory.ValidUntil.Sub(inventory.ObservedAt) > MaxInventoryLifetime { + return refusal("inventory.window", "must be a bounded observation window") + } + if now.Before(inventory.ObservedAt) || !now.Before(inventory.ValidUntil) { + return refusal("inventory.window", "is not fresh at the mutation boundary") + } + if len(inventory.Unsupported) != 0 { + return refusal("inventory.unsupported", "contains a test-namespace resource of an unsupported kind") + } + return nil +} + +func validateUTCTimestamp(value time.Time) error { + if value.IsZero() { + return ErrInvalidWipeInput + } + if _, offset := value.Zone(); offset != 0 { + return ErrInvalidWipeInput + } + return nil +} + +// validateObservedIdentity requires an explicit, canonical, configured-project +// identity of the expected provider kind. Every entry in the inventory must +// pass, including foreign resources which are later ignored. +func validateObservedIdentity(policy WipePolicy, path string, identity isolation.ResourceIdentity, kind isolation.ResourceKind) error { + wantKey, err := isolation.CanonicalTargetKey(identity) + if err != nil || identity.CanonicalKey != wantKey { + return refusal(path, "does not carry a complete canonical identity") + } + if identity.Project != policy.ProjectID { + return refusal(path, "names another project") + } + if identity.Service != isolation.ComputeServiceName || identity.Kind != kind { + return refusal(path, "is not the expected Compute resource kind") + } + return nil +} + +func validateObservedTimestamp(path string, createdAt, now time.Time) error { + if validateUTCTimestamp(createdAt) != nil || createdAt.After(now) { + return refusal(path+".createdAt", "must be a non-future UTC creation time") + } + return nil +} + +func sortedNames(values []string) []string { + result := append([]string(nil), values...) + sort.Strings(result) + return result +} + +func indexedField(base string, index int) string { + return fmt.Sprintf("%s[%d]", base, index) +} + +func inputError(path, reason string) error { + return fmt.Errorf("%w: %s %s", ErrInvalidWipeInput, path, reason) +} + +func refusal(path, reason string) error { + return fmt.Errorf("%w: %s %s", ErrInventoryRefused, path, reason) +} diff --git a/internal/isolation/cleanup/plan.go b/internal/isolation/cleanup/plan.go new file mode 100644 index 0000000..648c4f9 --- /dev/null +++ b/internal/isolation/cleanup/plan.go @@ -0,0 +1,149 @@ +// Copyright 2026 CtrlBoard.dev +// SPDX-License-Identifier: Apache-2.0 + +package cleanup + +import ( + "time" + + "github.com/thelostorbital/ctrldb/internal/isolation" +) + +// Disposition is the closed outcome class of one wipe plan. +type Disposition string + +const ( + // DispositionFirstRunObserveOnly records the first execution after + // bootstrap. It plans and records but deletes nothing. + DispositionFirstRunObserveOnly Disposition = "first-run-observe-only" + // DispositionHarnessNotUsable records that the harness is not open and + // usable; the wipe observes and records but deletes nothing. + DispositionHarnessNotUsable Disposition = "harness-not-usable" + // DispositionNothingExpired records a usable harness with no expired + // owned resource. + DispositionNothingExpired Disposition = "nothing-expired" + // DispositionDelete authorizes the sealed, ordered deletions. + DispositionDelete Disposition = "delete" +) + +// OutstandingDeletion is one deletion recorded by an earlier run whose +// completion was never confirmed. Attempts counts the issued attempts. +type OutstandingDeletion struct { + CanonicalKey string + Attempts uint32 +} + +// History is the durable wipe record summary read before planning. PriorRuns +// counts completed plan records; zero means this is the first execution. +type History struct { + PriorRuns uint64 + Outstanding []OutstandingDeletion +} + +// PlanInput is everything one wipe execution may consult. +type PlanInput struct { + Policy WipePolicy + State isolation.HarnessStateV1 + Inventory Inventory + LifetimeRecords []LifetimeRecord + History History + Now time.Time +} + +// WipePlan is the sealed result of Plan. Its selection is complete even when +// the disposition forbids deletion so the run can be recorded and reviewed. +type WipePlan struct { + Mode string + ProjectID string + HarnessIntegritySHA256 string + BootstrapPhase isolation.BootstrapPhase + TestUsability isolation.TestUsability + Disposition Disposition + RunNumber uint64 + Selection Selection + seal string +} + +// Sealed reports whether the plan and every deletion were produced unchanged +// by Plan. +func (plan WipePlan) Sealed() bool { + if plan.seal == "" || plan.seal != sealPlan(plan) { + return false + } + for _, deletion := range plan.Selection.Deletions { + if !deletion.Sealed() { + return false + } + } + return true +} + +// Plan selects deletions from the exhaustive inventory and decides whether +// this execution may issue them. It performs no I/O. +func Plan(input PlanInput) (WipePlan, error) { + if err := validatePolicy(input.Policy); err != nil { + return WipePlan{}, err + } + if err := validateNow(input.Now); err != nil { + return WipePlan{}, err + } + state := input.State + if state.IntegritySHA256() == "" || state.ProjectID() != input.Policy.ProjectID || state.EnvironmentClass() != "disposable" { + return WipePlan{}, inputError("state", "is not the configured project's disposable harness state") + } + selection, err := Select(input.Policy, state.CleanupCapabilities(), input.Inventory, input.LifetimeRecords, input.Now) + if err != nil { + return WipePlan{}, err + } + if err := applyAttempts(&selection, input.History); err != nil { + return WipePlan{}, err + } + plan := WipePlan{ + Mode: ModeTestWipe, ProjectID: input.Policy.ProjectID, HarnessIntegritySHA256: state.IntegritySHA256(), + BootstrapPhase: state.BootstrapPhase(), TestUsability: state.TestUsability(), + RunNumber: input.History.PriorRuns + 1, Selection: selection, + } + switch { + case input.History.PriorRuns == 0: + plan.Disposition = DispositionFirstRunObserveOnly + case state.BootstrapPhase() != isolation.BootstrapPhaseOpen || state.TestUsability() != isolation.TestUsabilityUsable: + plan.Disposition = DispositionHarnessNotUsable + case len(selection.Deletions) == 0: + plan.Disposition = DispositionNothingExpired + default: + plan.Disposition = DispositionDelete + } + plan.seal = sealPlan(plan) + return plan, nil +} + +// applyAttempts numbers each deletion from the durable outstanding record so +// a retry after partial completion is visibly a retry, never a first attempt. +func applyAttempts(selection *Selection, history History) error { + attempts := make(map[string]uint32, len(history.Outstanding)) + for index, outstanding := range history.Outstanding { + if outstanding.CanonicalKey == "" || outstanding.Attempts == 0 { + return inputError(indexedField("history.outstanding", index), "must identify one attempted deletion") + } + if _, duplicate := attempts[outstanding.CanonicalKey]; duplicate { + return inputError(indexedField("history.outstanding", index), "duplicates an earlier outstanding deletion") + } + attempts[outstanding.CanonicalKey] = outstanding.Attempts + } + for index := range selection.Deletions { + deletion := &selection.Deletions[index] + deletion.Attempt = attempts[deletion.Identity.CanonicalKey] + 1 + deletion.seal = sealDeletion(*deletion) + } + return nil +} + +func sealPlan(plan WipePlan) string { + record := planRecord(plan) + record.IntegritySHA256 = "" + fingerprint, err := canonicalFingerprint(record) + if err != nil { + return "" + } + return fingerprint +} diff --git a/internal/isolation/cleanup/plan_test.go b/internal/isolation/cleanup/plan_test.go new file mode 100644 index 0000000..e2e2ae2 --- /dev/null +++ b/internal/isolation/cleanup/plan_test.go @@ -0,0 +1,244 @@ +// Copyright 2026 CtrlBoard.dev +// SPDX-License-Identifier: Apache-2.0 + +package cleanup_test + +import ( + "bytes" + "context" + "errors" + "slices" + "strings" + "testing" + + "github.com/thelostorbital/ctrldb/internal/isolation" + "github.com/thelostorbital/ctrldb/internal/isolation/cleanup" +) + +func TestPlanDispositionsFailClosedUntilHarnessIsUsable(t *testing.T) { + t.Parallel() + tests := []struct { + name string + state isolation.HarnessStateV1 + history cleanup.History + want cleanup.Disposition + }{ + {name: "first run after bootstrap", state: pendingState(t), history: cleanup.History{}, want: cleanup.DispositionFirstRunObserveOnly}, + {name: "first run even when open", state: openState(t), history: cleanup.History{}, want: cleanup.DispositionFirstRunObserveOnly}, + {name: "pending harness", state: pendingState(t), history: cleanup.History{PriorRuns: 1}, want: cleanup.DispositionHarnessNotUsable}, + {name: "drifted harness", state: driftedState(t), history: cleanup.History{PriorRuns: 1}, want: cleanup.DispositionHarnessNotUsable}, + {name: "open usable harness", state: openState(t), history: cleanup.History{PriorRuns: 1}, want: cleanup.DispositionDelete}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + plan, err := cleanup.Plan(planInput(t, test.state, test.history)) + if err != nil { + t.Fatalf("Plan() error = %v", err) + } + if plan.Disposition != test.want || !plan.Sealed() || plan.RunNumber != test.history.PriorRuns+1 { + t.Fatalf("plan disposition = %q run %d sealed %t, want %q", plan.Disposition, plan.RunNumber, plan.Sealed(), test.want) + } + if len(plan.Selection.Deletions) != 5 { + t.Fatalf("every disposition must still record the complete selection, got %d", len(plan.Selection.Deletions)) + } + journal := &memoryJournal{} + var deleter cleanup.Deleter = forbiddenDeleter{t: t} + if test.want == cleanup.DispositionDelete { + deleter = &scriptedDeleter{t: t} + } + result, err := cleanup.Execute(context.Background(), plan, journal, deleter, fixedClock) + if err != nil { + t.Fatalf("Execute() error = %v", err) + } + if len(journal.plans) != 1 || journal.plans[0].Disposition != test.want || journal.plans[0].RunNumber != plan.RunNumber { + t.Fatalf("plan record = %+v", journal.plans) + } + wantDeleted := 0 + if test.want == cleanup.DispositionDelete { + wantDeleted = 5 + } + if result.Deleted != wantDeleted || result.Failed != 0 || result.Planned != 5 || result.Retained != 3 || result.Protected != 2 { + t.Fatalf("result = %+v", result) + } + }) + } +} + +func TestPlanReportsNothingExpiredForCleanUsableHarness(t *testing.T) { + t.Parallel() + input := planInput(t, openState(t), cleanup.History{PriorRuns: 4}) + input.Inventory.Instances = input.Inventory.Instances[1:] + input.Inventory.Disks = input.Inventory.Disks[2:] + input.Inventory.Firewalls = append(input.Inventory.Firewalls[:2], input.Inventory.Firewalls[4:]...) + plan, err := cleanup.Plan(input) + if err != nil { + t.Fatalf("Plan() error = %v", err) + } + if plan.Disposition != cleanup.DispositionNothingExpired || len(plan.Selection.Deletions) != 0 { + t.Fatalf("plan = %+v", plan) + } +} + +func TestPlanRejectsForeignOrTamperedInputs(t *testing.T) { + t.Parallel() + input := planInput(t, openState(t), cleanup.History{PriorRuns: 1}) + input.Policy.ProjectID = "another-project-7" + input.Inventory.ProjectID = "another-project-7" + if _, err := cleanup.Plan(input); !errors.Is(err, cleanup.ErrInvalidWipeInput) { + t.Fatalf("cross-project state error = %v", err) + } + if _, err := cleanup.Plan(cleanup.PlanInput{Policy: testPolicy(), Now: testNow}); !errors.Is(err, cleanup.ErrInvalidWipeInput) { + t.Fatalf("zero state error = %v", err) + } + broken := planInput(t, openState(t), cleanup.History{PriorRuns: 1}) + broken.Inventory.Exhaustive = false + if _, err := cleanup.Plan(broken); !errors.Is(err, cleanup.ErrInventoryRefused) { + t.Fatalf("non-exhaustive inventory error = %v", err) + } + duplicated := planInput(t, openState(t), cleanup.History{PriorRuns: 1, Outstanding: []cleanup.OutstandingDeletion{{CanonicalKey: "k", Attempts: 1}, {CanonicalKey: "k", Attempts: 2}}}) + if _, err := cleanup.Plan(duplicated); !errors.Is(err, cleanup.ErrInvalidWipeInput) { + t.Fatalf("duplicate outstanding error = %v", err) + } + + plan, err := cleanup.Plan(planInput(t, openState(t), cleanup.History{PriorRuns: 1})) + if err != nil { + t.Fatalf("Plan() error = %v", err) + } + tampered := plan + tampered.Disposition = cleanup.DispositionDelete + tampered.Selection.Deletions = append([]cleanup.Deletion(nil), plan.Selection.Deletions...) + tampered.Selection.Deletions[0].Identity.Name = "ctrldb-prod-db-1" + if tampered.Sealed() { + t.Fatal("tampered plan must not be sealed") + } + journal := &memoryJournal{} + if _, err := cleanup.Execute(context.Background(), tampered, journal, forbiddenDeleter{t: t}, fixedClock); !errors.Is(err, cleanup.ErrInvalidWipeInput) { + t.Fatalf("tampered Execute() error = %v", err) + } + if len(journal.plans) != 0 { + t.Fatal("a tampered plan must not be recorded") + } + if _, err := cleanup.Execute(context.Background(), plan, journal, nil, fixedClock); !errors.Is(err, cleanup.ErrInvalidWipeInput) { + t.Fatalf("nil deleter error = %v", err) + } + if _, err := cleanup.Execute(context.Background(), plan, &memoryJournal{failPlan: true}, forbiddenDeleter{t: t}, fixedClock); !errors.Is(err, cleanup.ErrWipeExecution) { + t.Fatalf("unrecorded plan error = %v", err) + } +} + +func TestExecuteRecordsBeforeIssuingAndConvergesAfterPartialFailure(t *testing.T) { + t.Parallel() + plan, err := cleanup.Plan(planInput(t, openState(t), cleanup.History{PriorRuns: 2})) + if err != nil { + t.Fatalf("Plan() error = %v", err) + } + journal := &memoryJournal{} + deleter := &scriptedDeleter{t: t, failAt: 2} + result, err := cleanup.Execute(context.Background(), plan, journal, deleter, fixedClock) + if !errors.Is(err, cleanup.ErrWipeExecution) { + t.Fatalf("Execute() error = %v", err) + } + if result.Deleted != 1 || result.Failed != 1 || len(result.Deletions) != 2 || len(deleter.calls) != 2 { + t.Fatalf("result = %+v calls = %d", result, len(deleter.calls)) + } + if result.Deletions[1].Status != cleanup.DeletionFailed || result.Deletions[1].Failure.String() == "" { + t.Fatalf("failed outcome = %+v", result.Deletions[1]) + } + statuses := make([]string, 0, len(journal.deletions)) + for _, record := range journal.deletions { + if record.PlanIntegrity != journal.plans[0].IntegritySHA256 || !strings.HasPrefix(record.SchemaVersion, "ctrldb.ctrlboard.dev/") || record.IntegritySHA256 == "" { + t.Fatalf("deletion record = %+v", record) + } + statuses = append(statuses, string(record.Status)+":"+record.Deletion.Name) + } + want := []string{ + "issued:ctrldb-test-run-0a1b-node", "deleted:ctrldb-test-run-0a1b-node", + "issued:ctrldb-test-run-0a1b-data", "failed:ctrldb-test-run-0a1b-data", + } + if !slices.Equal(statuses, want) { + t.Fatalf("deletion records = %v, want %v", statuses, want) + } + + // The next run observes the instance gone and retries the failed disk as + // attempt two while the untouched resources start at attempt one. + retry := planInput(t, openState(t), cleanup.History{PriorRuns: 3, Outstanding: []cleanup.OutstandingDeletion{ + {CanonicalKey: plan.Selection.Deletions[1].Identity.CanonicalKey, Attempts: 1}, + }}) + retry.Inventory.Instances = retry.Inventory.Instances[1:] + for index := range retry.Inventory.Disks[:2] { + retry.Inventory.Disks[index].AttachedInstances = nil + } + retryPlan, err := cleanup.Plan(retry) + if err != nil { + t.Fatalf("retry Plan() error = %v", err) + } + wantNames := []string{ + "compute.disks:ctrldb-test-run-0a1b-data", "compute.disks:ctrldb-test-run-0a1b-node", + "compute.firewalls:ctrldb-test-run-0a1b-iap-ssh", "compute.firewalls:ctrldb-test-run-0a1b-internal", + } + if got := deletionNames(retryPlan.Selection.Deletions); !slices.Equal(got, wantNames) { + t.Fatalf("retry deletions = %v, want %v", got, wantNames) + } + if retryPlan.Selection.Deletions[0].Attempt != 2 || retryPlan.Selection.Deletions[1].Attempt != 1 { + t.Fatalf("retry attempts = %d, %d", retryPlan.Selection.Deletions[0].Attempt, retryPlan.Selection.Deletions[1].Attempt) + } + retryJournal := &memoryJournal{} + retryDeleter := &scriptedDeleter{t: t} + retryResult, err := cleanup.Execute(context.Background(), retryPlan, retryJournal, retryDeleter, fixedClock) + if err != nil || retryResult.Deleted != 4 || retryResult.Failed != 0 { + t.Fatalf("retry Execute() = %+v, %v", retryResult, err) + } +} + +func TestExecuteStopsWhenContextIsCanceled(t *testing.T) { + t.Parallel() + plan, err := cleanup.Plan(planInput(t, openState(t), cleanup.History{PriorRuns: 1})) + if err != nil { + t.Fatalf("Plan() error = %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + journal := &memoryJournal{} + result, err := cleanup.Execute(ctx, plan, journal, forbiddenDeleter{t: t}, fixedClock) + if !errors.Is(err, cleanup.ErrWipeExecution) || result.Deleted != 0 || len(journal.deletions) != 0 { + t.Fatalf("canceled Execute() = %+v, %v, records %d", result, err, len(journal.deletions)) + } +} + +func TestWipeRecordRoundTripsCanonicallyAndRejectsTampering(t *testing.T) { + t.Parallel() + plan, err := cleanup.Plan(planInput(t, openState(t), cleanup.History{PriorRuns: 1})) + if err != nil { + t.Fatalf("Plan() error = %v", err) + } + record, err := plan.Record() + if err != nil { + t.Fatalf("Record() error = %v", err) + } + encoded, err := record.CanonicalJSON() + if err != nil { + t.Fatalf("CanonicalJSON() error = %v", err) + } + parsed, err := cleanup.ParseWipeRecordV1(encoded) + if err != nil { + t.Fatalf("ParseWipeRecordV1() error = %v", err) + } + again, err := parsed.CanonicalJSON() + if err != nil || !bytes.Equal(again, encoded) { + t.Fatalf("round trip changed the record: %v", err) + } + if bytes.Contains(encoded, []byte("seal")) { + t.Fatal("the durable record must not carry the in-memory seal") + } + for name, input := range map[string][]byte{ + "tampered name": bytes.Replace(encoded, []byte("ctrldb-test-run-0a1b-node"), []byte("ctrldb-prod-db-1"), 1), + "trailing data": append(append([]byte(nil), encoded...), '{', '}'), + "unknown field": bytes.Replace(encoded, []byte(`"mode":`), []byte(`"extra":1,"mode":`), 1), + "blank integrity": bytes.Replace(encoded, []byte(record.IntegritySHA256), []byte(strings.Repeat("0", 64)), 1), + } { + if _, err := cleanup.ParseWipeRecordV1(input); !errors.Is(err, cleanup.ErrInvalidWipeRecord) { + t.Fatalf("%s: error = %v", name, err) + } + } +} diff --git a/internal/isolation/cleanup/record.go b/internal/isolation/cleanup/record.go new file mode 100644 index 0000000..d9d127e --- /dev/null +++ b/internal/isolation/cleanup/record.go @@ -0,0 +1,208 @@ +// Copyright 2026 CtrlBoard.dev +// SPDX-License-Identifier: Apache-2.0 + +package cleanup + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "time" + + "github.com/thelostorbital/ctrldb/internal/isolation" + "github.com/thelostorbital/ctrldb/internal/redact" +) + +const ( + WipeRecordSchemaV1 = "ctrldb.ctrlboard.dev/test-wipe/v1" + DeletionRecordSchemaV1 = "ctrldb.ctrlboard.dev/test-wipe-deletion/v1" +) + +var ErrInvalidWipeRecord = errors.New("invalid test wipe record") + +// DeletionStatus is the closed lifecycle of one recorded deletion. +type DeletionStatus string + +const ( + DeletionIssued DeletionStatus = "issued" + DeletionDeleted DeletionStatus = "deleted" + DeletionFailed DeletionStatus = "failed" +) + +// PlannedDeletionV1 is the durable, seal-free projection of one deletion. +type PlannedDeletionV1 struct { + Sequence int `json:"sequence"` + Capability isolation.CleanupCapability `json:"capability"` + Project string `json:"project"` + Location string `json:"location"` + Name string `json:"name"` + RunID string `json:"runId"` + CreatedAt time.Time `json:"createdAt"` + ExpiredAt time.Time `json:"expiredAt"` + KeepDisks bool `json:"keepDisks"` + Attempt uint32 `json:"attempt"` +} + +// RecordedResourceV1 is one non-deleted resource the wipe accounted for. +type RecordedResourceV1 struct { + Capability isolation.CleanupCapability `json:"capability,omitempty"` + Project string `json:"project"` + Location string `json:"location"` + Name string `json:"name"` + Reason string `json:"reason,omitempty"` + ExpiresAt *time.Time `json:"expiresAt,omitempty"` + BlockedBy []string `json:"blockedBy,omitempty"` +} + +// WipeRecordV1 is the canonical `test/wipe/.json` document written +// before any deletion is issued. It contains no credentials, provider output, +// or free text. +type WipeRecordV1 struct { + SchemaVersion string `json:"schemaVersion"` + Mode string `json:"mode"` + ProjectID string `json:"projectId"` + RunNumber uint64 `json:"runNumber"` + HarnessIntegritySHA256 string `json:"harnessIntegritySha256"` + BootstrapPhase string `json:"bootstrapPhase"` + TestUsability string `json:"testUsability"` + Disposition Disposition `json:"disposition"` + InventoryRevision string `json:"inventoryRevision"` + PlannedAt time.Time `json:"plannedAt"` + MaxLifetimeSeconds int64 `json:"maxLifetimeSeconds"` + IgnoredForeign int `json:"ignoredForeign"` + Deletions []PlannedDeletionV1 `json:"deletions"` + Retained []RecordedResourceV1 `json:"retained"` + Deferred []RecordedResourceV1 `json:"deferred"` + Protected []RecordedResourceV1 `json:"protected"` + IntegritySHA256 string `json:"integritySha256"` +} + +// DeletionRecordV1 is written immediately before a deletion is issued and +// again with its outcome. The pre-issue record makes a crash between record +// and provider call visible to the next run. +type DeletionRecordV1 struct { + SchemaVersion string `json:"schemaVersion"` + PlanIntegrity string `json:"planIntegritySha256"` + Deletion PlannedDeletionV1 `json:"deletion"` + Status DeletionStatus `json:"status"` + IssuedAt time.Time `json:"issuedAt"` + CompletedAt *time.Time `json:"completedAt,omitempty"` + Failure redact.Text `json:"failure"` + IntegritySHA256 string `json:"integritySha256"` +} + +// Record returns the canonical durable record for a sealed plan. +func (plan WipePlan) Record() (WipeRecordV1, error) { + if !plan.Sealed() { + return WipeRecordV1{}, inputError("plan", "is not a sealed wipe plan") + } + record := planRecord(plan) + fingerprint, err := canonicalFingerprint(record) + if err != nil { + return WipeRecordV1{}, err + } + record.IntegritySHA256 = fingerprint + return record, nil +} + +// CanonicalJSON returns the deterministic encoding of a complete record. +func (record WipeRecordV1) CanonicalJSON() ([]byte, error) { + if record.SchemaVersion != WipeRecordSchemaV1 || !sha256Pattern.MatchString(record.IntegritySHA256) { + return nil, ErrInvalidWipeRecord + } + check := record + check.IntegritySHA256 = "" + fingerprint, err := canonicalFingerprint(check) + if err != nil || fingerprint != record.IntegritySHA256 { + return nil, ErrInvalidWipeRecord + } + return json.Marshal(record) +} + +// ParseWipeRecordV1 strictly decodes one canonical record and verifies its +// integrity. Unknown fields, trailing data, and tampering fail closed. +func ParseWipeRecordV1(input []byte) (WipeRecordV1, error) { + var record WipeRecordV1 + decoder := json.NewDecoder(bytes.NewReader(input)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&record); err != nil { + return WipeRecordV1{}, ErrInvalidWipeRecord + } + if err := decoder.Decode(new(any)); !errors.Is(err, io.EOF) { + return WipeRecordV1{}, ErrInvalidWipeRecord + } + canonical, err := record.CanonicalJSON() + if err != nil || !bytes.Equal(canonical, input) { + return WipeRecordV1{}, ErrInvalidWipeRecord + } + return record, nil +} + +func planRecord(plan WipePlan) WipeRecordV1 { + selection := plan.Selection + record := WipeRecordV1{ + SchemaVersion: WipeRecordSchemaV1, Mode: plan.Mode, ProjectID: plan.ProjectID, RunNumber: plan.RunNumber, + HarnessIntegritySHA256: plan.HarnessIntegritySHA256, BootstrapPhase: string(plan.BootstrapPhase), + TestUsability: string(plan.TestUsability), Disposition: plan.Disposition, + InventoryRevision: selection.InventoryRevision, PlannedAt: selection.Now, + MaxLifetimeSeconds: int64(selection.MaxLifetime / time.Second), IgnoredForeign: selection.Ignored, + Deletions: make([]PlannedDeletionV1, 0, len(selection.Deletions)), + Retained: make([]RecordedResourceV1, 0, len(selection.Retained)), + Deferred: make([]RecordedResourceV1, 0, len(selection.Deferred)), + Protected: make([]RecordedResourceV1, 0, len(selection.Protected)), + } + for _, deletion := range selection.Deletions { + record.Deletions = append(record.Deletions, plannedDeletion(deletion)) + } + for _, retained := range selection.Retained { + expires := retained.ExpiresAt + record.Retained = append(record.Retained, RecordedResourceV1{ + Capability: retained.Capability, Project: retained.Identity.Project, Location: retained.Identity.Location, + Name: retained.Identity.Name, Reason: "not expired", ExpiresAt: &expires, + }) + } + for _, deferred := range selection.Deferred { + record.Deferred = append(record.Deferred, RecordedResourceV1{ + Capability: isolation.CleanupComputeDisks, Project: deferred.Identity.Project, Location: deferred.Identity.Location, + Name: deferred.Identity.Name, Reason: deferred.Reason, BlockedBy: append([]string(nil), deferred.BlockedBy...), + }) + } + for _, protected := range selection.Protected { + record.Protected = append(record.Protected, RecordedResourceV1{ + Project: protected.Identity.Project, Location: protected.Identity.Location, + Name: protected.Identity.Name, Reason: "permanent harness singleton", + }) + } + return record +} + +func plannedDeletion(deletion Deletion) PlannedDeletionV1 { + return PlannedDeletionV1{ + Sequence: deletion.Sequence, Capability: deletion.Capability, Project: deletion.Identity.Project, + Location: deletion.Identity.Location, Name: deletion.Identity.Name, RunID: deletion.RunID, + CreatedAt: deletion.CreatedAt, ExpiredAt: deletion.ExpiredAt, KeepDisks: deletion.KeepDisks, Attempt: deletion.Attempt, + } +} + +func sealDeletionRecord(record DeletionRecordV1) (DeletionRecordV1, error) { + record.SchemaVersion = DeletionRecordSchemaV1 + record.IntegritySHA256 = "" + fingerprint, err := canonicalFingerprint(record) + if err != nil { + return DeletionRecordV1{}, err + } + record.IntegritySHA256 = fingerprint + return record, nil +} + +func canonicalFingerprint(value any) (string, error) { + encoded, err := json.Marshal(value) + if err != nil { + return "", inputError("record", "could not be encoded canonically") + } + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]), nil +} diff --git a/internal/isolation/cleanup/selection.go b/internal/isolation/cleanup/selection.go new file mode 100644 index 0000000..cad1094 --- /dev/null +++ b/internal/isolation/cleanup/selection.go @@ -0,0 +1,406 @@ +// Copyright 2026 CtrlBoard.dev +// SPDX-License-Identifier: Apache-2.0 + +package cleanup + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "sort" + "strings" + "time" + + "github.com/thelostorbital/ctrldb/internal/config" + "github.com/thelostorbital/ctrldb/internal/isolation" +) + +// deletionSealDomain separates the sealed-deletion fingerprint from every +// other SHA-256 value in the repository. +const deletionSealDomain = "ctrldb.ctrlboard.dev/test-wipe/deletion-seal/v1" + +// Deletion is one selected, ordered removal. Its seal can only be produced by +// this package; a provider adapter must refuse an unsealed value so that a +// plain resource name can never reach a delete verb. +type Deletion struct { + Sequence int + Capability isolation.CleanupCapability + Identity isolation.ResourceIdentity + RunID string + CreatedAt time.Time + ExpiredAt time.Time + KeepDisks bool + Attempt uint32 + seal string +} + +// Sealed reports whether the deletion was produced unchanged by Select. +func (deletion Deletion) Sealed() bool { + return deletion.seal != "" && deletion.seal == sealDeletion(deletion) +} + +// RetainedResource is an owned resource which has not reached expiry. +type RetainedResource struct { + Capability isolation.CleanupCapability + Identity isolation.ResourceIdentity + ExpiresAt time.Time +} + +// DeferredResource is a selected disk still attached to an instance which the +// wipe does not own or has not selected. It is recorded and left in place. +type DeferredResource struct { + Identity isolation.ResourceIdentity + Reason string + BlockedBy []string +} + +// ProtectedResource is a permanent harness singleton observed in the test +// namespace. It is never a wipe candidate. +type ProtectedResource struct { + Identity isolation.ResourceIdentity +} + +// Selection is the complete, ordered, sealed result of one exhaustive +// inventory: instances first, then disks, then run firewall rules. +type Selection struct { + ProjectID string + InventoryRevision string + Now time.Time + MaxLifetime time.Duration + Ignored int + Deletions []Deletion + Retained []RetainedResource + Deferred []DeferredResource + Protected []ProtectedResource +} + +// Select evaluates the whole inventory. Any ambiguous, cross-project, +// unsupported, duplicated, or unprovable test-namespace resource refuses the +// entire selection; nothing is silently skipped. +func Select(policy WipePolicy, capabilities []isolation.CleanupCapability, inventory Inventory, records []LifetimeRecord, now time.Time) (Selection, error) { + if err := validatePolicy(policy); err != nil { + return Selection{}, err + } + if err := validateNow(now); err != nil { + return Selection{}, err + } + if err := isolation.ValidateCleanupCapabilities(capabilities); err != nil { + return Selection{}, inputError("capabilities", "must be the exact recorded cleanup capability set") + } + if err := validateInventoryWindow(policy, inventory, now); err != nil { + return Selection{}, err + } + if err := rejectDuplicateIdentities(inventory); err != nil { + return Selection{}, err + } + selection := Selection{ProjectID: policy.ProjectID, InventoryRevision: inventory.Revision, Now: now, MaxLifetime: policy.MaxLifetime} + instances, err := selectInstances(policy, inventory, now, &selection) + if err != nil { + return Selection{}, err + } + if err := selectDisks(policy, inventory, now, instances, &selection); err != nil { + return Selection{}, err + } + if err := selectFirewalls(policy, inventory, records, now, &selection); err != nil { + return Selection{}, err + } + for index := range selection.Deletions { + selection.Deletions[index].Sequence = index + 1 + selection.Deletions[index].seal = sealDeletion(selection.Deletions[index]) + } + return selection, nil +} + +func rejectDuplicateIdentities(inventory Inventory) error { + seen := make(map[string]struct{}) + check := func(path string, identity isolation.ResourceIdentity) error { + if identity.CanonicalKey == "" { + return refusal(path, "does not carry a complete canonical identity") + } + if _, duplicate := seen[identity.CanonicalKey]; duplicate { + return refusal(path, "duplicates an earlier inventory entry") + } + seen[identity.CanonicalKey] = struct{}{} + return nil + } + for index, item := range inventory.Instances { + if err := check(indexedField("inventory.instances", index), item.Identity); err != nil { + return err + } + } + for index, item := range inventory.Disks { + if err := check(indexedField("inventory.disks", index), item.Identity); err != nil { + return err + } + } + for index, item := range inventory.Firewalls { + if err := check(indexedField("inventory.firewalls", index), item.Identity); err != nil { + return err + } + } + return nil +} + +type ownershipClass uint8 + +const ( + ownershipForeign ownershipClass = iota + 1 + ownershipRun +) + +// classifyLabelCapable applies D-157 to instances and disks. A name in the +// test namespace without the three reserved labels, reserved labels without +// the run prefix, or a run resource whose run-id label does not bind its own +// name are all ambiguous and refuse the inventory. +func classifyLabelCapable(path, name string, labels map[string]string) (ownershipClass, string, error) { + prefixed := strings.HasPrefix(name, config.TestResourcePrefix) + labelled := labels[config.LabelManagedBy] == config.LabelManagedByValue && + labels[config.LabelEnvironment] == config.TestEnvironmentLabel && + labels[config.LabelPurpose] == config.TestResourcePurposeLabel + switch { + case !prefixed && !labelled: + return ownershipForeign, "", nil + case prefixed && !labelled: + return 0, "", refusal(path, "uses the test namespace without the reserved disposable labels") + case !prefixed && labelled: + return 0, "", refusal(path, "carries disposable labels outside the test namespace") + } + runID := labels[isolation.LabelRunID] + runPrefix, err := isolation.RunResourcePrefix(runID) + if err != nil { + return 0, "", refusal(path, "does not carry a valid run-id label") + } + if !strings.HasPrefix(name, runPrefix) || len(name) == len(runPrefix) { + return 0, "", refusal(path, "run-id label does not bind the exact run prefix") + } + if !config.IsTestResource(config.GeneratedResource{Name: name, Labels: labels}) { + return 0, "", refusal(path, "does not have exact disposable identity") + } + return ownershipRun, runID, nil +} + +type expirable struct { + target isolation.ExpirableTarget + runID string +} + +func selectInstances(policy WipePolicy, inventory Inventory, now time.Time, selection *Selection) (map[string]struct{}, error) { + candidates := make([]expirable, 0, len(inventory.Instances)) + for index, item := range inventory.Instances { + path := indexedField("inventory.instances", index) + if err := validateObservedIdentity(policy, path, item.Identity, isolation.ComputeInstanceKind); err != nil { + return nil, err + } + if err := validateObservedTimestamp(path, item.CreatedAt, now); err != nil { + return nil, err + } + class, runID, err := classifyLabelCapable(path, item.Identity.Name, item.Labels) + if err != nil { + return nil, err + } + if class == ownershipForeign { + selection.Ignored++ + continue + } + candidates = append(candidates, expirable{target: isolation.ExpirableTarget{ + Target: isolation.MutationTarget{Identity: item.Identity, Labels: cloneLabels(item.Labels)}, CreatedAt: item.CreatedAt, + }, runID: runID}) + } + expired, err := expiredTargets(policy, candidates, now, isolation.CleanupComputeInstances, selection) + if err != nil { + return nil, err + } + selected := make(map[string]struct{}, len(expired)) + for _, item := range expired { + selected[item.target.Target.Identity.Name] = struct{}{} + selection.Deletions = append(selection.Deletions, Deletion{ + Capability: isolation.CleanupComputeInstances, Identity: item.target.Target.Identity, RunID: item.runID, + CreatedAt: item.target.CreatedAt, ExpiredAt: item.target.CreatedAt.Add(policy.MaxLifetime), KeepDisks: true, + }) + } + return selected, nil +} + +func selectDisks(policy WipePolicy, inventory Inventory, now time.Time, selectedInstances map[string]struct{}, selection *Selection) error { + candidates := make([]expirable, 0, len(inventory.Disks)) + attachments := make(map[string][]string, len(inventory.Disks)) + for index, item := range inventory.Disks { + path := indexedField("inventory.disks", index) + if err := validateObservedIdentity(policy, path, item.Identity, isolation.ComputeDiskKind); err != nil { + return err + } + if err := validateObservedTimestamp(path, item.CreatedAt, now); err != nil { + return err + } + class, runID, err := classifyLabelCapable(path, item.Identity.Name, item.Labels) + if err != nil { + return err + } + if class == ownershipForeign { + selection.Ignored++ + continue + } + attachments[item.Identity.CanonicalKey] = sortedNames(item.AttachedInstances) + candidates = append(candidates, expirable{target: isolation.ExpirableTarget{ + Target: isolation.MutationTarget{Identity: item.Identity, Labels: cloneLabels(item.Labels)}, CreatedAt: item.CreatedAt, + }, runID: runID}) + } + expired, err := expiredTargets(policy, candidates, now, isolation.CleanupComputeDisks, selection) + if err != nil { + return err + } + for _, item := range expired { + identity := item.target.Target.Identity + var blockers []string + for _, user := range attachments[identity.CanonicalKey] { + if _, ok := selectedInstances[user]; !ok { + blockers = append(blockers, user) + } + } + if len(blockers) != 0 { + selection.Deferred = append(selection.Deferred, DeferredResource{Identity: identity, Reason: "attached to an instance outside this selection", BlockedBy: blockers}) + continue + } + selection.Deletions = append(selection.Deletions, Deletion{ + Capability: isolation.CleanupComputeDisks, Identity: identity, RunID: item.runID, + CreatedAt: item.target.CreatedAt, ExpiredAt: item.target.CreatedAt.Add(policy.MaxLifetime), + }) + } + return nil +} + +// expiredTargets delegates age evaluation to the isolation guard, which +// re-validates every candidate and fails the whole set on any ambiguity. +func expiredTargets(policy WipePolicy, candidates []expirable, now time.Time, capability isolation.CleanupCapability, selection *Selection) ([]expirable, error) { + targets := make([]isolation.ExpirableTarget, len(candidates)) + byKey := make(map[string]expirable, len(candidates)) + for index, candidate := range candidates { + targets[index] = candidate.target + byKey[candidate.target.Target.Identity.CanonicalKey] = candidate + } + expired, err := isolation.SelectExpiredTargets(isolation.CleanupPolicy{ProjectID: policy.ProjectID}, targets, now, policy.MaxLifetime) + if err != nil { + return nil, refusal(string(capability), "did not pass the isolation cleanup guard") + } + expiredKeys := make(map[string]struct{}, len(expired)) + result := make([]expirable, 0, len(expired)) + for _, item := range expired { + key := item.Target.Identity.CanonicalKey + expiredKeys[key] = struct{}{} + result = append(result, byKey[key]) + } + for _, candidate := range candidates { + if _, ok := expiredKeys[candidate.target.Target.Identity.CanonicalKey]; ok { + continue + } + selection.Retained = append(selection.Retained, RetainedResource{ + Capability: capability, Identity: candidate.target.Target.Identity, ExpiresAt: candidate.target.CreatedAt.Add(policy.MaxLifetime), + }) + } + return result, nil +} + +func selectFirewalls(policy WipePolicy, inventory Inventory, records []LifetimeRecord, now time.Time, selection *Selection) error { + byName, err := firewallRecordsByName(records) + if err != nil { + return err + } + cleanupPolicy := isolation.CleanupPolicy{ProjectID: policy.ProjectID} + deletions := make([]Deletion, 0) + for index, item := range inventory.Firewalls { + path := indexedField("inventory.firewalls", index) + if err := validateObservedIdentity(policy, path, item.Identity, isolation.ComputeFirewallKind); err != nil { + return err + } + if err := validateObservedTimestamp(path, item.CreatedAt, now); err != nil { + return err + } + name := item.Identity.Name + if name == isolation.TestIAPSSHFirewallName || name == isolation.TestInternalFirewallName { + selection.Protected = append(selection.Protected, ProtectedResource{Identity: item.Identity}) + continue + } + if !strings.HasPrefix(name, config.TestResourcePrefix) { + selection.Ignored++ + continue + } + record, ok := byName[name] + if !ok { + return refusal(path, "uses the test namespace without a durable run lifetime record") + } + target := isolation.RunFirewallCleanupTarget{ + Identity: item.Identity, Description: item.Description, RunLifetime: record.Contract, + ExpectedRecord: record.Expected, ObservedAt: inventory.ObservedAt, + } + if err := isolation.ValidateRunFirewallCleanupTarget(cleanupPolicy, target, isolation.RunFirewallCleanupRecordedTeardown, now, policy.MaxLifetime); err != nil { + return refusal(path, "does not match its durable run lifetime record") + } + if now.Before(record.Contract.ExpiresAt) { + selection.Retained = append(selection.Retained, RetainedResource{Capability: isolation.CleanupComputeFirewalls, Identity: item.Identity, ExpiresAt: record.Contract.ExpiresAt}) + continue + } + if err := isolation.ValidateRunFirewallCleanupTarget(cleanupPolicy, target, isolation.RunFirewallCleanupExpiredWipe, now, policy.MaxLifetime); err != nil { + return refusal(path, "has not reached a provable expiry") + } + deletions = append(deletions, Deletion{ + Capability: isolation.CleanupComputeFirewalls, Identity: item.Identity, RunID: record.Contract.RunID, + CreatedAt: item.CreatedAt, ExpiredAt: record.Contract.ExpiresAt, + }) + } + sort.SliceStable(deletions, func(i, j int) bool { return deletions[i].Identity.CanonicalKey < deletions[j].Identity.CanonicalKey }) + selection.Deletions = append(selection.Deletions, deletions...) + return nil +} + +func firewallRecordsByName(records []LifetimeRecord) (map[string]LifetimeRecord, error) { + byName := make(map[string]LifetimeRecord, 2*len(records)) + for index, record := range records { + path := indexedField("lifetimeRecords", index) + if _, err := isolation.RunLifetimeContractFingerprint(record.Contract); err != nil { + return nil, inputError(path, "is not a complete run lifetime record") + } + for _, purpose := range []isolation.FirewallPurpose{isolation.FirewallPurposeIAPSSH, isolation.FirewallPurposeInternalMongo} { + name, err := isolation.RunFirewallRuleName(record.Contract.RunID, purpose) + if err != nil { + return nil, inputError(path, "does not derive exact run firewall names") + } + if _, duplicate := byName[name]; duplicate { + return nil, inputError(path, "duplicates an earlier run lifetime record") + } + byName[name] = record + } + } + return byName, nil +} + +type deletionSealPayload struct { + Domain string `json:"domain"` + Sequence int `json:"sequence"` + Capability isolation.CleanupCapability `json:"capability"` + Identity isolation.ResourceIdentity `json:"identity"` + RunID string `json:"runId"` + CreatedAt time.Time `json:"createdAt"` + ExpiredAt time.Time `json:"expiredAt"` + KeepDisks bool `json:"keepDisks"` + Attempt uint32 `json:"attempt"` +} + +func sealDeletion(deletion Deletion) string { + encoded, err := json.Marshal(deletionSealPayload{ + Domain: deletionSealDomain, Sequence: deletion.Sequence, Capability: deletion.Capability, Identity: deletion.Identity, + RunID: deletion.RunID, CreatedAt: deletion.CreatedAt, ExpiredAt: deletion.ExpiredAt, KeepDisks: deletion.KeepDisks, Attempt: deletion.Attempt, + }) + if err != nil { + return "" + } + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]) +} + +func cloneLabels(labels map[string]string) map[string]string { + result := make(map[string]string, len(labels)) + for key, value := range labels { + result[key] = value + } + return result +} diff --git a/internal/isolation/cleanup/selection_test.go b/internal/isolation/cleanup/selection_test.go new file mode 100644 index 0000000..01efbd3 --- /dev/null +++ b/internal/isolation/cleanup/selection_test.go @@ -0,0 +1,246 @@ +// Copyright 2026 CtrlBoard.dev +// SPDX-License-Identifier: Apache-2.0 + +package cleanup_test + +import ( + "errors" + "slices" + "strings" + "testing" + "time" + + "github.com/thelostorbital/ctrldb/internal/config" + "github.com/thelostorbital/ctrldb/internal/isolation" + "github.com/thelostorbital/ctrldb/internal/isolation/cleanup" +) + +func TestSelectOrdersOwnedExpiredResourcesAndSealsThem(t *testing.T) { + t.Parallel() + inventory, records := fullInventory(t) + selection, err := cleanup.Select(testPolicy(), isolation.InitialCleanupCapabilities(), inventory, records, testNow) + if err != nil { + t.Fatalf("Select() error = %v", err) + } + want := []string{ + "compute.instances:ctrldb-test-run-0a1b-node", + "compute.disks:ctrldb-test-run-0a1b-data", + "compute.disks:ctrldb-test-run-0a1b-node", + "compute.firewalls:ctrldb-test-run-0a1b-iap-ssh", + "compute.firewalls:ctrldb-test-run-0a1b-internal", + } + if got := deletionNames(selection.Deletions); !slices.Equal(got, want) { + t.Fatalf("deletions = %v, want %v", got, want) + } + for index, deletion := range selection.Deletions { + if deletion.Sequence != index+1 || !deletion.Sealed() || deletion.RunID != testRunID { + t.Fatalf("deletion %d = %+v", index, deletion) + } + if deletion.Capability == isolation.CleanupComputeInstances && !deletion.KeepDisks { + t.Fatalf("instance deletion must retain disks: %+v", deletion) + } + if deletion.Capability != isolation.CleanupComputeInstances && deletion.KeepDisks { + t.Fatalf("only instances carry retained-disk semantics: %+v", deletion) + } + } + if selection.Ignored != 2 || len(selection.Protected) != 2 || len(selection.Retained) != 3 || len(selection.Deferred) != 0 { + t.Fatalf("selection accounting = ignored %d protected %d retained %d deferred %d", selection.Ignored, len(selection.Protected), len(selection.Retained), len(selection.Deferred)) + } + tampered := selection.Deletions[0] + tampered.Identity.Name = "ctrldb-test-run-0a1b-other" + if tampered.Sealed() { + t.Fatal("a modified deletion must not remain sealed") + } + if (cleanup.Deletion{Sequence: 1}).Sealed() { + t.Fatal("a caller-constructed deletion must not be sealed") + } +} + +func TestSelectHonorsExactMaxLifetimeBoundary(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + age time.Duration + selected bool + }{ + {name: "exactly max lifetime", age: testLifetime, selected: true}, + {name: "one second younger", age: testLifetime - time.Second, selected: false}, + {name: "one second older", age: testLifetime + time.Second, selected: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + inventory := cleanup.Inventory{ + ProjectID: testProject, Revision: testRevision, ObservedAt: testNow.Add(-time.Minute), ValidUntil: testNow.Add(time.Minute), Exhaustive: true, + Instances: []cleanup.InstanceObservation{{Identity: instanceIdentity(t, "ctrldb-test-run-0a1b-node"), Labels: runLabels(testRunID), CreatedAt: testNow.Add(-test.age)}}, + } + selection, err := cleanup.Select(testPolicy(), isolation.InitialCleanupCapabilities(), inventory, nil, testNow) + if err != nil { + t.Fatalf("Select() error = %v", err) + } + if (len(selection.Deletions) == 1) != test.selected || (len(selection.Retained) == 1) == test.selected { + t.Fatalf("deletions = %d retained = %d, want selected = %t", len(selection.Deletions), len(selection.Retained), test.selected) + } + }) + } +} + +func TestSelectDefersExpiredDiskAttachedToUnselectedInstance(t *testing.T) { + t.Parallel() + inventory := cleanup.Inventory{ + ProjectID: testProject, Revision: testRevision, ObservedAt: testNow.Add(-time.Minute), ValidUntil: testNow.Add(time.Minute), Exhaustive: true, + Instances: []cleanup.InstanceObservation{{Identity: instanceIdentity(t, "ctrldb-test-run-9z8y-node"), Labels: runLabels(otherRunID), CreatedAt: freshAt()}}, + Disks: []cleanup.DiskObservation{{ + Identity: diskIdentity(t, "ctrldb-test-run-0a1b-data"), Labels: runLabels(testRunID), CreatedAt: expiredAt(), + AttachedInstances: []string{"ctrldb-test-run-9z8y-node"}, + }}, + } + selection, err := cleanup.Select(testPolicy(), isolation.InitialCleanupCapabilities(), inventory, nil, testNow) + if err != nil { + t.Fatalf("Select() error = %v", err) + } + if len(selection.Deletions) != 0 || len(selection.Deferred) != 1 || !slices.Equal(selection.Deferred[0].BlockedBy, []string{"ctrldb-test-run-9z8y-node"}) { + t.Fatalf("deletions = %v deferred = %+v", selection.Deletions, selection.Deferred) + } +} + +func TestSelectRefusesEveryAmbiguousOrUnsafeInventory(t *testing.T) { + t.Parallel() + fresh := func(t *testing.T) cleanup.Inventory { + t.Helper() + inventory, _ := fullInventory(t) + return inventory + } + foreignProject := func(t *testing.T) isolation.ResourceIdentity { + t.Helper() + value := instanceIdentity(t, "ctrldb-test-run-0a1b-node") + value.Project = "foreign-project-42" + key, err := isolation.CanonicalTargetKey(value) + if err != nil { + t.Fatalf("CanonicalTargetKey() error = %v", err) + } + value.CanonicalKey = key + return value + } + tests := []struct { + name string + mutate func(t *testing.T, inventory *cleanup.Inventory, records *[]cleanup.LifetimeRecord, policy *cleanup.WipePolicy) + wantErr error + }{ + {name: "prefix without labels", mutate: func(_ *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Instances[0].Labels = map[string]string{config.LabelManagedBy: config.LabelManagedByValue} + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "labels without prefix", mutate: func(t *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Instances[2] = cleanup.InstanceObservation{Identity: instanceIdentity(t, "ctrldb-prod-node"), Labels: runLabels(testRunID), CreatedAt: expiredAt()} + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "missing run-id label", mutate: func(_ *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + delete(inventory.Instances[0].Labels, isolation.LabelRunID) + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "run-id label not binding name", mutate: func(_ *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Instances[0].Labels[isolation.LabelRunID] = otherRunID + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "production environment label on prefixed disk", mutate: func(_ *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Disks[0].Labels[config.LabelEnvironment] = "production" + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "cross-project candidate", mutate: func(t *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Instances[0].Identity = foreignProject(t) + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "cross-project inventory", mutate: func(_ *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.ProjectID = "foreign-project-42" + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "non-exhaustive inventory", mutate: func(_ *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Exhaustive = false + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "stale inventory", mutate: func(_ *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.ObservedAt, inventory.ValidUntil = testNow.Add(-3*time.Minute), testNow.Add(-time.Minute) + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "overlong inventory window", mutate: func(_ *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.ValidUntil = inventory.ObservedAt.Add(cleanup.MaxInventoryLifetime + time.Second) + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "unsupported kind present", mutate: func(t *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Unsupported = []cleanup.UnsupportedObservation{{Identity: identity(t, "snapshots", isolation.ResourceScopeGlobal, "global", "ctrldb-test-run-0a1b-snap")}} + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "duplicate identity", mutate: func(_ *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Instances = append(inventory.Instances, inventory.Instances[0]) + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "disk listed as instance", mutate: func(t *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Instances[0].Identity = diskIdentity(t, "ctrldb-test-run-0a1b-node-x") + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "canonical key mismatch", mutate: func(_ *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Instances[0].Identity.CanonicalKey = strings.Repeat("x", 12) + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "future creation time", mutate: func(_ *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Disks[0].CreatedAt = testNow.Add(time.Minute) + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "run firewall without lifetime record", mutate: func(_ *testing.T, _ *cleanup.Inventory, records *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + *records = (*records)[1:] + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "run firewall description drift", mutate: func(_ *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Firewalls[2].Description = "ctrldb:test-isolation:lifetime-sha256=" + strings.Repeat("0", 64) + ";revoke=WF-TEST-01" + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "unrecorded test-namespace firewall", mutate: func(t *testing.T, inventory *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + inventory.Firewalls = append(inventory.Firewalls, cleanup.FirewallObservation{Identity: firewallIdentity(t, "ctrldb-test-manual-rule"), CreatedAt: expiredAt()}) + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "lifetime record beyond max lifetime", mutate: func(t *testing.T, inventory *cleanup.Inventory, records *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + (*records)[0] = lifetimeRecord(t, testRunID, expiredAt(), expiredAt().Add(testLifetime+time.Second)) + inventory.Firewalls[2].Description = firewallDescription(t, (*records)[0]) + inventory.Firewalls[3].Description = firewallDescription(t, (*records)[0]) + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "stale lifetime record observation", mutate: func(_ *testing.T, _ *cleanup.Inventory, records *[]cleanup.LifetimeRecord, _ *cleanup.WipePolicy) { + (*records)[0].Expected.ObservedAt = testNow.Add(-10 * time.Minute) + (*records)[0].Expected.ValidUntil = testNow.Add(-6 * time.Minute) + }, wantErr: cleanup.ErrInventoryRefused}, + {name: "wrong project policy", mutate: func(_ *testing.T, _ *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, policy *cleanup.WipePolicy) { + policy.ProjectID = "Example Project" + }, wantErr: cleanup.ErrInvalidWipeInput}, + {name: "max lifetime too small", mutate: func(_ *testing.T, _ *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, policy *cleanup.WipePolicy) { + policy.MaxLifetime = cleanup.MinWipeLifetime - time.Second + }, wantErr: cleanup.ErrInvalidWipeInput}, + {name: "max lifetime too large", mutate: func(_ *testing.T, _ *cleanup.Inventory, _ *[]cleanup.LifetimeRecord, policy *cleanup.WipePolicy) { + policy.MaxLifetime = cleanup.MaxWipeLifetime + time.Second + }, wantErr: cleanup.ErrInvalidWipeInput}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + inventory, records := fullInventory(t) + policy := testPolicy() + test.mutate(t, &inventory, &records, &policy) + selection, err := cleanup.Select(policy, isolation.InitialCleanupCapabilities(), inventory, records, testNow) + if !errors.Is(err, test.wantErr) { + t.Fatalf("Select() error = %v, want %v", err, test.wantErr) + } + if len(selection.Deletions) != 0 { + t.Fatalf("a refused inventory must select nothing: %v", selection.Deletions) + } + }) + } + if _, err := cleanup.Select(testPolicy(), []isolation.CleanupCapability{isolation.CleanupComputeInstances}, fresh(t), nil, testNow); !errors.Is(err, cleanup.ErrInvalidWipeInput) { + t.Fatalf("partial capability set error = %v", err) + } + if _, err := cleanup.Select(testPolicy(), isolation.InitialCleanupCapabilities(), fresh(t), nil, testNow.In(time.FixedZone("x", 3600))); !errors.Is(err, cleanup.ErrInvalidWipeInput) { + t.Fatalf("non-UTC clock error = %v", err) + } +} + +func TestSelectNeverTargetsPermanentOrForeignResources(t *testing.T) { + t.Parallel() + inventory := cleanup.Inventory{ + ProjectID: testProject, Revision: testRevision, ObservedAt: testNow.Add(-time.Minute), ValidUntil: testNow.Add(time.Minute), Exhaustive: true, + Instances: []cleanup.InstanceObservation{ + {Identity: instanceIdentity(t, "ctrldb-prod-db-1"), Labels: map[string]string{config.LabelManagedBy: config.LabelManagedByValue, config.LabelEnvironment: "production"}, CreatedAt: expiredAt()}, + {Identity: instanceIdentity(t, "ctrldb-rst-rehearsal"), Labels: map[string]string{config.LabelManagedBy: config.LabelManagedByValue, config.LabelEnvironment: "production"}, CreatedAt: expiredAt()}, + }, + Firewalls: []cleanup.FirewallObservation{ + {Identity: firewallIdentity(t, isolation.TestIAPSSHFirewallName), CreatedAt: expiredAt()}, + {Identity: firewallIdentity(t, isolation.TestInternalFirewallName), CreatedAt: expiredAt()}, + {Identity: firewallIdentity(t, "ctrldb-prod-lease-abc"), CreatedAt: expiredAt()}, + }, + } + selection, err := cleanup.Select(testPolicy(), isolation.InitialCleanupCapabilities(), inventory, nil, testNow) + if err != nil { + t.Fatalf("Select() error = %v", err) + } + if len(selection.Deletions) != 0 || len(selection.Protected) != 2 || selection.Ignored != 3 { + t.Fatalf("selection = %+v", selection) + } +}