From 488aba0fb7aa06c54aeab8cd887f99c7d0918909 Mon Sep 17 00:00:00 2001 From: Ayush Patel Date: Wed, 13 May 2026 17:39:12 +0530 Subject: [PATCH 1/3] feat: add concurrency.group to serialize tests within a group Signed-off-by: Ayush Patel --- pkg/apis/v1alpha1/test.go | 12 ++++++++++++ pkg/apis/v1alpha1/zz_generated.deepcopy.go | 21 +++++++++++++++++++++ pkg/runner/runner.go | 20 +++++++++++++++++++- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/pkg/apis/v1alpha1/test.go b/pkg/apis/v1alpha1/test.go index 116195b7b..11a7da1e5 100644 --- a/pkg/apis/v1alpha1/test.go +++ b/pkg/apis/v1alpha1/test.go @@ -52,6 +52,10 @@ type TestSpec struct { // +optional Concurrent *bool `json:"concurrent,omitempty"` + // Concurrency defines concurrency options for the test. + // +optional + Concurrency *ConcurrencyOptions `json:"concurrency,omitempty"` + // SkipDelete determines whether the resources created by the test should be deleted after the test is executed. // +optional SkipDelete *bool `json:"skipDelete,omitempty"` @@ -112,6 +116,14 @@ type TestSpec struct { DeletionPropagationPolicy *metav1.DeletionPropagation `json:"deletionPropagationPolicy,omitempty"` } +// ConcurrencyOptions defines concurrency options for a test. +type ConcurrencyOptions struct { + // Group assigns the test to a concurrency group. Tests in the same group run + // sequentially; tests in different groups run in parallel. + // +optional + Group string `json:"group,omitempty"` +} + // Scenario defines per scenario bindings. type Scenario struct { // Scenario name. diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index 51eb4a0f3..7eabacd90 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -27,6 +27,22 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConcurrencyOptions) DeepCopyInto(out *ConcurrencyOptions) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConcurrencyOptions. +func (in *ConcurrencyOptions) DeepCopy() *ConcurrencyOptions { + if in == nil { + return nil + } + out := new(ConcurrencyOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ActionBindings) DeepCopyInto(out *ActionBindings) { *out = *in @@ -1312,6 +1328,11 @@ func (in *TestSpec) DeepCopyInto(out *TestSpec) { *out = new(bool) **out = **in } + if in.Concurrency != nil { + in, out := &in.Concurrency, &out.Concurrency + *out = new(ConcurrencyOptions) + **out = **in + } if in.SkipDelete != nil { in, out := &in.SkipDelete, &out.SkipDelete *out = new(bool) diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index d148aca44..58aaed371 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -3,6 +3,7 @@ package runner import ( "context" "fmt" + "sync" "testing" "time" @@ -83,6 +84,18 @@ func (r *runner) run(ctx context.Context, m mainstart, nsOptions v1alpha2.Namesp tc.IncFailed() return } + // build per-group mutexes so tests in the same group serialize + groupMutexes := map[string]*sync.Mutex{} + for i := range tests { + if tests[i].Test == nil { + continue + } + if c := tests[i].Test.Spec.Concurrency; c != nil && c.Group != "" { + if _, ok := groupMutexes[c.Group]; !ok { + groupMutexes[c.Group] = &sync.Mutex{} + } + } + } // loop through tests for i := range tests { test := tests[i] @@ -106,7 +119,12 @@ func (r *runner) run(ctx context.Context, m mainstart, nsOptions v1alpha2.Namesp // setup logger sink ctx = logging.WithSink(ctx, newSink(r.clock, tc.Quiet(), t.Log)) // setup concurrency - if test.Test.Spec.Concurrent == nil || *test.Test.Spec.Concurrent { + if c := test.Test.Spec.Concurrency; c != nil && c.Group != "" { + t.Parallel() + mu := groupMutexes[c.Group] + mu.Lock() + defer mu.Unlock() + } else if test.Test.Spec.Concurrent == nil || *test.Test.Spec.Concurrent { t.Parallel() } // setup reporting From fea81d373d4f4097b6e1f0472f47ff7f95bc0316 Mon Sep 17 00:00:00 2001 From: Ayush Patel Date: Wed, 13 May 2026 18:20:57 +0530 Subject: [PATCH 2/3] chore: update generated files for concurrency.group Signed-off-by: Ayush Patel --- .crds/chainsaw.kyverno.io_tests.yaml | 9 ++++++ pkg/apis/v1alpha1/zz_generated.deepcopy.go | 32 +++++++++---------- .../docs/reference/apis/chainsaw.v1alpha1.md | 14 ++++++++ 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/.crds/chainsaw.kyverno.io_tests.yaml b/.crds/chainsaw.kyverno.io_tests.yaml index a06a35fc2..18f9cba18 100644 --- a/.crds/chainsaw.kyverno.io_tests.yaml +++ b/.crds/chainsaw.kyverno.io_tests.yaml @@ -884,6 +884,15 @@ spec: description: Concurrent determines whether the test should run concurrently with other tests. type: boolean + concurrency: + description: Concurrency defines concurrency options for the test. + properties: + group: + description: |- + Group assigns the test to a concurrency group. Tests in the same group run + sequentially; tests in different groups run in parallel. + type: string + type: object delayBeforeCleanup: description: DelayBeforeCleanup adds a delay between the time a test ends and the time cleanup starts. diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index 7eabacd90..3f1fde42f 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -27,22 +27,6 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConcurrencyOptions) DeepCopyInto(out *ConcurrencyOptions) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConcurrencyOptions. -func (in *ConcurrencyOptions) DeepCopy() *ConcurrencyOptions { - if in == nil { - return nil - } - out := new(ConcurrencyOptions) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ActionBindings) DeepCopyInto(out *ActionBindings) { *out = *in @@ -549,6 +533,22 @@ func (in *Command) DeepCopy() *Command { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConcurrencyOptions) DeepCopyInto(out *ConcurrencyOptions) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConcurrencyOptions. +func (in *ConcurrencyOptions) DeepCopy() *ConcurrencyOptions { + if in == nil { + return nil + } + out := new(ConcurrencyOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Configuration) DeepCopyInto(out *Configuration) { *out = *in diff --git a/website/docs/reference/apis/chainsaw.v1alpha1.md b/website/docs/reference/apis/chainsaw.v1alpha1.md index c76bb532a..7f5202775 100644 --- a/website/docs/reference/apis/chainsaw.v1alpha1.md +++ b/website/docs/reference/apis/chainsaw.v1alpha1.md @@ -410,6 +410,19 @@ during the testing process.

| `args` | `[]string` | | |

Args is the command arguments.

| | `workDir` | `string` | | |

WorkDir is the working directory for command.

| +## ConcurrencyOptions {#chainsaw-kyverno-io-v1alpha1-ConcurrencyOptions} + +**Appears in:** + +- [TestSpec](#chainsaw-kyverno-io-v1alpha1-TestSpec) + +

ConcurrencyOptions defines concurrency options for a test.

+ + +| Field | Type | Required | Inline | Description | +|---|---|---|---|---| +| `group` | `string` | | |

Group assigns the test to a concurrency group. Tests in the same group run sequentially; tests in different groups run in parallel.

| + ## ConfigurationSpec {#chainsaw-kyverno-io-v1alpha1-ConfigurationSpec} **Appears in:** @@ -907,6 +920,7 @@ If a resource doesn't exist yet in the cluster it will fail.

| `clusters` | [`Clusters`](#chainsaw-kyverno-io-v1alpha1-Clusters) | | |

Clusters holds a registry to clusters to support multi-cluster tests.

| | `skip` | `bool` | | |

Skip determines whether the test should skipped.

| | `concurrent` | `bool` | | |

Concurrent determines whether the test should run concurrently with other tests.

| +| `concurrency` | [`ConcurrencyOptions`](#chainsaw-kyverno-io-v1alpha1-ConcurrencyOptions) | | |

Concurrency defines concurrency options for the test.

| | `skipDelete` | `bool` | | |

SkipDelete determines whether the resources created by the test should be deleted after the test is executed.

| | `template` | `bool` | | |

Template determines whether resources should be considered for templating.

| | `compiler` | `policy/v1alpha1.Compiler` | | |

Compiler defines the default compiler to use when evaluating expressions.

| From 0a65578354085f6da1db6782ddbc1946aa2538f7 Mon Sep 17 00:00:00 2001 From: Ayush Patel Date: Wed, 13 May 2026 18:25:16 +0530 Subject: [PATCH 3/3] test: add coverage for concurrency.group runner logic Signed-off-by: Ayush Patel --- pkg/runner/runner_test.go | 97 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index c94074952..7f8fa62b6 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -310,6 +310,103 @@ func Test_runner_Run(t *testing.T) { want: &summaryResult{ passed: 1, }, + }, { + name: "test with concurrency group", + config: model.Configuration{ + Namespace: v1alpha2.NamespaceOptions{ + Name: "chain-saw", + }, + }, + tc: func() enginecontext.TestContext { + client := &fake.FakeClient{ + GetFn: func(ctx context.Context, call int, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption) error { + return kerrors.NewNotFound(v1alpha1.Resource("Namespace"), "chain-saw") + }, + CreateFn: func(ctx context.Context, call int, obj ctrlclient.Object, opts ...ctrlclient.CreateOption) error { + return nil + }, + DeleteFn: func(ctx context.Context, call int, obj ctrlclient.Object, opts ...ctrlclient.DeleteOption) error { + return nil + }, + } + return mockTC(client) + }(), + tests: []discovery.Test{{ + Test: &model.Test{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "chainsaw.kyverno.io/v1alpha1", + Kind: "Test", + }, + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.TestSpec{ + Concurrency: &v1alpha1.ConcurrencyOptions{Group: "group-a"}, + Steps: []v1alpha1.TestStep{{ + TestStepSpec: v1alpha1.TestStepSpec{ + Try: []v1alpha1.Operation{{ + Script: &v1alpha1.Script{Content: "echo hello"}, + }}, + }, + }}, + }, + }, + }}, + want: &summaryResult{passed: 1}, + }, { + name: "multiple tests in same concurrency group", + config: model.Configuration{ + Namespace: v1alpha2.NamespaceOptions{ + Name: "chain-saw", + }, + }, + tc: func() enginecontext.TestContext { + client := &fake.FakeClient{ + GetFn: func(ctx context.Context, call int, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption) error { + return kerrors.NewNotFound(v1alpha1.Resource("Namespace"), "chain-saw") + }, + CreateFn: func(ctx context.Context, call int, obj ctrlclient.Object, opts ...ctrlclient.CreateOption) error { + return nil + }, + DeleteFn: func(ctx context.Context, call int, obj ctrlclient.Object, opts ...ctrlclient.DeleteOption) error { + return nil + }, + } + return mockTC(client) + }(), + tests: []discovery.Test{ + { + Test: &model.Test{ + TypeMeta: metav1.TypeMeta{APIVersion: "chainsaw.kyverno.io/v1alpha1", Kind: "Test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-1"}, + Spec: v1alpha1.TestSpec{ + Concurrency: &v1alpha1.ConcurrencyOptions{Group: "group-a"}, + Steps: []v1alpha1.TestStep{{ + TestStepSpec: v1alpha1.TestStepSpec{ + Try: []v1alpha1.Operation{{ + Script: &v1alpha1.Script{Content: "echo one"}, + }}, + }, + }}, + }, + }, + }, + { + Test: &model.Test{ + TypeMeta: metav1.TypeMeta{APIVersion: "chainsaw.kyverno.io/v1alpha1", Kind: "Test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-2"}, + Spec: v1alpha1.TestSpec{ + Concurrency: &v1alpha1.ConcurrencyOptions{Group: "group-a"}, + Steps: []v1alpha1.TestStep{{ + TestStepSpec: v1alpha1.TestStepSpec{ + Try: []v1alpha1.Operation{{ + Script: &v1alpha1.Script{Content: "echo two"}, + }}, + }, + }}, + }, + }, + }, + }, + want: &summaryResult{passed: 2}, }, { name: "With scanrio", config: model.Configuration{