feat: add concurrency.group to serialize tests within a group

.crds/chainsaw.kyverno.io_tests.yaml

@@ -884,6 +884,15 @@ spec:
                 description: Concurrent determines whether the test should run concurrently
                   with other tests.
                 type: boolean
+              concurrency:
+                description: Concurrency defines concurrency options for the test.
+                properties:
+                  group:
+                    description: |-
+                      Group assigns the test to a concurrency group. Tests in the same group run
+                      sequentially; tests in different groups run in parallel.
+                    type: string
+                type: object
               delayBeforeCleanup:
                 description: DelayBeforeCleanup adds a delay between the time a test
                   ends and the time cleanup starts.

pkg/apis/v1alpha1/test.go

@@ -52,6 +52,10 @@ type TestSpec struct {
 	// +optional
 	Concurrent *bool `json:"concurrent,omitempty"`
 
+	// Concurrency defines concurrency options for the test.
+	// +optional
+	Concurrency *ConcurrencyOptions `json:"concurrency,omitempty"`
+
 	// SkipDelete determines whether the resources created by the test should be deleted after the test is executed.
 	// +optional
 	SkipDelete *bool `json:"skipDelete,omitempty"`
@@ -112,6 +116,14 @@ type TestSpec struct {
 	DeletionPropagationPolicy *metav1.DeletionPropagation `json:"deletionPropagationPolicy,omitempty"`
 }
 
+// ConcurrencyOptions defines concurrency options for a test.
+type ConcurrencyOptions struct {
+	// Group assigns the test to a concurrency group. Tests in the same group run
+	// sequentially; tests in different groups run in parallel.
+	// +optional
+	Group string `json:"group,omitempty"`
+}
+
 // Scenario defines per scenario bindings.
 type Scenario struct {
 	// Scenario name.

pkg/runner/runner.go

@@ -3,6 +3,7 @@ package runner
 import (
 	"context"
 	"fmt"
+	"sync"
 	"testing"
 	"time"
 
@@ -83,6 +84,18 @@ func (r *runner) run(ctx context.Context, m mainstart, nsOptions v1alpha2.Namesp
 				tc.IncFailed()
 				return
 			}
+			// build per-group mutexes so tests in the same group serialize
+			groupMutexes := map[string]*sync.Mutex{}
+			for i := range tests {
+				if tests[i].Test == nil {
+					continue
+				}
+				if c := tests[i].Test.Spec.Concurrency; c != nil && c.Group != "" {
+					if _, ok := groupMutexes[c.Group]; !ok {
+						groupMutexes[c.Group] = &sync.Mutex{}
+					}
+				}
+			}
 			// loop through tests
 			for i := range tests {
 				test := tests[i]
@@ -106,7 +119,12 @@ func (r *runner) run(ctx context.Context, m mainstart, nsOptions v1alpha2.Namesp
 						// setup logger sink
 						ctx = logging.WithSink(ctx, newSink(r.clock, tc.Quiet(), t.Log))
 						// setup concurrency
-						if test.Test.Spec.Concurrent == nil || *test.Test.Spec.Concurrent {
+						if c := test.Test.Spec.Concurrency; c != nil && c.Group != "" {
+							t.Parallel()
+							mu := groupMutexes[c.Group]
+							mu.Lock()
+							defer mu.Unlock()
+						} else if test.Test.Spec.Concurrent == nil || *test.Test.Spec.Concurrent {
 							t.Parallel()
 						}
 						// setup reporting

pkg/runner/runner_test.go

@@ -310,6 +310,103 @@ func Test_runner_Run(t *testing.T) {
 		want: &summaryResult{
 			passed: 1,
 		},
+	}, {
+		name: "test with concurrency group",
+		config: model.Configuration{
+			Namespace: v1alpha2.NamespaceOptions{
+				Name: "chain-saw",
+			},
+		},
+		tc: func() enginecontext.TestContext {
+			client := &fake.FakeClient{
+				GetFn: func(ctx context.Context, call int, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption) error {
+					return kerrors.NewNotFound(v1alpha1.Resource("Namespace"), "chain-saw")
+				},
+				CreateFn: func(ctx context.Context, call int, obj ctrlclient.Object, opts ...ctrlclient.CreateOption) error {
+					return nil
+				},
+				DeleteFn: func(ctx context.Context, call int, obj ctrlclient.Object, opts ...ctrlclient.DeleteOption) error {
+					return nil
+				},
+			}
+			return mockTC(client)
+		}(),
+		tests: []discovery.Test{{
+			Test: &model.Test{
+				TypeMeta: metav1.TypeMeta{
+					APIVersion: "chainsaw.kyverno.io/v1alpha1",
+					Kind:       "Test",
+				},
+				ObjectMeta: metav1.ObjectMeta{Name: "test"},
+				Spec: v1alpha1.TestSpec{
+					Concurrency: &v1alpha1.ConcurrencyOptions{Group: "group-a"},
+					Steps: []v1alpha1.TestStep{{
+						TestStepSpec: v1alpha1.TestStepSpec{
+							Try: []v1alpha1.Operation{{
+								Script: &v1alpha1.Script{Content: "echo hello"},
+							}},
+						},
+					}},
+				},
+			},
+		}},
+		want: &summaryResult{passed: 1},
+	}, {
+		name: "multiple tests in same concurrency group",
+		config: model.Configuration{
+			Namespace: v1alpha2.NamespaceOptions{
+				Name: "chain-saw",
+			},
+		},
+		tc: func() enginecontext.TestContext {
+			client := &fake.FakeClient{
+				GetFn: func(ctx context.Context, call int, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption) error {
+					return kerrors.NewNotFound(v1alpha1.Resource("Namespace"), "chain-saw")
+				},
+				CreateFn: func(ctx context.Context, call int, obj ctrlclient.Object, opts ...ctrlclient.CreateOption) error {
+					return nil
+				},
+				DeleteFn: func(ctx context.Context, call int, obj ctrlclient.Object, opts ...ctrlclient.DeleteOption) error {
+					return nil
+				},
+			}
+			return mockTC(client)
+		}(),
+		tests: []discovery.Test{
+			{
+				Test: &model.Test{
+					TypeMeta:   metav1.TypeMeta{APIVersion: "chainsaw.kyverno.io/v1alpha1", Kind: "Test"},
+					ObjectMeta: metav1.ObjectMeta{Name: "test-1"},
+					Spec: v1alpha1.TestSpec{
+						Concurrency: &v1alpha1.ConcurrencyOptions{Group: "group-a"},
+						Steps: []v1alpha1.TestStep{{
+							TestStepSpec: v1alpha1.TestStepSpec{
+								Try: []v1alpha1.Operation{{
+									Script: &v1alpha1.Script{Content: "echo one"},
+								}},
+							},
+						}},
+					},
+				},
+			},
+			{
+				Test: &model.Test{
+					TypeMeta:   metav1.TypeMeta{APIVersion: "chainsaw.kyverno.io/v1alpha1", Kind: "Test"},
+					ObjectMeta: metav1.ObjectMeta{Name: "test-2"},
+					Spec: v1alpha1.TestSpec{
+						Concurrency: &v1alpha1.ConcurrencyOptions{Group: "group-a"},
+						Steps: []v1alpha1.TestStep{{
+							TestStepSpec: v1alpha1.TestStepSpec{
+								Try: []v1alpha1.Operation{{
+									Script: &v1alpha1.Script{Content: "echo two"},
+								}},
+							},
+						}},
+					},
+				},
+			},
+		},
+		want: &summaryResult{passed: 2},
 	}, {
 		name: "With scanrio",
 		config: model.Configuration{

website/docs/reference/apis/chainsaw.v1alpha1.md

@@ -410,6 +410,19 @@ during the testing process.</p>
 | `args` | `[]string` |  |  | <p>Args is the command arguments.</p> |
 | `workDir` | `string` |  |  | <p>WorkDir is the working directory for command.</p> |
 
+## ConcurrencyOptions     {#chainsaw-kyverno-io-v1alpha1-ConcurrencyOptions}
+
+**Appears in:**
+
+- [TestSpec](#chainsaw-kyverno-io-v1alpha1-TestSpec)
+
+<p>ConcurrencyOptions defines concurrency options for a test.</p>
+
+
+| Field | Type | Required | Inline | Description |
+|---|---|---|---|---|
+| `group` | `string` |  |  | <p>Group assigns the test to a concurrency group. Tests in the same group run sequentially; tests in different groups run in parallel.</p> |
+
 ## ConfigurationSpec     {#chainsaw-kyverno-io-v1alpha1-ConfigurationSpec}
 
 **Appears in:**
@@ -907,6 +920,7 @@ If a resource doesn't exist yet in the cluster it will fail.</p>
 | `clusters` | [`Clusters`](#chainsaw-kyverno-io-v1alpha1-Clusters) |  |  | <p>Clusters holds a registry to clusters to support multi-cluster tests.</p> |
 | `skip` | `bool` |  |  | <p>Skip determines whether the test should skipped.</p> |
 | `concurrent` | `bool` |  |  | <p>Concurrent determines whether the test should run concurrently with other tests.</p> |
+| `concurrency` | [`ConcurrencyOptions`](#chainsaw-kyverno-io-v1alpha1-ConcurrencyOptions) |  |  | <p>Concurrency defines concurrency options for the test.</p> |
 | `skipDelete` | `bool` |  |  | <p>SkipDelete determines whether the resources created by the test should be deleted after the test is executed.</p> |
 | `template` | `bool` |  |  | <p>Template determines whether resources should be considered for templating.</p> |
 | `compiler` | `policy/v1alpha1.Compiler` |  |  | <p>Compiler defines the default compiler to use when evaluating expressions.</p> |