From e75be08404e582b34f8b9050ae47a2aeeb7d6131 Mon Sep 17 00:00:00 2001 From: krishbajaj1609 Date: Tue, 12 May 2026 00:13:04 +0530 Subject: [PATCH] add unmanged crd detection to remove duplicated openapi path --- .../templates/cluster-roles.yaml | 6 ++ go.mod | 2 +- pkg/app/policyserver.go | 11 ++- pkg/crdcoexistence/detect.go | 64 +++++++++++++++ pkg/crdcoexistence/detect_test.go | 82 +++++++++++++++++++ pkg/crdcoexistence/periodic.go | 38 +++++++++ pkg/server/config.go | 9 ++ 7 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 pkg/crdcoexistence/detect.go create mode 100644 pkg/crdcoexistence/detect_test.go create mode 100644 pkg/crdcoexistence/periodic.go diff --git a/charts/reports-server/templates/cluster-roles.yaml b/charts/reports-server/templates/cluster-roles.yaml index 8dc6eb85..9c4a677e 100644 --- a/charts/reports-server/templates/cluster-roles.yaml +++ b/charts/reports-server/templates/cluster-roles.yaml @@ -87,6 +87,12 @@ rules: - update - watch - deletecollection +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list - apiGroups: - '' - events.k8s.io diff --git a/go.mod b/go.mod index 4fa9fea5..ed224b25 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( k8s.io/component-base v0.29.2 k8s.io/klog/v2 v2.130.1 k8s.io/kube-aggregator v0.29.2 + k8s.io/apiextensions-apiserver v0.29.2 k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 openreports.io v0.0.0-00010101000000-000000000000 sigs.k8s.io/wg-policy-prototypes v0.0.0-20231226153523-db3ef51d230f @@ -286,7 +287,6 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.29.2 // indirect k8s.io/cli-runtime v0.29.2 // indirect k8s.io/kms v0.29.2 // indirect k8s.io/kubectl v0.29.2 // indirect diff --git a/pkg/app/policyserver.go b/pkg/app/policyserver.go index 4da7a972..53e6f4b4 100644 --- a/pkg/app/policyserver.go +++ b/pkg/app/policyserver.go @@ -10,6 +10,7 @@ import ( "time" "github.com/kyverno/reports-server/pkg/app/opts" + "github.com/kyverno/reports-server/pkg/crdcoexistence" "github.com/kyverno/reports-server/pkg/server" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -87,8 +88,16 @@ func runCommand(o *opts.Options, stopCh <-chan struct{}) error { if err != nil { return err } + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-stopCh + cancel() + }() + go crdcoexistence.PeriodicallyCheckForNewConflicts(ctx, cancel, config.Rest, config.OpenAPIIgnorePrefixes) + go func() { - if err := s.RunUntil(stopCh); err != nil { + if err := s.RunUntil(ctx.Done()); err != nil { klog.ErrorS(err, "failed to run server") os.Exit(1) } diff --git a/pkg/crdcoexistence/detect.go b/pkg/crdcoexistence/detect.go new file mode 100644 index 00000000..d5e9d12a --- /dev/null +++ b/pkg/crdcoexistence/detect.go @@ -0,0 +1,64 @@ +package crdcoexistence + +import ( + "context" + "time" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" +) + +var conflictGroups = map[string]bool{ + "wgpolicyk8s.io": true, + "openreports.io": true, +} + +// DetectConflictingCRDs checks the cluster for CRDs in API groups that +// reports-server also serves via APIService. When both a CRD and an APIService +// exist for the same group/version, the kube-apiserver's OpenAPI aggregator +// produces duplicate-path errors (K8s 1.28+). For each conflicting group found, +// the corresponding OpenAPI path prefix is returned so it can be added to +// IgnorePrefixes — suppressing reports-server's OpenAPI paths for that group +// while keeping the APIService request routing fully functional. +func DetectConflictingCRDs(restConfig *rest.Config) []string { + if restConfig == nil { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + client, err := apiextensionsclient.NewForConfig(restConfig) + if err != nil { + klog.V(2).InfoS("unable to create apiextensions client for CRD conflict detection, skipping", "error", err) + return nil + } + + crdList, err := client.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{}) + if err != nil { + klog.V(2).InfoS("unable to list CRDs for conflict detection, skipping", "error", err) + return nil + } + + return detectFromCRDList(crdList.Items) +} + +func detectFromCRDList(crds []apiextensionsv1.CustomResourceDefinition) []string { + seen := make(map[string]bool) + var prefixes []string + + for _, crd := range crds { + group := crd.Spec.Group + if !conflictGroups[group] || seen[group] { + continue + } + seen[group] = true + klog.InfoS("detected third-party CRD for API group served by reports-server, suppressing OpenAPI paths to avoid duplicate-path errors", + "group", group) + prefixes = append(prefixes, "/apis/"+group+"/") + } + return prefixes +} diff --git a/pkg/crdcoexistence/detect_test.go b/pkg/crdcoexistence/detect_test.go new file mode 100644 index 00000000..a1995824 --- /dev/null +++ b/pkg/crdcoexistence/detect_test.go @@ -0,0 +1,82 @@ +package crdcoexistence + +import ( + "sort" + "testing" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestDetectConflictingCRDs_NilConfig(t *testing.T) { + if result := DetectConflictingCRDs(nil); result != nil { + t.Errorf("expected nil, got %v", result) + } +} + +func TestDetectFromCRDList(t *testing.T) { + makeCRD := func(name, group string) apiextensionsv1.CustomResourceDefinition { + return apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{Group: group}, + } + } + + tests := []struct { + name string + crds []apiextensionsv1.CustomResourceDefinition + expected []string + }{ + { + name: "no CRDs", + }, + { + name: "unrelated CRD", + crds: []apiextensionsv1.CustomResourceDefinition{makeCRD("foos.example.com", "example.com")}, + }, + { + name: "wgpolicyk8s CRD", + crds: []apiextensionsv1.CustomResourceDefinition{makeCRD("policyreports.wgpolicyk8s.io", "wgpolicyk8s.io")}, + expected: []string{"/apis/wgpolicyk8s.io/"}, + }, + { + name: "openreports CRD", + crds: []apiextensionsv1.CustomResourceDefinition{makeCRD("reports.openreports.io", "openreports.io")}, + expected: []string{"/apis/openreports.io/"}, + }, + { + name: "both groups", + crds: []apiextensionsv1.CustomResourceDefinition{ + makeCRD("policyreports.wgpolicyk8s.io", "wgpolicyk8s.io"), + makeCRD("reports.openreports.io", "openreports.io"), + }, + expected: []string{"/apis/openreports.io/", "/apis/wgpolicyk8s.io/"}, + }, + { + name: "duplicate CRDs in same group", + crds: []apiextensionsv1.CustomResourceDefinition{ + makeCRD("policyreports.wgpolicyk8s.io", "wgpolicyk8s.io"), + makeCRD("clusterpolicyreports.wgpolicyk8s.io", "wgpolicyk8s.io"), + }, + expected: []string{"/apis/wgpolicyk8s.io/"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := detectFromCRDList(tt.crds) + sort.Strings(result) + sort.Strings(tt.expected) + + if len(result) != len(tt.expected) { + t.Errorf("expected %v, got %v", tt.expected, result) + return + } + for i := range result { + if result[i] != tt.expected[i] { + t.Errorf("expected %v, got %v", tt.expected, result) + } + } + }) + } +} diff --git a/pkg/crdcoexistence/periodic.go b/pkg/crdcoexistence/periodic.go new file mode 100644 index 00000000..f772e30e --- /dev/null +++ b/pkg/crdcoexistence/periodic.go @@ -0,0 +1,38 @@ +package crdcoexistence + +import ( + "context" + "time" + + "k8s.io/client-go/rest" + "k8s.io/klog/v2" +) + +// PeriodicallyCheckForNewConflicts re-runs CRD conflict detection on a timer. +// If a new conflicting CRD appears that was not present at startup, the cancel +// function is called to trigger a graceful restart. +func PeriodicallyCheckForNewConflicts(ctx context.Context, cancel context.CancelFunc, restConfig *rest.Config, initialPrefixes []string) { + known := make(map[string]bool, len(initialPrefixes)) + for _, p := range initialPrefixes { + known[p] = true + } + + ticker := time.NewTicker(2 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + current := DetectConflictingCRDs(restConfig) + for _, p := range current { + if !known[p] { + klog.InfoS("new conflicting CRD detected since startup, restarting to apply OpenAPI suppression", "prefix", p) + cancel() + return + } + } + } + } +} diff --git a/pkg/server/config.go b/pkg/server/config.go index 9fff5175..92d90652 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/go-multierror" "github.com/kyverno/reports-server/pkg/api" "github.com/kyverno/reports-server/pkg/app/opts" + "github.com/kyverno/reports-server/pkg/crdcoexistence" "github.com/kyverno/reports-server/pkg/storage" storageapi "github.com/kyverno/reports-server/pkg/storage/api" "github.com/kyverno/reports-server/pkg/storage/db" @@ -41,6 +42,7 @@ type Config struct { SkipMigration bool APIServiceReconcileInterval time.Duration KubeClient *kubernetes.Clientset + OpenAPIIgnorePrefixes []string } func NewServerConfig(o opts.Options) (*Config, error) { @@ -88,6 +90,12 @@ func NewServerConfig(o opts.Options) (*Config, error) { return nil, err } + ignorePrefixes := crdcoexistence.DetectConflictingCRDs(restConfig) + if len(ignorePrefixes) > 0 { + apiserver.OpenAPIConfig.IgnorePrefixes = append(apiserver.OpenAPIConfig.IgnorePrefixes, ignorePrefixes...) + apiserver.OpenAPIV3Config.IgnorePrefixes = append(apiserver.OpenAPIV3Config.IgnorePrefixes, ignorePrefixes...) + } + config := &Config{ Apiserver: apiserver, Rest: restConfig, @@ -100,6 +108,7 @@ func NewServerConfig(o opts.Options) (*Config, error) { Store: store, SkipMigration: o.SkipMigration, APIServiceReconcileInterval: o.APIServiceReconcileInterval, + OpenAPIIgnorePrefixes: ignorePrefixes, } return config, nil