test(ssi): e2e tests for SSI Instrumentation Rules via Remote Config

pkg/ssi/testutils/pod.go

@@ -43,6 +43,10 @@ const (
 	InjectionStatusAnnotation = "internal.apm.datadoghq.com/injection-status"
 	// InjectedLibrariesAnnotation records the JSON array of components the webhook attempted to inject.
 	InjectedLibrariesAnnotation = "internal.apm.datadoghq.com/injected-libraries"
+	// AppliedTargetAnnotation is the JSON of the local SSI target that matched the pod.
+	AppliedTargetAnnotation = "internal.apm.datadoghq.com/applied-target"
+	// AppliedPolicyAnnotation is the compact JSON of the remote-config policy that matched the pod.
+	AppliedPolicyAnnotation = "internal.apm.datadoghq.com/applied-policy"
 )
 
 // CSIDriverStatus annotation values (kept in sync with the annotation package).
@@ -227,6 +231,27 @@ func (v *PodValidator) RequireCSIDriverStatus(t *testing.T, expected string) {
 	v.RequireAnnotations(t, map[string]string{CSIDriverStatusAnnotation: expected})
 }
 
+// RequireAppliedTargetName ensures applied-target is set and its JSON name matches expected.
+func (v *PodValidator) RequireAppliedTargetName(t *testing.T, expected string) {
+	requireAppliedJSONName(t, v.raw, AppliedTargetAnnotation, expected)
+}
+
+// RequireAppliedPolicyName ensures applied-policy is set and its JSON name matches expected.
+func (v *PodValidator) RequireAppliedPolicyName(t *testing.T, expected string) {
+	requireAppliedJSONName(t, v.raw, AppliedPolicyAnnotation, expected)
+}
+
+func requireAppliedJSONName(t *testing.T, pod *corev1.Pod, key, expected string) {
+	t.Helper()
+	raw, exists := pod.Annotations[key]
+	require.True(t, exists, "annotation %s should exist", key)
+	var payload struct {
+		Name string `json:"name"`
+	}
+	require.NoError(t, json.Unmarshal([]byte(raw), &payload), "annotation %s is not JSON: %s", key, raw)
+	require.Equal(t, expected, payload.Name, "annotation %s name", key)
+}
+
 // RequireInjectedLibraries ensures the injected-libraries annotation lists exactly the expected
 // component-name -> status pairs (e.g. {"injector":"injected","python":"injected"}). Image references
 // are intentionally not asserted as they depend on the resolved registry and digest.
@@ -263,6 +288,13 @@ func (v *PodValidator) RequireMissingVolumeNames(t *testing.T, missing []string)
 	}
 }
 
+const instrumentationInstallTypeEnvVar = "DD_INSTRUMENTATION_INSTALL_TYPE"
+
+// RequireInstallType ensures DD_INSTRUMENTATION_INSTALL_TYPE is set on the given containers.
+func (v *PodValidator) RequireInstallType(t *testing.T, expected string, expectedContainers []string) {
+	v.RequireEnvs(t, map[string]string{instrumentationInstallTypeEnvVar: expected}, expectedContainers)
+}
+
 // RequireEnvs ensures the expected env vars exist in the expected containers with the expected values.
 func (v *PodValidator) RequireEnvs(t *testing.T, expected map[string]string, expectedContainers []string) {
 	for _, name := range expectedContainers {

test/new-e2e/tests/ssi/BUILD.bazel

@@ -46,6 +46,10 @@ dd_agent_go_test(
         "testdata/injection_mode.yaml",
         "testdata/local_sdk_injection.yaml",
         "testdata/namespace_selection.yaml",
+        "testdata/rc_deny_targeted_namespace_policy.json",
+        "testdata/rc_host_linux_only_policy.json",
+        "testdata/rc_namespace_other_policy.json",
+        "testdata/rc_policies.yaml",
         "testdata/registry_allow_list.yaml",
         "testdata/workload_selection.yaml",
     ],
@@ -55,6 +59,10 @@ dd_agent_go_test(
         "testdata/injection_mode.yaml",
         "testdata/local_sdk_injection.yaml",
         "testdata/namespace_selection.yaml",
+        "testdata/rc_deny_targeted_namespace_policy.json",
+        "testdata/rc_host_linux_only_policy.json",
+        "testdata/rc_namespace_other_policy.json",
+        "testdata/rc_policies.yaml",
         "testdata/registry_allow_list.yaml",
         "testdata/workload_selection.yaml",
     ],
@@ -67,11 +75,13 @@ dd_agent_go_test(
         "//test/e2e-framework/components/kubernetes",
         "//test/e2e-framework/testing/e2e",
         "//test/e2e-framework/testing/environments",
+        "//test/fakeintake/client",
         "@com_github_pulumi_pulumi_kubernetes_sdk_v4//go/kubernetes",
         "@com_github_pulumi_pulumi_kubernetes_sdk_v4//go/kubernetes/core/v1:core",
         "@com_github_pulumi_pulumi_kubernetes_sdk_v4//go/kubernetes/meta/v1:meta",
         "@com_github_pulumi_pulumi_kubernetes_sdk_v4//go/kubernetes/rbac/v1:rbac",
         "@com_github_pulumi_pulumi_sdk_v3//go/pulumi",
         "@com_github_stretchr_testify//require",
+        "@io_k8s_client_go//kubernetes",
     ],
 )

test/new-e2e/tests/ssi/provisioner.go

@@ -6,6 +6,7 @@
 package ssi
 
 import (
+	"encoding/json"
 	"strings"
 
 	"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
@@ -131,9 +132,34 @@ func kindLocalProvisioner(opts ProvisionerOptions) provisioners.TypedProvisioner
 	if opts.AgentDependentWorkloadAppFunc != nil {
 		localOpts = append(localOpts, provlocal.WithAgentDependentWorkloadApp(opts.AgentDependentWorkloadAppFunc))
 	}
+	for _, image := range localAgentImagesFromStackParams() {
+		localOpts = append(localOpts, provlocal.WithKindLoadImage(image))
+	}
 	return provlocal.Provisioner(localOpts...)
 }
 
+func localAgentImagesFromStackParams() []string {
+	raw, err := runner.GetProfile().ParamStore().GetWithDefault(parameters.StackParameters, "")
+	if err != nil || raw == "" {
+		return nil
+	}
+	var params map[string]string
+	if err := json.Unmarshal([]byte(raw), &params); err != nil {
+		return nil
+	}
+
+	var images []string
+	for _, key := range []string{
+		"ddagent:" + config.DDAgentFullImagePathParamName,
+		"ddagent:" + config.DDClusterAgentFullImagePathParamName,
+	} {
+		if image := params[key]; image != "" {
+			images = append(images, image)
+		}
+	}
+	return images
+}
+
 // kindProvisioner returns an AWS Kind VM provisioner
 func kindProvisioner(opts ProvisionerOptions) provisioners.TypedProvisioner[environments.Kubernetes] {
 	runOpts := []kindvm.RunOption{

test/new-e2e/tests/ssi/ssi_test.go

@@ -11,6 +11,7 @@ package ssi
 
 import (
 	_ "embed"
+	"fmt"
 	"testing"
 	"time"
 
@@ -20,6 +21,7 @@ import (
 	rbacv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/rbac/v1"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 	"github.com/stretchr/testify/require"
+	kubeClient "k8s.io/client-go/kubernetes"
 
 	"github.com/DataDog/datadog-agent/pkg/ssi/testutils"
 	"github.com/DataDog/datadog-agent/pkg/util/testutil/flake"
@@ -29,6 +31,7 @@ import (
 	compkube "github.com/DataDog/datadog-agent/test/e2e-framework/components/kubernetes"
 	"github.com/DataDog/datadog-agent/test/e2e-framework/testing/e2e"
 	"github.com/DataDog/datadog-agent/test/e2e-framework/testing/environments"
+	fakeintake "github.com/DataDog/datadog-agent/test/fakeintake/client"
 )
 
 //go:embed testdata/base.yaml
@@ -49,6 +52,36 @@ var workloadSelectionHelmValues string
 //go:embed testdata/registry_allow_list.yaml
 var registryAllowListHelmValues string
 
+//go:embed testdata/rc_policies.yaml
+var rcPoliciesHelmValues string
+
+//go:embed testdata/rc_host_linux_only_policy.json
+var rcHostLinuxOnlyPolicyJSON []byte
+
+//go:embed testdata/rc_namespace_other_policy.json
+var rcNamespaceOtherPolicyJSON []byte
+
+//go:embed testdata/rc_deny_targeted_namespace_policy.json
+var rcDenyTargetedNamespacePolicyJSON []byte
+
+const (
+	apmPoliciesRCProduct              = "APM_POLICIES"
+	rcHostLinuxOnlyConfigID           = "1.host-linux-only"
+	rcHostLinuxOnlyConfigName         = "config"
+	rcNamespaceOtherConfigID          = "1.namespace-other"
+	rcNamespaceOtherConfigName        = "config"
+	rcDenyTargetedNamespaceConfigID   = "1.deny-targeted-namespace"
+	rcDenyTargetedNamespaceConfigName = "config"
+	rcFakeIntakeDefaultOrgID          = "42"
+	rcHelmTargetNamespace             = "targeted-namespace"
+	rcHelmTargetApp                   = "rc-target-python"
+	rcOtherNamespace                  = "other"
+	rcAnnotatedPodApp                 = "rc-ann-only"
+	rcUnannotatedPodApp               = "rc-unannotated"
+	rcHelmTargetName                  = "python-apps"
+	rcNamespaceOtherPolicyName        = "namespace other: matches admission namespace fact"
+)
+
 // ssiSuite runs all SSI test groups on a single cluster, calling UpdateEnv at the start of
 // each group to update the env (workloads, helm values).
 type ssiSuite struct {
@@ -547,6 +580,177 @@ func (v *ssiSuite) TestRegistryAllowList() {
 	})
 }
 
+func (v *ssiSuite) TestRemoteConfig() {
+	// rc_policies.yaml: helm SSI target (namespace injection=yes, pod language=python) plus two
+	// workloads in namespace "other", outside that target: annotated lib-injection and unannotated.
+	// RC uptake is asserted by pod mutation. RestartUntil re-admits until the
+	// expected outcome appears so we do not race the cluster-agent RC client.
+	agentOptions := []kubernetesagentparams.Option{
+		kubernetesagentparams.WithHelmValues(rcPoliciesHelmValues),
+		kubernetesagentparams.WithTimeout(600),
+	}
+	provisionerOpts := ProvisionerOptions{
+		AgentOptions: agentOptions,
+		AgentDependentWorkloadAppFunc: func(e config.Env, kubeProvider *kubernetes.Provider, dependsOnAgent pulumi.ResourceOption) (*compkube.Workload, error) {
+			return singlestep.Scenario(e, kubeProvider, "rc-policies", []singlestep.Namespace{
+				{
+					Name: rcHelmTargetNamespace,
+					Labels: map[string]string{
+						"injection": "yes",
+					},
+					Apps: []singlestep.App{
+						{
+							Name:    rcHelmTargetApp,
+							Image:   "gcr.io/datadoghq/injector-dev/python",
+							Version: "d425e7df",
+							Port:    8080,
+							PodLabels: map[string]string{
+								"language": "python",
+							},
+						},
+					},
+				},
+				{
+					Name: rcOtherNamespace,
+					Apps: []singlestep.App{
+						{
+							Name:    rcAnnotatedPodApp,
+							Image:   "gcr.io/datadoghq/injector-dev/python",
+							Version: "d425e7df",
+							Port:    8080,
+							PodLabels: map[string]string{
+								"admission.datadoghq.com/enabled": "true",
+							},
+							PodAnnotations: map[string]string{
+								"admission.datadoghq.com/python-lib.version": "v3.18.1",
+							},
+						},
+						{
+							Name:    rcUnannotatedPodApp,
+							Image:   "gcr.io/datadoghq/injector-dev/python",
+							Version: "d425e7df",
+							Port:    8080,
+						},
+					},
+				},
+			}, dependsOnAgent)
+		},
+	}
+
+	v.UpdateEnv(Provisioner(provisionerOpts))
+
+	fi := v.Env().FakeIntake.Client()
+
+	// Host-only RC cannot match K8s admission facts. A matching policy is published first so
+	// the annotated pod leaves the helm baseline; replacing that same RC document with the
+	// host-only payload must restore lib-injection. Delete-then-add would restore the
+	// baseline as soon as the matching policy is gone, before host-only is applied.
+	v.Run("HostOnlyPolicyDoesNotMatchK8s", func() {
+		k8s := v.Env().KubernetesCluster.Client()
+
+		cleanup := v.pushAPMPolicy(fi, rcHostLinuxOnlyConfigID, rcHostLinuxOnlyConfigName, rcNamespaceOtherPolicyJSON)
+		defer cleanup()
+
+		RestartUntil(v.T(), k8s, rcOtherNamespace, rcAnnotatedPodApp, hasInstallType(rcAnnotatedPodApp, "k8s_single_step"))
+
+		_ = v.pushAPMPolicy(fi, rcHostLinuxOnlyConfigID, rcHostLinuxOnlyConfigName, rcHostLinuxOnlyPolicyJSON)
+		pod := RestartUntil(v.T(), k8s, rcOtherNamespace, rcAnnotatedPodApp, hasInstallType(rcAnnotatedPodApp, "k8s_lib_injection"))
+
+		podValidator := testutils.NewPodValidator(pod, testutils.InjectionModeAuto)
+		podValidator.RequireInjection(v.T(), []string{rcAnnotatedPodApp})
+		podValidator.RequireInstallType(v.T(), "k8s_lib_injection", []string{rcAnnotatedPodApp})
+		podValidator.RequireMissingEnvs(v.T(), []string{"DD_TRACE_ENABLED"}, []string{rcAnnotatedPodApp})
+		// Library annotations short-circuit applied-target / applied-policy metadata.
+		podValidator.RequireMissingAnnotations(v.T(), []string{testutils.AppliedTargetAnnotation, testutils.AppliedPolicyAnnotation})
+
+		RestartPod(v.T(), k8s, rcOtherNamespace, rcUnannotatedPodApp)
+		unannotated := FindPodInNamespace(v.T(), k8s, rcOtherNamespace, rcUnannotatedPodApp)
+		unannotatedValidator := testutils.NewPodValidator(unannotated, testutils.InjectionModeAuto)
+		unannotatedValidator.RequireNoInjection(v.T())
+		unannotatedValidator.RequireMissingAnnotations(v.T(), []string{testutils.AppliedTargetAnnotation, testutils.AppliedPolicyAnnotation})
+
+		v.requireHelmTargetStillSSI(k8s)
+	})
+
+	// RC policy matching namespace "other" enables SSI on the annotated pod and on the unannotated
+	// pod (true on-demand, still outside the helm target). Helm local targeting is unchanged.
+	v.Run("NamespacePolicyEnablesSSIOutsideHelmTarget", func() {
+		k8s := v.Env().KubernetesCluster.Client()
+
+		cleanup := v.pushAPMPolicy(fi, rcNamespaceOtherConfigID, rcNamespaceOtherConfigName, rcNamespaceOtherPolicyJSON)
+		defer cleanup()
+
+		pod := RestartUntil(v.T(), k8s, rcOtherNamespace, rcAnnotatedPodApp, hasInstallType(rcAnnotatedPodApp, "k8s_single_step"))
+
+		podValidator := testutils.NewPodValidator(pod, testutils.InjectionModeAuto)
+		podValidator.RequireInjection(v.T(), []string{rcAnnotatedPodApp})
+		podValidator.RequireInstallType(v.T(), "k8s_single_step", []string{rcAnnotatedPodApp})
+		podValidator.RequireEnvs(v.T(), map[string]string{"DD_TRACE_ENABLED": "true"}, []string{rcAnnotatedPodApp})
+		// SSI mode follows the policy match; annotation short-circuit still skips applied-policy JSON.
+		podValidator.RequireMissingAnnotations(v.T(), []string{testutils.AppliedTargetAnnotation, testutils.AppliedPolicyAnnotation})
+
+		RestartPod(v.T(), k8s, rcOtherNamespace, rcUnannotatedPodApp)
+		unannotated := WaitForMutatedPodInNamespace(v.T(), k8s, rcOtherNamespace, rcUnannotatedPodApp)
+		unannotatedValidator := testutils.NewPodValidator(unannotated, testutils.InjectionModeAuto)
+		unannotatedValidator.RequireInjection(v.T(), []string{rcUnannotatedPodApp})
+		unannotatedValidator.RequireInstallType(v.T(), "k8s_single_step", []string{rcUnannotatedPodApp})
+		unannotatedValidator.RequireEnvs(v.T(), map[string]string{"DD_TRACE_ENABLED": "true"}, []string{rcUnannotatedPodApp})
+		unannotatedValidator.RequireAppliedPolicyName(v.T(), rcNamespaceOtherPolicyName)
+		unannotatedValidator.RequireMissingAnnotations(v.T(), []string{testutils.AppliedTargetAnnotation})
+
+		v.requireHelmTargetStillSSI(k8s)
+	})
+
+	// Remote policies are evaluated before local helm targets. A deny matching
+	// targeted-namespace blocks SSI on the helm workload; namespace "other" is unchanged.
+	v.Run("NamespacePolicyOverridesHelmTarget", func() {
+		k8s := v.Env().KubernetesCluster.Client()
+
+		cleanup := v.pushAPMPolicy(fi, rcDenyTargetedNamespaceConfigID, rcDenyTargetedNamespaceConfigName, rcDenyTargetedNamespacePolicyJSON)
+		defer cleanup()
+
+		helmPod := RestartUntil(v.T(), k8s, rcHelmTargetNamespace, rcHelmTargetApp, noInjection(rcHelmTargetApp))
+		helmValidator := testutils.NewPodValidator(helmPod, testutils.InjectionModeAuto)
+		helmValidator.RequireNoInjection(v.T())
+		helmValidator.RequireMissingAnnotations(v.T(), []string{testutils.AppliedTargetAnnotation, testutils.AppliedPolicyAnnotation})
+
+		RestartPod(v.T(), k8s, rcOtherNamespace, rcAnnotatedPodApp)
+		annotated := WaitForMutatedPodInNamespace(v.T(), k8s, rcOtherNamespace, rcAnnotatedPodApp)
+		annotatedValidator := testutils.NewPodValidator(annotated, testutils.InjectionModeAuto)
+		annotatedValidator.RequireInjection(v.T(), []string{rcAnnotatedPodApp})
+		annotatedValidator.RequireInstallType(v.T(), "k8s_lib_injection", []string{rcAnnotatedPodApp})
+		annotatedValidator.RequireMissingEnvs(v.T(), []string{"DD_TRACE_ENABLED"}, []string{rcAnnotatedPodApp})
+		annotatedValidator.RequireMissingAnnotations(v.T(), []string{testutils.AppliedTargetAnnotation, testutils.AppliedPolicyAnnotation})
+
+		RestartPod(v.T(), k8s, rcOtherNamespace, rcUnannotatedPodApp)
+		unannotated := FindPodInNamespace(v.T(), k8s, rcOtherNamespace, rcUnannotatedPodApp)
+		unannotatedValidator := testutils.NewPodValidator(unannotated, testutils.InjectionModeAuto)
+		unannotatedValidator.RequireNoInjection(v.T())
+		unannotatedValidator.RequireMissingAnnotations(v.T(), []string{testutils.AppliedTargetAnnotation, testutils.AppliedPolicyAnnotation})
+	})
+}
+
+func (v *ssiSuite) requireHelmTargetStillSSI(k8s kubeClient.Interface) {
+	v.T().Helper()
+
+	RestartPod(v.T(), k8s, rcHelmTargetNamespace, rcHelmTargetApp)
+	pod := WaitForMutatedPodInNamespace(v.T(), k8s, rcHelmTargetNamespace, rcHelmTargetApp)
+	podValidator := testutils.NewPodValidator(pod, testutils.InjectionModeAuto)
+	podValidator.RequireInjection(v.T(), []string{rcHelmTargetApp})
+	podValidator.RequireInstallType(v.T(), "k8s_single_step", []string{rcHelmTargetApp})
+	podValidator.RequireLibraryVersions(v.T(), map[string]string{"python": "v3.18.1"})
+	podValidator.RequireAppliedTargetName(v.T(), rcHelmTargetName)
+	podValidator.RequireMissingAnnotations(v.T(), []string{testutils.AppliedPolicyAnnotation})
+}
+
+func (v *ssiSuite) pushAPMPolicy(fi *fakeintake.Client, configID, configName string, payload []byte) func() {
+	v.T().Helper()
+	require.NoError(v.T(), fi.RCAddConfig("", apmPoliciesRCProduct, configID, configName, payload))
+	return func() {
+		require.NoError(v.T(), fi.RCDeleteConfig(fmt.Sprintf("%s/%s/%s/%s", rcFakeIntakeDefaultOrgID, apmPoliciesRCProduct, configID, configName)))
+	}
+}
+
 func isOpenShift() bool {
 	switch getProvisionerType() {
 	case ProvisionerOpenShift, ProvisionerOpenShiftLocal:

test/new-e2e/tests/ssi/testdata/base.yaml

@@ -1,9 +1,15 @@
 ---
+agents:
+  enabled: false
+clusterChecksRunner:
+  enabled: false
 clusterAgent:
   admissionController:
     configMode: "hostip"
 
 datadog:
+  kubeStateMetricsCore:
+    useClusterCheckRunners: false
   csi:
     # Required: otherwise at the end of TestInjectionMode the CSI driver is removed before
     # the apps that use it, so those apps fail to terminate.

test/new-e2e/tests/ssi/testdata/injection_mode.yaml

@@ -1,4 +1,6 @@
 ---
+agents:
+  enabled: true
 clusterAgent:
   admissionController:
     configMode: "hostip"