diff --git a/CHANGELOG.md b/CHANGELOG.md index 69ecaf9f97d..57afd4d7be1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,7 +76,9 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio - **General**: Allow excluding labels from being propagated from ScaledObject and ScaledJob to generated HPA and Job objects ([#6849](https://github.com/kedacore/keda/issues/6849)) - **General**: Improve Events emitted from ScaledObject controller ([#6802](https://github.com/kedacore/keda/issues/6802)) +- **General**: Support token store in secret and fallback to current token string ([#6026](https://github.com/kedacore/keda/issues/6026)) - **Datadog Scaler**: Add a specific timeout configuration parameter for the Datadog trigger ([#6999](https://github.com/kedacore/keda/pull/6999)) +- **Datadog Scaler**: Fix bug with datadogNamespace config ([#6828](https://github.com/kedacore/keda/pull/6828)) - **Datadog Scaler**: Improve Datadog scaler error messages ([#6999](https://github.com/kedacore/keda/pull/6999)) - **Metrics API**: Support multiple auth methods simultaneously in Metrics API scaler ([#6642](https://github.com/kedacore/keda/issues/6642)) - **Temporal Scaler**: Support custom tlsServerName ([#6820](https://github.com/kedacore/keda/pull/6820)) diff --git a/apis/keda/v1alpha1/triggerauthentication_types.go b/apis/keda/v1alpha1/triggerauthentication_types.go index 3c9154d21ab..9025266d509 100644 --- a/apis/keda/v1alpha1/triggerauthentication_types.go +++ b/apis/keda/v1alpha1/triggerauthentication_types.go @@ -235,8 +235,12 @@ type HashiCorpVault struct { // Credential defines the Hashicorp Vault credentials depending on the authentication method type Credential struct { // +optional + // +kubebuilder:deprecated:warning="field `token` is deprecated and might be removed in further versions, use `tokenSecretRef` field instead" Token string `json:"token,omitempty"` + // +optional + TokenSecretRef *SecretKeyRef `json:"tokenSecretRef,omitempty"` + // +optional ServiceAccount string `json:"serviceAccount,omitempty"` } diff --git a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml index bbd2e34b920..89614a45213 100644 --- a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml +++ b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml @@ -453,6 +453,16 @@ spec: type: string token: type: string + tokenSecretRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object type: object mount: type: string diff --git a/config/crd/bases/keda.sh_triggerauthentications.yaml b/config/crd/bases/keda.sh_triggerauthentications.yaml index c81f78a4d79..fa8d812bb87 100644 --- a/config/crd/bases/keda.sh_triggerauthentications.yaml +++ b/config/crd/bases/keda.sh_triggerauthentications.yaml @@ -452,6 +452,16 @@ spec: type: string token: type: string + tokenSecretRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object type: object mount: type: string diff --git a/pkg/scaling/resolver/hashicorpvault_handler.go b/pkg/scaling/resolver/hashicorpvault_handler.go index e80b7302cda..3e00a5c1559 100644 --- a/pkg/scaling/resolver/hashicorpvault_handler.go +++ b/pkg/scaling/resolver/hashicorpvault_handler.go @@ -17,6 +17,7 @@ limitations under the License. package resolver import ( + "context" "encoding/json" "errors" "fmt" @@ -25,26 +26,36 @@ import ( "github.com/go-logr/logr" vaultapi "github.com/hashicorp/vault/api" + corev1listers "k8s.io/client-go/listers/core/v1" + k8sClient "sigs.k8s.io/controller-runtime/pkg/client" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" ) // HashicorpVaultHandler is specification of Hashi Corp Vault type HashicorpVaultHandler struct { - vault *kedav1alpha1.HashiCorpVault - client *vaultapi.Client - stopCh chan struct{} + vault *kedav1alpha1.HashiCorpVault + client *vaultapi.Client + k8sClient k8sClient.Client + stopCh chan struct{} + ctx context.Context + logger *logr.Logger + secretsLister corev1listers.SecretLister } // NewHashicorpVaultHandler creates a HashicorpVaultHandler object -func NewHashicorpVaultHandler(v *kedav1alpha1.HashiCorpVault) *HashicorpVaultHandler { +func NewHashicorpVaultHandler(ctx context.Context, client k8sClient.Client, v *kedav1alpha1.HashiCorpVault, logger *logr.Logger, secretListener corev1listers.SecretLister) *HashicorpVaultHandler { return &HashicorpVaultHandler{ - vault: v, + vault: v, + ctx: ctx, + logger: logger, + secretsLister: secretListener, + k8sClient: client, } } // Initialize the Vault client -func (vh *HashicorpVaultHandler) Initialize(logger logr.Logger) error { +func (vh *HashicorpVaultHandler) Initialize() error { config := vaultapi.DefaultConfig() client, err := vaultapi.NewClient(config) if err != nil { @@ -77,7 +88,7 @@ func (vh *HashicorpVaultHandler) Initialize(logger logr.Logger) error { if renew, ok := lookup.Data["renewable"].(bool); ok && renew { vh.stopCh = make(chan struct{}) - go vh.renewToken(logger) + go vh.renewToken() } vh.client = client @@ -92,14 +103,24 @@ func (vh *HashicorpVaultHandler) token(client *vaultapi.Client) (string, error) switch vh.vault.Authentication { case kedav1alpha1.VaultAuthenticationToken: // Got token from VAULT_TOKEN env variable - switch { - case len(client.Token()) > 0: - break - case len(vh.vault.Credential.Token) > 0: - token = vh.vault.Credential.Token - default: - return token, errors.New("could not get Vault token") + if len(client.Token()) > 0 { + return "", nil } + + // consume token from k8s secret + if vh.vault.Credential.TokenSecretRef != nil && vh.vault.Credential.TokenSecretRef.Name != "" && vh.vault.Credential.TokenSecretRef.Key != "" { + token = resolveAuthSecret(vh.ctx, vh.k8sClient, *vh.logger, vh.vault.Credential.TokenSecretRef.Name, vh.vault.Namespace, vh.vault.Credential.TokenSecretRef.Key, vh.secretsLister) + if token != "" { + return token, nil + } + } + + // if the token is not set in the secret will fallback to the previews approach + if len(vh.vault.Credential.Token) > 0 { + return vh.vault.Credential.Token, nil + } + return token, errors.New("could not get Vault token") + case kedav1alpha1.VaultAuthenticationKubernetes: if len(vh.vault.Mount) == 0 { return token, errors.New("auth mount not in config") @@ -141,10 +162,10 @@ func (vh *HashicorpVaultHandler) token(client *vaultapi.Client) (string, error) } // renewToken takes charge of renewing the vault token -func (vh *HashicorpVaultHandler) renewToken(logger logr.Logger) { +func (vh *HashicorpVaultHandler) renewToken() { secret, err := vh.client.Auth().Token().RenewSelf(0) if err != nil { - logger.Error(err, "Vault renew token: failed to create the payload") + vh.logger.Error(err, "Vault renew token: failed to create the payload") } renewer, err := vh.client.NewLifetimeWatcher(&vaultapi.RenewerInput{ @@ -153,7 +174,7 @@ func (vh *HashicorpVaultHandler) renewToken(logger logr.Logger) { //Increment: 60, }) if err != nil { - logger.Error(err, "Vault renew token: cannot create the renewer") + vh.logger.Error(err, "Vault renew token: cannot create the renewer") } go renewer.Renew() @@ -169,7 +190,7 @@ RenewWatcherLoop: break RenewWatcherLoop case err := <-renewer.DoneCh(): if err != nil { - logger.Error(err, "error renewing token") + vh.logger.Error(err, "error renewing token") } break RenewWatcherLoop } diff --git a/pkg/scaling/resolver/hashicorpvault_handler_test.go b/pkg/scaling/resolver/hashicorpvault_handler_test.go index 5051faeb332..cb167391181 100644 --- a/pkg/scaling/resolver/hashicorpvault_handler_test.go +++ b/pkg/scaling/resolver/hashicorpvault_handler_test.go @@ -17,6 +17,7 @@ limitations under the License. package resolver import ( + "context" "encoding/base64" "encoding/json" "fmt" @@ -27,7 +28,14 @@ import ( vaultapi "github.com/hashicorp/vault/api" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + corev1listers "k8s.io/client-go/listers/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" ) @@ -93,7 +101,7 @@ var pkiRequestTestDataset = []pkiRequestTestData{ } func TestGetPkiRequest(t *testing.T) { - vault := NewHashicorpVaultHandler(nil) + vault := NewHashicorpVaultHandler(context.TODO(), nil, nil, nil, nil) for _, testData := range pkiRequestTestDataset { var secret kedav1alpha1.VaultSecret @@ -181,8 +189,9 @@ func TestHashicorpVaultHandler_getSecretValue_specify_secret_type(t *testing.T) Token: vaultTestToken, }, } - vaultHandler := NewHashicorpVaultHandler(&vault) - err := vaultHandler.Initialize(logf.Log.WithName("test")) + l := logf.Log.WithName("test") + vaultHandler := NewHashicorpVaultHandler(context.TODO(), nil, &vault, &l, nil) + err := vaultHandler.Initialize() defer vaultHandler.Stop() assert.Nil(t, err) secrets := []kedav1alpha1.VaultSecret{{ @@ -321,8 +330,9 @@ func TestHashicorpVaultHandler_ResolveSecret(t *testing.T) { Token: vaultTestToken, }, } - vaultHandler := NewHashicorpVaultHandler(&vault) - err := vaultHandler.Initialize(logf.Log.WithName("test")) + l := logf.Log.WithName("test") + vaultHandler := NewHashicorpVaultHandler(context.TODO(), nil, &vault, &l, nil) + err := vaultHandler.Initialize() defer vaultHandler.Stop() assert.Nil(t, err) @@ -357,8 +367,9 @@ func TestHashicorpVaultHandler_ResolveSecret_UsingRootToken(t *testing.T) { Token: vaultTestToken, }, } - vaultHandler := NewHashicorpVaultHandler(&vault) - err := vaultHandler.Initialize(logf.Log.WithName("test")) + l := logf.Log.WithName("test") + vaultHandler := NewHashicorpVaultHandler(context.TODO(), nil, &vault, &l, nil) + err := vaultHandler.Initialize() defer vaultHandler.Stop() assert.Nil(t, err) @@ -394,8 +405,9 @@ func TestHashicorpVaultHandler_DefaultKubernetesVaultRole(t *testing.T) { Role: "my-role", } - vaultHandler := NewHashicorpVaultHandler(&vault) - err := vaultHandler.Initialize(logf.Log.WithName("test")) + l := logf.Log.WithName("test") + vaultHandler := NewHashicorpVaultHandler(context.TODO(), nil, &vault, &l, nil) + err := vaultHandler.Initialize() defer vaultHandler.Stop() assert.Errorf(t, err, "open %s : no such file or directory", defaultServiceAccountPath) assert.Equal(t, vaultHandler.vault.Credential.ServiceAccount, defaultServiceAccountPath) @@ -412,8 +424,9 @@ func TestHashicorpVaultHandler_ResolveSecrets_SameCertAndKey(t *testing.T) { Token: vaultTestToken, }, } - vaultHandler := NewHashicorpVaultHandler(&vault) - err := vaultHandler.Initialize(logf.Log.WithName("test")) + l := logf.Log.WithName("test") + vaultHandler := NewHashicorpVaultHandler(context.TODO(), nil, &vault, &l, nil) + err := vaultHandler.Initialize() defer vaultHandler.Stop() assert.Nil(t, err) secrets := []kedav1alpha1.VaultSecret{{ @@ -480,8 +493,9 @@ func TestHashicorpVaultHandler_fetchSecret(t *testing.T) { Token: vaultTestToken, }, } - vaultHandler := NewHashicorpVaultHandler(&vault) - err := vaultHandler.Initialize(logf.Log.WithName("test")) + l := logf.Log.WithName("test") + vaultHandler := NewHashicorpVaultHandler(context.TODO(), nil, &vault, &l, nil) + err := vaultHandler.Initialize() defer vaultHandler.Stop() assert.Nil(t, err) @@ -501,30 +515,98 @@ func TestHashicorpVaultHandler_fetchSecret(t *testing.T) { } type initializeTestData struct { - name string - namespace string - token string - isError bool + name string + namespace string + token string + expectedToken string + isError bool + secretKey string + secretName string + existing []runtime.Object } var initialiseTestDataSet = []initializeTestData{ { - name: "Namespace and Token", - namespace: "testNamespace", - token: "testToken", - isError: false, + name: "Namespace and Token", + namespace: "testNamespace", + token: "testToken", + expectedToken: "testToken", + isError: false, + }, + { + name: "No Namespace", + namespace: "", + token: "testToken", + expectedToken: "testToken", + isError: false, + }, + { + name: "Token in Secret Name", + namespace: "testNamespace", + token: "testToken", + expectedToken: secretData, + isError: false, + secretKey: secretKey, + secretName: secretName, + existing: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "testNamespace", + Name: secretName, + }, + Data: map[string][]byte{ + secretKey: []byte(secretData), + }, + }, + }, + }, + { + name: "Token in Secret Not Found, fallback", + namespace: "testNamespace", + token: "testToken", + expectedToken: "testToken", + isError: false, + secretKey: "otherSecretKey", + secretName: secretName, + existing: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: secretName, + }, + Data: map[string][]byte{ + secretKey: []byte(secretData), + }, + }, + }, }, { - name: "No Namespace", - namespace: "", - token: "testToken", - isError: false, + name: "Token in Secret Not Found, fallback", + namespace: "testNamespace", + token: "", + expectedToken: "testToken", + isError: true, + secretKey: "otherSecretKey", + secretName: secretName, + existing: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: secretName, + }, + Data: map[string][]byte{ + secretKey: []byte(secretData), + }, + }, + }, }, } func TestHashicorpVaultHandler_Initialize(t *testing.T) { server := mockVault(t, false) defer server.Close() + logf.SetLogger(zap.New(zap.UseDevMode(true))) + var secretsLister corev1listers.SecretLister for _, testData := range initialiseTestDataSet { func() { @@ -532,20 +614,25 @@ func TestHashicorpVaultHandler_Initialize(t *testing.T) { Address: server.URL, Authentication: kedav1alpha1.VaultAuthenticationToken, Credential: &kedav1alpha1.Credential{ + TokenSecretRef: &kedav1alpha1.SecretKeyRef{ + Name: testData.secretName, + Key: testData.secretKey, + }, Token: testData.token, }, Namespace: testData.namespace, } - vaultHandler := NewHashicorpVaultHandler(&vault) - err := vaultHandler.Initialize(logf.Log.WithName("test")) + l := logf.Log.WithName("test") + vaultHandler := NewHashicorpVaultHandler(context.Background(), fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(testData.existing...).Build(), &vault, &l, secretsLister) + err := vaultHandler.Initialize() defer vaultHandler.Stop() - assert.Nil(t, err) if testData.isError { assert.NotNilf(t, err, "test %s: expected error but got success, testData - %+v", testData.name, testData) } else { + assert.Nil(t, err) assert.Equalf(t, vaultHandler.client.Address(), server.URL, "test case %s", testData.name) - assert.Equalf(t, vaultHandler.client.Token(), testData.token, "test case %s", testData.name) + assert.Equalf(t, vaultHandler.client.Token(), testData.expectedToken, "test case %s", testData.name) assert.Equalf(t, vaultHandler.client.Namespace(), testData.namespace, "test case %s", testData.name) } }() @@ -610,7 +697,8 @@ func TestHashicorpVaultHandler_Token_VaultTokenAuth(t *testing.T) { Role: testData.role, Mount: testData.mount, } - vaultHandler := NewHashicorpVaultHandler(&vault) + l := logf.Log.WithName("test") + vaultHandler := NewHashicorpVaultHandler(context.TODO(), nil, &vault, &l, nil) defer vaultHandler.Stop() config := vaultapi.DefaultConfig() diff --git a/pkg/scaling/resolver/scale_resolvers.go b/pkg/scaling/resolver/scale_resolvers.go index 6e772d951e4..e8af532a633 100644 --- a/pkg/scaling/resolver/scale_resolvers.go +++ b/pkg/scaling/resolver/scale_resolvers.go @@ -274,8 +274,8 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge } } if triggerAuthSpec.HashiCorpVault != nil && len(triggerAuthSpec.HashiCorpVault.Secrets) > 0 { - vault := NewHashicorpVaultHandler(triggerAuthSpec.HashiCorpVault) - err := vault.Initialize(logger) + vault := NewHashicorpVaultHandler(ctx, client, triggerAuthSpec.HashiCorpVault, &logger, authClientSet.SecretLister) + err = vault.Initialize() defer vault.Stop() if err != nil { logger.Error(err, "error authenticating to Vault", "triggerAuthRef.Name", triggerAuthRef.Name) diff --git a/tests/secret-providers/hashicorp_vault/hashicorp_vault_test.go b/tests/secret-providers/hashicorp_vault/hashicorp_vault_test.go index cd135e50d4a..85d8b8c4ec4 100644 --- a/tests/secret-providers/hashicorp_vault/hashicorp_vault_test.go +++ b/tests/secret-providers/hashicorp_vault/hashicorp_vault_test.go @@ -46,6 +46,8 @@ var ( prometheusServerName = fmt.Sprintf("%s-prom-server", testName) minReplicaCount = 0 maxReplicaCount = 1 + HashiCorpSecretName = "hashicorp-secret-name" + HashiCorpSecretKey = "hashicorp-secret-key" ) type templateData struct { @@ -59,6 +61,8 @@ type templateData struct { SecretName string HashiCorpAuthentication string HashiCorpToken string + HashiCorpSecretName string + HashiCorpSecretKey string PostgreSQLStatefulSetName string PostgreSQLConnectionStringBase64 string PostgreSQLUsername string @@ -119,6 +123,17 @@ data: postgresql_conn_str: {{.PostgreSQLConnectionStringBase64}} ` + secretHashicorpTokenTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: {{.HashiCorpSecretName}} + namespace: {{.TestNamespace}} +type: Opaque +data: + {{.HashiCorpSecretKey}}: {{.HashiCorpToken}} +` + triggerAuthenticationTemplate = ` apiVersion: keda.sh/v1alpha1 kind: TriggerAuthentication @@ -136,6 +151,25 @@ spec: key: connectionString path: {{.VaultSecretPath}} ` + triggerAuthenticationWithSecretTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthenticationName}} + namespace: {{.TestNamespace}} +spec: + hashiCorpVault: + address: http://vault.{{.VaultNamespace}}:8200 + authentication: token + credential: + tokenSecretRef: + name: {{.HashiCorpSecretName}} + key: {{.HashiCorpSecretKey}} + secrets: + - parameter: connection + key: connectionString + path: {{.VaultSecretPath}} +` scaledObjectTemplate = ` apiVersion: keda.sh/v1alpha1 @@ -460,16 +494,25 @@ func TestSecretsEngine(t *testing.T) { name string vaultEngineVersion uint vaultSecretPath string + useSecrets bool }{ { name: "vault kv engine v1", vaultEngineVersion: 1, vaultSecretPath: "secret/keda", + useSecrets: false, }, { name: "vault kv engine v2", vaultEngineVersion: 2, vaultSecretPath: "secret/data/keda", + useSecrets: false, + }, + { + name: "vault kv engine v1", + vaultEngineVersion: 1, + vaultSecretPath: "secret/keda", + useSecrets: true, }, } @@ -477,9 +520,9 @@ func TestSecretsEngine(t *testing.T) { t.Run(test.name, func(t *testing.T) { // Create kubernetes resources for PostgreSQL server kc := GetKubernetesClient(t) - data, postgreSQLtemplates := getPostgreSQLTemplateData() + testData, postgreSQLtemplates := getPostgreSQLTemplateData() - CreateKubernetesResources(t, kc, testNamespace, data, postgreSQLtemplates) + CreateKubernetesResources(t, kc, testNamespace, testData, postgreSQLtemplates) hashiCorpToken, _ := setupHashiCorpVault(t, kc, test.vaultEngineVersion, false, false) assert.True(t, WaitForStatefulsetReplicaReadyCount(t, kc, postgreSQLStatefulSetName, testNamespace, 1, 60, 3), @@ -490,22 +533,27 @@ func TestSecretsEngine(t *testing.T) { ok, out, errOut, err := WaitForSuccessfulExecCommandOnSpecificPod(t, postgresqlPodName, testNamespace, psqlCreateTableCmd, 60, 3) assert.True(t, ok, "executing a command on PostreSQL Pod should work; Output: %s, ErrorOutput: %s, Error: %s", out, errOut, err) - - // Create kubernetes resources for testing - data, templates := getTemplateData() - data.HashiCorpToken = RemoveANSI(hashiCorpToken) - data.VaultSecretPath = test.vaultSecretPath - - KubectlApplyMultipleWithTemplate(t, data, templates) + var templates []Template + if test.useSecrets { + // Create kubernetes resources for testing using secrets to get the token + testData, templates = getTemplateDataWithTokenSecret() + } else { + // Create kubernetes resources for testing + testData, templates = getTemplateData() + } + testData.HashiCorpToken = RemoveANSI(hashiCorpToken) + testData.VaultSecretPath = test.vaultSecretPath + + KubectlApplyMultipleWithTemplate(t, testData, templates) assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3), "replica count should be %d after 3 minutes", minReplicaCount) - testScaleOut(t, kc, data) + testScaleOut(t, kc, testData) // cleanup - KubectlDeleteMultipleWithTemplate(t, data, templates) + KubectlDeleteMultipleWithTemplate(t, testData, templates) cleanupHashiCorpVault(t) - DeleteKubernetesResources(t, testNamespace, data, postgreSQLtemplates) + DeleteKubernetesResources(t, testNamespace, testData, postgreSQLtemplates) }) } } @@ -651,6 +699,8 @@ var data = templateData{ VaultNamespace: vaultNamespace, VaultPromDomain: vaultPromDomain, VaultPkiCommonName: fmt.Sprintf("keda.%s.svc", testNamespace), + HashiCorpSecretName: HashiCorpSecretName, + HashiCorpSecretKey: HashiCorpSecretKey, } func getPostgreSQLTemplateData() (templateData, []Template) { @@ -678,3 +728,12 @@ func getTemplateData() (templateData, []Template) { {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, } } +func getTemplateDataWithTokenSecret() (templateData, []Template) { + return data, []Template{ + {Name: "secretTemplate", Config: secretTemplate}, + {Name: "secretHashicorpTokenTemplate", Config: secretHashicorpTokenTemplate}, + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "triggerAuthenticationTemplate", Config: triggerAuthenticationWithSecretTemplate}, + {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, + } +}