fix(AWSCreds): Caused by race condition while processing s3 artifact

CHANGELOG.md

@@ -1,3 +1,118 @@
+# v1.4.0
+
+## Features
+- feat(k8s): Add support for GA ingress apiVersion in helm chart and operator (#287)
+
+## Improvements
+- chore(docker): Update alpine images operator + halyard (#292)
+- chore(release): Update deployment manifest with specific release tag (#282)
+- docs(k8s): Kubernetes compatibility matrix (#285)
+- chore(release): Manifests update
+
+# v1.3.1
+
+## Improvements
+- chore(docker): Update alpine images operator + halyard (#292) (#293)
+
+## Dependencies
+- Updated halyard version to operator-a6ac1d4
+
+# v1.3.0
+
+## Features
+- feat(lambda/validation): Validation regarding the AWS Lambda using the GO SDK to get the Lambda Functions using the AWS provider credentials
+- feat(cloudfoundry/validation): Add a CloudFoundry validation for each account (#222)
+- feat(validator/aws): Add AWS account validator (#195)
+- feat(health-check): Increase timeout and validate ready replicas for status (#219)
+- feat(halyard): Bump version (#269)
+
+## Bug Fixes
+- fix(build): Kind unable to start control-plane (#279)
+- fix(actions): Update metallb condition (#259)
+- fix(it): Fix integration tests (#236)
+- fix(expose): Override public service port (#210)
+- fix(validation): Validate primary account for kubernetes provider (#209)
+- fix(timeout): Avoid revalidation when patching Spinnaker Status (#192)
+- fix(build): No push dev image to scan.connect.redhat.com (#184)
+- Fix transforming k8s secrets (#262)
+
+## Improvements
+- chore(ci): Upgrading actions (#284)
+- chore(ci): Skipping integration tests (#283)
+- chore(dependency): Update upstream oss halyard version (#280, #278)
+- chore(build): Split actions for PRs and master (#273)
+- chore(build): Make sure forks can run tests, but not create releases (#268)
+- chore(halyard): Updated halyard version (#232, #213, #200, #197)
+- chore(cve): Fix for CVE-2020-13757 (#193)
+- update(kind): Update yml reference files (#258, #257)
+- update(operator): Update files for the new API of k8s v1.22 (#252) (#256)
+- doc(fix): Add Plugins section to README (#220)
+
+## Dependencies
+- Updated halyard version to operator-b135799
+- Support for Kubernetes v1.22 API changes
+
+# v1.2.5
+
+## Improvements
+- chore(release): v1.2.5 (#233)
+
+## Dependencies
+- Updated halyard version to operator-ccae06e
+
+# v1.2.4
+
+## Features
+- feat(health-check): Increase timeout and validate ready replicas for status (#219) (#221)
+
+# v1.2.3
+
+## Improvements
+- chore(release): v1.2.3 (#217)
+
+# v1.2.2
+
+## Improvements
+- chore(halyard): Updated halyard version (#200) (#201)
+
+## Dependencies
+- Updated halyard version to operator-7162184
+
+# v1.2.1
+
+## Improvements
+- chore(release): v1.2.1 (#198)
+
+## Dependencies
+- Updated halyard version to operator-8e0406f
+
+# v1.2.0
+
+## Features
+- feat(health-check): Check spinnaker status (#168)
+- feat(ubi): Add build for UBI images (#158)
+
+## Bug Fixes
+- fix(build): No push dev image to scan.connect.redhat.com (#184) (#185)
+- fix(ingress): Fix panic when overriding endpoint with ingress (#181)
+- fix(test): Fix integration tests (#179, #171)
+- fix(k8s-context): Use the context passed in SpinnakerService when validating Kubernetes accounts (#173)
+- fix(ingress): Support ingress with load balancer IP (GCE/bare metal) (#170)
+- fix(ubi): Fix UBI run issue (#169)
+- fix(test): Added ingress permissions to role used in tests (#155)
+- fix(expose/ingress): Solve issue for spinsvc status URL (#154)
+
+## Improvements
+- chore(halyard): Update version (#183)
+- chore(release): Updated halyard version (#174)
+- chore(license): Update Copyright section (#162)
+- chore(coverage): Add code coverage (#161)
+- chore(mergify): Add mergify config (#157)
+- chore(release): Update changelog
+
+## Dependencies
+- Updated halyard version to operator-c1d641c
+
 # Unreleased (1.1.2)
 
 - chore: Update halyard version.

build-tools/Dockerfile

@@ -24,7 +24,7 @@ RUN wget -nv https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-c
   && mkdir -p /opt && cd /opt \
   && tar -xzf /google-cloud-sdk-${GOOGLE_CLOUD_SDK_VERSION}-linux-x86_64.tar.gz \
   && rm /google-cloud-sdk-${GOOGLE_CLOUD_SDK_VERSION}-linux-x86_64.tar.gz \
-  && CLOUDSDK_PYTHON="python3" /opt/google-cloud-sdk/install.sh --usage-reporting=false --bash-completion=false  --additional-components app-engine-java app-engine-go \
+  && CLOUDSDK_PYTHON="python3" /opt/google-cloud-sdk/install.sh --usage-reporting=false --bash-completion=false  --additional-components app-engine-java \
   && rm -rf ~/.config/gcloud \
   && gcloud components remove --quiet anthoscli \
   && rm -rf /opt/google-cloud-sdk/.install/.backup

pkg/deploy/spindeploy/transformer/secrets.go

@@ -3,6 +3,9 @@ package transformer
 import (
 	"context"
 	"fmt"
+	"path"
+	"strings"
+
 	secups "github.com/armory/go-yaml-tools/pkg/secrets"
 	"github.com/armory/spinnaker-operator/pkg/apis/spinnaker/interfaces"
 	"github.com/armory/spinnaker-operator/pkg/generated"
@@ -15,9 +18,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
-	"path"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"strings"
 )
 
 const (
@@ -32,6 +33,8 @@ const (
 	monitoringContainerName = "monitoring-daemon"
 )
 
+var clouddriverServices = []string{"clouddriver", "clouddriver-ro", "clouddriver-rw", "clouddriver-ro-deck", "clouddriver-caching"}
+
 // secretsTransformer maps Kubernetes secrets onto the deployment of the service that requires it
 // Either as a mounted file (encryptedFile) or an environment variable (tokens, passwords...)
 type secretsTransformer struct {
@@ -90,7 +93,22 @@ func (s *secretsTransformer) replaceK8sSecretsFromAwsKeys(spinCfg *interfaces.Sp
 		return err
 	}
 	if artifactKeys != nil {
-		s.k8sSecrets.awsCredsByService["clouddriver"] = artifactKeys
+		// Merge artifact keys with existing provider keys instead of overwriting
+		var finalKeys *awsCredentials
+		if existingKeys, exists := s.k8sSecrets.awsCredsByService["clouddriver"]; exists {
+			finalKeys = s.mergeAwsCredentials(existingKeys, artifactKeys)
+		} else {
+			finalKeys = artifactKeys
+		}
+		// Apply the merged credentials to all clouddriver service variants
+		for _, svcName := range clouddriverServices {
+			s.k8sSecrets.awsCredsByService[svcName] = finalKeys
+		}
+	} else if providerKeys != nil {
+		// If only provider keys exist, apply them to all clouddriver service variants
+		for _, svcName := range clouddriverServices {
+			s.k8sSecrets.awsCredsByService[svcName] = providerKeys
+		}
 	}
 	can, ok := spinCfg.Config[awsCanary]
 	if !ok {
@@ -130,8 +148,20 @@ func (s *secretsTransformer) getAndReplace(svc, accessKeyProp, secretKeyProp str
 	if err != nil {
 		return nil, err
 	}
+	var genAccessKey v1.EnvVar
+	// Check if access key is also a Kubernetes secret reference
+	if secups.IsEncryptedSecret(accessKeyRaw) {
+		accessKeySecretName, accessKeySecretKey, err := secrets.ParseKubernetesSecretParams(accessKeyRaw)
+		if err != nil {
+			return nil, err
+		}
+		genAccessKey = envVarFromSecretReference("AWS_ACCESS_KEY_ID", accessKeySecretName, accessKeySecretKey)
+	} else {
+		// Access key is plain text
+		genAccessKey = envVarFromRawString("AWS_ACCESS_KEY_ID", accessKeyRaw)
+	}
 	return &awsCredentials{
-		genAccessKey:  envVarFromRawString("AWS_ACCESS_KEY_ID", accessKeyRaw),
+		genAccessKey:  genAccessKey,
 		genSecretKey:  envVarFromSecretReference("AWS_SECRET_ACCESS_KEY", secretName, secretKey),
 		svcSecretKeys: []v1.EnvVar{envVarFromSecretReference(envVarName, secretName, secretKey)},
 	}, nil
@@ -164,7 +194,17 @@ func (s *secretsTransformer) getAndReplaceArray(svc, rootProp, accessKeyProp, se
 		if !ok {
 			return nil, fmt.Errorf("aws secret access key specified without access key under %s", root)
 		}
-		genAccessKey = envVarFromRawString("AWS_ACCESS_KEY_ID", accessKey)
+		// Check if access key is also a Kubernetes secret reference
+		if secups.IsEncryptedSecret(accessKey) {
+			accessKeySecretName, accessKeySecretKey, err := secrets.ParseKubernetesSecretParams(accessKey)
+			if err != nil {
+				return nil, err
+			}
+			genAccessKey = envVarFromSecretReference("AWS_ACCESS_KEY_ID", accessKeySecretName, accessKeySecretKey)
+		} else {
+			// Access key is plain text
+			genAccessKey = envVarFromRawString("AWS_ACCESS_KEY_ID", accessKey)
+		}
 		genSecretKey = envVarFromSecretReference("AWS_SECRET_ACCESS_KEY", secretName, secretKey)
 		svcSecretKeys = append(svcSecretKeys, envVarFromSecretReference(envVarName, secretName, secretKey))
 	}
@@ -188,6 +228,25 @@ func (s *secretsTransformer) getAndReplaceArray(svc, rootProp, accessKeyProp, se
 	}, nil
 }
 
+// mergeAwsCredentials merges two awsCredentials structs, preserving both sets of credentials
+func (s *secretsTransformer) mergeAwsCredentials(existing, new *awsCredentials) *awsCredentials {
+	merged := &awsCredentials{
+		// Use the new credentials for generic AWS keys (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
+		// This follows the "last one wins" pattern from getAndReplaceArray
+		genAccessKey: new.genAccessKey,
+		genSecretKey: new.genSecretKey,
+		// Combine service-specific secret keys from both credentials
+		svcSecretKeys: make([]v1.EnvVar, 0, len(existing.svcSecretKeys)+len(new.svcSecretKeys)),
+	}
+
+	// Add new service-specific keys first (artifact credentials)
+	merged.svcSecretKeys = append(merged.svcSecretKeys, new.svcSecretKeys...)
+	// Add existing service-specific keys after (provider credentials)
+	merged.svcSecretKeys = append(merged.svcSecretKeys, existing.svcSecretKeys...)
+
+	return merged
+}
+
 func (s *secretsTransformer) sanitizeK8sSecret(object interface{}, ctx context.Context) (interface{}, error) {
 	h := func(val string) (string, error) {
 		if !secups.IsEncryptedSecret(val) {
@@ -217,10 +276,10 @@ func (s *secretsTransformer) TransformManifests(ctx context.Context, gen *genera
 			if ok && sec.Object["kind"] == "Secret" {
 				var secret v1.Secret
 				runtime.DefaultUnstructuredConverter.FromUnstructured(sec.Object, &secret)
-					err := kCollector.mapSecrets(&secret)
-					if err != nil {
-						return err
-					}
+				err := kCollector.mapSecrets(&secret)
+				if err != nil {
+					return err
+				}
 				cfg.Resources[k] = &secret
 			}
 		}