feat: ECS Cluster Target Discovery

source/Calamari.Aws/AwsModule.cs

@@ -1,4 +1,5 @@
 using Autofac;
+using Calamari.Aws.Discovery;
 using Calamari.Aws.Inputs.Ecs;
 using Calamari.Aws.Integration.Ecs;
 
@@ -10,5 +11,10 @@ protected override void Load(ContainerBuilder builder)
     {
         builder.RegisterType<EcsStackNameGenerator>().As<IEcsStackNameGenerator>().SingleInstance();
         builder.RegisterType<EcsImageNameResolver>().As<IEcsImageNameResolver>().SingleInstance();
+
+        builder.RegisterType<EcsClientFactory>().As<IEcsClientFactory>().SingleInstance();
+        builder.RegisterType<EcsDiscoverer>().As<IEcsDiscoverer>().InstancePerDependency();
+        builder.RegisterType<AwsTargetDiscoveryContextResolver>().As<IAwsTargetDiscoveryContextResolver>().SingleInstance();
+        builder.RegisterType<EcsClusterDiscoveryWriter>().As<IEcsClusterDiscoveryWriter>().InstancePerDependency();
     }
 }

source/Calamari.Aws/Behaviours/EcsClusterDiscoveryBehaviour.cs

@@ -1,16 +1,121 @@
-using System.Threading.Tasks;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Amazon.Runtime;
+using Calamari.Aws.Discovery;
+using Calamari.Aws.Integration.Ecs;
 using Calamari.Common.Commands;
+using Calamari.Common.Features.Discovery;
+using Calamari.Common.Plumbing.Logging;
 using Calamari.Common.Plumbing.Pipeline;
+using Octopus.Calamari.Contracts.TargetDiscovery;
+using Octopus.CoreUtilities.Extensions;
+using Task = System.Threading.Tasks.Task;
 
 namespace Calamari.Aws.Behaviours;
 
-public class EcsClusterDiscoveryBehaviour : IDeployBehaviour
+public class EcsClusterDiscoveryBehaviour(IEcsDiscoverer ecsDiscoverer, IAwsTargetDiscoveryContextResolver contextResolver, IEcsClusterDiscoveryWriter clusterDiscoveryWriter, ILog log) : IDeployBehaviour
 {
     public bool IsEnabled(RunningDeployment context) => true;
+
+    public async Task Execute(RunningDeployment deployment)
+    {
+        const string contextVariableName = TargetDiscoverySpecialVariables.TargetDiscoveryContext;
+
+        var contextJson = deployment.Variables.Get(contextVariableName);
+        if (string.IsNullOrEmpty(contextJson))
+        {
+            log.Warn($"Could not find target discovery context in variable {contextVariableName}.");
+            log.Warn("Aborting target discovery.");
+            return;
+        }
+
+        if (!contextResolver.TryResolve(contextJson, log, out var discoveryContext))
+        {
+            log.Warn("Aborting target discovery. Could not resolve context.");
+            return;
+        }
+
+        var authentication = discoveryContext.Authentication;
+        var scope = discoveryContext.Scope;
+
+        LogAuthenticationDetails(authentication);
+
+        if (!authentication.TryGetCredentials(log, out var credentials))
+        {
+            log.Warn("Aborting target discovery. Invalid credentials.");
+            return;
+        }
+
+        var discoveredTargetCount = 0;
+        try
+        {
+            foreach (var region in authentication.Regions)
+            {
+                foreach (var cluster in await ecsDiscoverer.DiscoverClustersInRegion(credentials, region))
+                {
+                    var tags = (cluster.Tags ?? [])
+                               .Select(tag => new KeyValuePair<string, string>(tag.Key, tag.Value))
+                               .ToTargetTags();
+
+                    var matchResult = scope.Match(tags);
+                    if (matchResult.IsSuccess)
+                    {
+                        discoveredTargetCount++;
+                        log.Info($"Discovered matching ECS cluster '{cluster.ClusterName}' in {region}.");
+                        clusterDiscoveryWriter.WriteTargetCreationServiceMessage(region, cluster, authentication, scope, matchResult);
+                    }
+                    else
+                    {
+                        log.Verbose($"ECS cluster '{cluster.ClusterName}' in {region} does not match target requirements:");
+                        foreach (var reason in matchResult.FailureReasons)
+                        {
+                            log.Verbose($"- {reason}");
+                        }
+                    }
+                }
+            }
+        }
+        catch (AmazonServiceException ex)
+        {
+            log.Warn("Error connecting to AWS to look for ECS clusters:");
+            log.Warn(ex.Message);
+            log.Warn("Aborting target discovery.");
+            return;
+        }
+
+        log.Info(discoveredTargetCount > 0
+                     ? $"{discoveredTargetCount} ECS cluster target{(discoveredTargetCount > 1 ? "s" : "")} found."
+                     : "Could not find any ECS cluster targets.");
+    }
 
 
-    public Task Execute(RunningDeployment context)
+    void LogAuthenticationDetails(IAwsAuthenticationDetails authentication)
     {
-        return Task.CompletedTask;
+        log.Verbose("Looking for ECS clusters in AWS using:");
+        log.Verbose($"\tAccount: {authentication.AccountId}");
+        log.Verbose($"\tRegions: [{string.Join(",", authentication.Regions)}]");
+
+        if (authentication.Role.Type == "assumeRole")
+        {
+            log.Verbose("\tRole:");
+            log.Verbose($"\t\tARN: {authentication.Role.Arn}");
+            if (!authentication.Role.SessionName.IsNullOrEmpty())
+            {
+                log.Verbose($"\t\tSession Name: {authentication.Role.SessionName}");
+            }
+            if (authentication.Role.SessionDuration != null)
+            {
+                log.Verbose($"\t\tSession Duration: {authentication.Role.SessionDuration}");
+            }
+            if (!authentication.Role.ExternalId.IsNullOrEmpty())
+            {
+                log.Verbose($"\t\tExternal Id: {authentication.Role.ExternalId}");
+            }
+        }
+        else
+        {
+            log.Verbose("\tRole: No IAM Role provided.");
+        }
     }
-}
+}

source/Calamari.Aws/Commands/EcsClusterHealthCheck.cs

@@ -34,6 +34,6 @@ public override int Execute(string[] commandLineArguments)
 
         return 0;
 
-        IAmazonECS ClientFactory() => EcsClientFactory.Create(environment);
+        IAmazonECS ClientFactory() => EcsClientFactoryHelper.Create(environment);
     }
 }

source/Calamari.Aws/Commands/UpdateEcsServiceCommand.cs

@@ -31,7 +31,7 @@ public override int Execute(string[] commandLineArguments)
         var inputs = ReadAndValidateInputs();
         var environment = AwsEnvironmentGeneration.Create(log, variables).GetAwaiter().GetResult();
 
-        using var ecsClient = EcsClientFactory.Create(environment);
+        using var ecsClient = EcsClientFactoryHelper.Create(environment);
 
         new ConventionProcessor(new RunningDeployment(variables),
                                 [

source/Calamari.Aws/Kubernetes/Discovery/AwsAuthenticationDetails.cs → source/Calamari.Aws/Discovery/AwsAuthenticationDetails.cs

@@ -5,11 +5,12 @@
 using Amazon.Runtime;
 using Amazon.SecurityToken;
 using Amazon.SecurityToken.Model;
-using Calamari.Common.Features.Discovery;
 using Calamari.Common.Plumbing.Logging;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 using Octopus.Calamari.Contracts.TargetDiscovery;
 
-namespace Calamari.Aws.Kubernetes.Discovery;
+namespace Calamari.Aws.Discovery;
 
 public class AwsAccessKeyAuthenticationDetails : AwsAuthenticationDetails<AwsAccessKeyCredentials>, IAwsAuthenticationDetails
 {
@@ -151,12 +152,52 @@ static bool TryGetInstanceProfileAwsCredentials(ILog log, out AWSCredentials cre
     }
 }
 
-public interface IAwsAuthenticationDetails
+[JsonConverter(typeof(AwsAuthenticationDetailsConverter))]
+public interface IAwsAuthenticationDetails : ITargetDiscoveryAuthenticationDetails
 {
     AwsAssumedRole Role { get; set; }
     IEnumerable<string> Regions { get; set; }
+    string AccountId { get; }
     bool TryGetCredentials(ILog log, out AWSCredentials credentials);
 }
+
+/// <summary>
+/// JSON Deserializer to handle matching Context Credentials to AWS Basic or OIDC credentials
+/// </summary>
+public class AwsAuthenticationDetailsConverter : JsonConverter
+{
+    public override bool CanConvert(Type objectType) => objectType == typeof(IAwsAuthenticationDetails);
+
+    public override bool CanWrite => false;
+
+    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+    {
+        if (reader.TokenType == JsonToken.Null)
+            return null;
+
+        var jObject = JObject.Load(reader);
+        var credentialsType = jObject
+                              .GetValue("Credentials", StringComparison.OrdinalIgnoreCase)?.Value<JObject>()
+                              ?.GetValue("Type", StringComparison.OrdinalIgnoreCase)?.Value<string>();
+
+        // Instantate and populate to avoid recursion issues in JSON.NET,
+        IAwsAuthenticationDetails authenticationDetails = credentialsType switch
+        {
+            "account" => new AwsAccessKeyAuthenticationDetails(),
+            "oidcAccount" => new AwsOidcAuthenticationDetails(),
+            "worker" => new AwsWorkerAuthenticationDetails(),
+            _ => throw new JsonSerializationException(
+                     $"Unable to resolve AWS authentication details: unsupported credentials type '{credentialsType}'.")
+        };
+
+        using var jObjectReader = jObject.CreateReader();
+        serializer.Populate(jObjectReader, authenticationDetails);
+        return authenticationDetails;
+    }
+
+    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        => throw new NotSupportedException($"{nameof(AwsAuthenticationDetailsConverter)} is only used for deserialisation.");
+}
 
 public class AwsAuthenticationDetails<TCredentials> : ITargetDiscoveryAuthenticationDetails
     where TCredentials : AwsCredentialsBase
@@ -183,7 +224,10 @@ protected AWSCredentials GetCredentialsWithAssumedRoleIfNeeded(AWSCredentials cr
     public string AuthenticationMethod { get; set; } = string.Empty;
 
     public AwsCredentials<TCredentials> Credentials { get; set; }
-
+
+    [JsonIgnore]
+    public string AccountId => Credentials?.AccountId;
+
     public AwsAssumedRole Role { get; set; }
 
     public IEnumerable<string> Regions { get; set; }

source/Calamari.Aws/Discovery/AwsTargetDiscoveryContextResolver.cs

@@ -0,0 +1,39 @@
+using Calamari.Common.Plumbing.Logging;
+using Newtonsoft.Json;
+using Octopus.Calamari.Contracts.TargetDiscovery;
+
+namespace Calamari.Aws.Discovery;
+
+public interface IAwsTargetDiscoveryContextResolver
+{
+    bool TryResolve(string contextJson, ILog log, out TargetDiscoveryContext<IAwsAuthenticationDetails> context);
+}
+
+/// <summary>
+/// Deserializes the Discovery Context into AWS Credentials
+/// </summary>
+public class AwsTargetDiscoveryContextResolver: IAwsTargetDiscoveryContextResolver
+{
+    public bool TryResolve(string contextJson, ILog log, out TargetDiscoveryContext<IAwsAuthenticationDetails> context)
+    {
+        context = null;
+        try
+        {
+            context = JsonConvert.DeserializeObject<TargetDiscoveryContext<IAwsAuthenticationDetails>>(contextJson);
+        }
+        catch (JsonException ex)
+        {
+            log.Warn($"AWS target discovery context is in the wrong format: {ex.Message}");
+            return false;
+        }
+
+        if (context?.Authentication == null || context?.Scope == null)
+        {
+            log.Warn("AWS target discovery context is in the wrong format.");
+            context = null;
+            return false;
+        }
+
+        return true;
+    }
+}

source/Calamari.Aws/Integration/Ecs/AwsEcsServiceMessageNames.cs

@@ -0,0 +1,20 @@
+namespace Calamari.Aws.Integration.Ecs;
+
+/// <summary>
+/// The service message name and attribute keys used by server.
+/// Note: Should these migrate to CalamariContracts?
+/// </summary>
+public static class AwsEcsServiceMessageNames
+{
+    public const string CreateTargetName = "create-aws-ecs-target";
+    public const string AccountIdOrNameAttribute = "octopusAccountIdOrName";
+    public const string ClusterNameAttribute = "clusterName";
+    public const string WorkerPoolIdOrNameAttribute = "octopusDefaultWorkerPoolIdOrName";
+    public const string ClusterRegionAttribute = "clusterRegion";
+    public const string UseInstanceRole = "useInstanceRole";
+    public const string AssumeRole = "assumeRole";
+    public const string AssumeRoleArn = "assumeRoleArn";
+    public const string AssumeRoleSession = "assumeRoleSession";
+    public const string AssumeRoleSessionDurationSeconds = "assumeRoleSessionDurationSeconds";
+    public const string AssumeRoleExternalId = "assumeRoleExternalId";
+}

source/Calamari.Aws/Integration/Ecs/EcsClientFactory.cs

@@ -1,11 +1,18 @@
+using Amazon;
 using Amazon.ECS;
-using Calamari.Aws.Util;
-using Calamari.CloudAccounts;
+using Amazon.Runtime;
 
 namespace Calamari.Aws.Integration.Ecs;
 
-public static class EcsClientFactory
+public interface IEcsClientFactory
 {
-    public static IAmazonECS Create(AwsEnvironmentGeneration environment) =>
-        new AmazonECSClient(environment.AwsCredentials, environment.AsClientConfig<AmazonECSConfig>());
+    IAmazonECS Create(AWSCredentials credentials, string region);
 }
+
+public class EcsClientFactory : IEcsClientFactory
+{
+    public IAmazonECS Create(AWSCredentials credentials, string region)
+    {
+        return new AmazonECSClient(credentials, RegionEndpoint.GetBySystemName(region));
+    }
+}

source/Calamari.Aws/Integration/Ecs/EcsClientFactoryHelper.cs

@@ -0,0 +1,12 @@
+using Amazon.ECS;
+using Calamari.Aws.Util;
+using Calamari.CloudAccounts;
+
+namespace Calamari.Aws.Integration.Ecs;
+
+// TODO: Replace Static Helper with concrete class/interface to enable better testing
+public static class EcsClientFactoryHelper
+{
+    public static IAmazonECS Create(AwsEnvironmentGeneration environment) =>
+        new AmazonECSClient(environment.AwsCredentials, environment.AsClientConfig<AmazonECSConfig>());
+}