diff --git a/.github/workflows/update-azure-vm-sizes.yml b/.github/workflows/update-azure-vm-sizes.yml new file mode 100644 index 00000000000..a3a163c90a7 --- /dev/null +++ b/.github/workflows/update-azure-vm-sizes.yml @@ -0,0 +1,48 @@ +name: Update Azure VM Sizes + +on: + workflow_dispatch: + schedule: + - cron: '0 6 1 * *' # Monthly on the 1st at 06:00 UTC + +permissions: + contents: write + pull-requests: write + +jobs: + generate-and-pr: + runs-on: ubuntu-latest + if: ${{ github.repository_owner == 'microsoft' }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Azure Login + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Generate updated Azure VM size descriptors + working-directory: src/Aspire.Hosting.Azure.Kubernetes/tools + run: | + set -e + "$GITHUB_WORKSPACE/dotnet.sh" run GenVmSizes.cs + + - name: Generate GitHub App Token + id: app-token + uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 + with: + app-id: ${{ secrets.ASPIRE_BOT_APP_ID }} + private-key: ${{ secrets.ASPIRE_BOT_PRIVATE_KEY }} + + - name: Create or update pull request + uses: ./.github/actions/create-pull-request + with: + token: ${{ steps.app-token.outputs.token }} + branch: update-azure-vm-sizes + base: main + commit-message: "[Automated] Update Azure VM Sizes" + labels: | + area-integrations + area-engineering-systems + title: "[Automated] Update Azure VM Sizes" + body: "Auto-generated update of Azure VM size descriptors (AksNodeVmSizes.Generated.cs)." diff --git a/Aspire.slnx b/Aspire.slnx index 56db3bc54b8..61eab697421 100644 --- a/Aspire.slnx +++ b/Aspire.slnx @@ -93,6 +93,7 @@ + @@ -467,6 +468,7 @@ + diff --git a/Directory.Packages.props b/Directory.Packages.props index 67a3b7bc551..e760b0d3c91 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -44,6 +44,7 @@ + diff --git a/docs/specs/aks-support.md b/docs/specs/aks-support.md new file mode 100644 index 00000000000..02c56f20490 --- /dev/null +++ b/docs/specs/aks-support.md @@ -0,0 +1,620 @@ +# AKS Support in Aspire — Implementation Spec + +## Problem Statement + +Aspire's `Aspire.Hosting.Kubernetes` package currently supports end-to-end deployment to any conformant Kubernetes cluster (including AKS) via Helm charts. However, the support is **generic Kubernetes** — it has no awareness of Azure-specific capabilities. Users who want to deploy to AKS must manually provision the cluster, configure workload identity, set up monitoring, and manage networking outside of Aspire. + +The goal is to create a first-class AKS experience in Aspire that supports: +- **Provisioning the AKS cluster itself** via Azure.Provisioning +- **Workload identity** (Azure AD federated credentials for pods) +- **Azure Monitor integration** (Container Insights, Log Analytics, managed Prometheus/Grafana) +- **VNet integration** (subnet delegation, private clusters) +- **Network perimeter support** (NSP, private endpoints for backing Azure services) + +## Current State Analysis + +### Kubernetes Publishing (Aspire.Hosting.Kubernetes) +- **Helm-chart based** deployment model with 5-step pipeline: Publish → Prepare → Deploy → Summary → Uninstall +- `KubernetesEnvironmentResource` is the root compute environment +- `KubernetesResource` wraps each Aspire resource into Deployment/Service/ConfigMap/Secret YAML +- `HelmDeploymentEngine` executes `helm upgrade --install` +- No Azure awareness — works with any kubeconfig-accessible cluster +- Identity support: ❌ None +- Networking: Basic K8s Service/Ingress only +- Monitoring: OTLP to Aspire Dashboard only + +### Azure Provisioning Patterns (established) +- `AzureProvisioningResource` base class → generates Bicep via `Azure.Provisioning` SDK +- `AzureResourceInfrastructure` builder → `CreateExistingOrNewProvisionableResource()` factory +- `BicepOutputReference` for cross-resource wiring +- `AppIdentityAnnotation` + `IAppIdentityResource` for managed identity attachment +- Role assignments via `AddRoleAssignments()` / `IAddRoleAssignmentsContext` + +### Azure Container Apps (reference pattern) +- `AzureContainerAppEnvironmentResource` : `AzureProvisioningResource`, `IAzureComputeEnvironmentResource` +- Implements `IAzureContainerRegistry`, `IAzureDelegatedSubnetResource` +- Auto-creates Container Registry, Log Analytics, managed identity +- Subscribes to `BeforeStartEvent` → creates ContainerApp per compute resource → adds `DeploymentTargetAnnotation` + +### Azure Networking (established) +- VNet, Subnet, NSG, NAT Gateway, Private DNS Zone, Private Endpoint, Public IP resources +- `IAzurePrivateEndpointTarget` interface (implemented by Storage, SQL, etc.) +- `IAzureNspAssociationTarget` for network security perimeters +- `DelegatedSubnetAnnotation` for service delegation +- `PrivateEndpointTargetAnnotation` to deny public access + +### Azure Identity (established) +- `AzureUserAssignedIdentityResource` with Id, ClientId, PrincipalId outputs +- `AppIdentityAnnotation` attaches identity to compute resources +- Container Apps sets `AZURE_CLIENT_ID` + `AZURE_TOKEN_CREDENTIALS=ManagedIdentityCredential` +- **No workload identity or federated credential support** exists today + +### Azure Monitoring (established) +- `AzureLogAnalyticsWorkspaceResource` via `Azure.Provisioning.OperationalInsights` +- `AzureApplicationInsightsResource` via `Azure.Provisioning.ApplicationInsights` +- Container Apps links Log Analytics workspace to environment + +## Proposed Architecture + +### New Package: `Aspire.Hosting.Azure.Kubernetes` + +This package provides a unified `AddAzureKubernetesEnvironment()` entry point that internally invokes `AddKubernetesEnvironment()` (from the generic K8s package) and layers on AKS-specific Azure provisioning. This mirrors the established pattern of `AddAzureContainerAppEnvironment()` which internally sets up the Container Apps infrastructure. + +```text +Aspire.Hosting.Azure.Kubernetes +├── depends on: Aspire.Hosting.Kubernetes +├── depends on: Aspire.Hosting.Azure +├── depends on: Azure.Provisioning.Kubernetes (v1.0.0-beta.3) +├── depends on: Azure.Provisioning.Roles (for federated credentials) +├── depends on: Azure.Provisioning.Network (for VNet integration) +└── depends on: Azure.Provisioning.OperationalInsights (for monitoring) +``` + +### Design Principle: Unified Environment Resource + +Just as `AddAzureContainerAppEnvironment("aca")` creates a single resource that is both the Azure provisioning target AND the compute environment, `AddAzureKubernetesEnvironment("aks")` creates a single `AzureKubernetesEnvironmentResource` that: +1. Extends `AzureProvisioningResource` (generates Bicep for AKS cluster + supporting resources) +2. Implements `IAzureComputeEnvironmentResource` (serves as the compute target) +3. Internally creates and manages a `KubernetesEnvironmentResource` for Helm-based deployment +4. Registers the `KubernetesInfrastructure` eventing subscriber (same as `AddKubernetesEnvironment`) + +### Integration Points + +```text +┌─────────────────────────────────────────────────────────────┐ +│ User's AppHost │ +│ │ +│ var aks = builder.AddAzureKubernetesEnvironment("aks") │ +│ .WithDelegatedSubnet(subnet) │ +│ .WithAzureLogAnalyticsWorkspace(logAnalytics) │ +│ .WithWorkloadIdentity() │ +│ .WithVersion("1.30") │ +│ .WithHelm(...) ← from K8s environment │ +│ .WithDashboard(); ← from K8s environment │ +│ │ +│ var db = builder.AddAzureSqlServer("sql") │ +│ .WithPrivateEndpoint(subnet); ← existing pattern │ +│ │ +│ builder.AddProject() │ +│ .WithReference(db) │ +│ .WithAzureWorkloadIdentity(identity); ← new │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Detailed Design + +### 1. Unified AKS Environment Resource + +**New resource**: `AzureKubernetesEnvironmentResource` + +This is the single entry point — analogous to `AzureContainerAppEnvironmentResource`. It extends `AzureProvisioningResource` to generate Bicep for the AKS cluster and supporting infrastructure, while also serving as the compute environment by internally delegating to `KubernetesEnvironmentResource` for Helm-based deployment. + +```csharp +public class AzureKubernetesEnvironmentResource( + string name, + Action configureInfrastructure) + : AzureProvisioningResource(name, configureInfrastructure), + IAzureComputeEnvironmentResource, + IAzureContainerRegistry, // For ACR integration + IAzureDelegatedSubnetResource, // For VNet integration + IAzureNspAssociationTarget // For NSP association +{ + // The underlying generic K8s environment (created internally) + internal KubernetesEnvironmentResource KubernetesEnvironment { get; set; } = default!; + + // AKS cluster outputs + public BicepOutputReference Id => new("id", this); + public BicepOutputReference ClusterFqdn => new("clusterFqdn", this); + public BicepOutputReference OidcIssuerUrl => new("oidcIssuerUrl", this); + public BicepOutputReference KubeletIdentityObjectId => new("kubeletIdentityObjectId", this); + public BicepOutputReference NodeResourceGroup => new("nodeResourceGroup", this); + public BicepOutputReference NameOutputReference => new("name", this); + + // ACR outputs (like AzureContainerAppEnvironmentResource) + internal BicepOutputReference ContainerRegistryName => new("AZURE_CONTAINER_REGISTRY_NAME", this); + internal BicepOutputReference ContainerRegistryUrl => new("AZURE_CONTAINER_REGISTRY_ENDPOINT", this); + internal BicepOutputReference ContainerRegistryManagedIdentityId + => new("AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID", this); + + // Service delegation + string IAzureDelegatedSubnetResource.DelegatedSubnetServiceName + => "Microsoft.ContainerService/managedClusters"; + + // Configuration + internal string? KubernetesVersion { get; set; } + internal AksSkuTier SkuTier { get; set; } = AksSkuTier.Free; + internal bool OidcIssuerEnabled { get; set; } = true; + internal bool WorkloadIdentityEnabled { get; set; } = true; + internal AzureContainerRegistryResource? DefaultContainerRegistry { get; set; } + internal AzureLogAnalyticsWorkspaceResource? LogAnalyticsWorkspace { get; set; } + + // Node pool configuration + internal List NodePools { get; } = [ + new AksNodePoolConfig("system", "Standard_D4s_v5", 1, 3, AksNodePoolMode.System) + ]; + + // Networking + internal AksNetworkProfile? NetworkProfile { get; set; } + internal AzureSubnetResource? SubnetResource { get; set; } + internal bool IsPrivateCluster { get; set; } +} +``` + +**Entry point extension** (mirrors `AddAzureContainerAppEnvironment`): +```csharp +public static class AzureKubernetesEnvironmentExtensions +{ + public static IResourceBuilder AddAzureKubernetesEnvironment( + this IDistributedApplicationBuilder builder, + [ResourceName] string name) + { + // 1. Set up Azure provisioning infrastructure + builder.AddAzureProvisioning(); + builder.Services.Configure( + o => o.SupportsTargetedRoleAssignments = true); + + // 2. Register the AKS-specific infrastructure eventing subscriber + builder.Services.TryAddEventingSubscriber(); + + // 3. Also register the generic K8s infrastructure (for Helm chart generation) + builder.AddKubernetesInfrastructureCore(); + + // 4. Create the unified environment resource + var resource = new AzureKubernetesEnvironmentResource(name, ConfigureAksInfrastructure); + + // 5. Create the inner KubernetesEnvironmentResource (for Helm deployment) + resource.KubernetesEnvironment = new KubernetesEnvironmentResource($"{name}-k8s") + { + HelmChartName = builder.Environment.ApplicationName.ToHelmChartName(), + Dashboard = builder.CreateDashboard($"{name}-dashboard") + }; + + // 6. Auto-create ACR (like Container Apps does) + var acr = CreateDefaultContainerRegistry(builder, name); + resource.DefaultContainerRegistry = acr; + + return builder.AddResource(resource); + } + + // Configuration extensions + public static IResourceBuilder WithVersion( + this IResourceBuilder builder, string version); + public static IResourceBuilder WithSkuTier( + this IResourceBuilder builder, AksSkuTier tier); + public static IResourceBuilder WithNodePool( + this IResourceBuilder builder, + string name, string vmSize, int minCount, int maxCount, + AksNodePoolMode mode = AksNodePoolMode.User); + + // Networking (matches existing pattern: WithDelegatedSubnet) + public static IResourceBuilder WithDelegatedSubnet( + this IResourceBuilder builder, + IResourceBuilder subnet); + public static IResourceBuilder AsPrivateCluster( + this IResourceBuilder builder); + + // Identity + public static IResourceBuilder WithWorkloadIdentity( + this IResourceBuilder builder); + + // Monitoring (matches existing pattern: WithAzureLogAnalyticsWorkspace) + public static IResourceBuilder WithAzureLogAnalyticsWorkspace( + this IResourceBuilder builder, + IResourceBuilder workspaceBuilder); + public static IResourceBuilder WithContainerInsights( + this IResourceBuilder builder, + IResourceBuilder? logAnalytics = null); + + // Container Registry + public static IResourceBuilder WithContainerRegistry( + this IResourceBuilder builder, + IResourceBuilder registry); + + // Helm configuration (delegates to inner KubernetesEnvironmentResource) + public static IResourceBuilder WithHelm( + this IResourceBuilder builder, + Action configure); + public static IResourceBuilder WithDashboard( + this IResourceBuilder builder); +} +``` + +**`AzureKubernetesInfrastructure`** (eventing subscriber, mirrors `AzureContainerAppsInfrastructure`): +```csharp +internal sealed class AzureKubernetesInfrastructure( + ILogger logger, + DistributedApplicationExecutionContext executionContext) + : IDistributedApplicationEventingSubscriber +{ + private async Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken ct) + { + var aksEnvironments = @event.Model.Resources + .OfType().ToArray(); + + foreach (var environment in aksEnvironments) + { + foreach (var r in @event.Model.GetComputeResources()) + { + var computeEnv = r.GetComputeEnvironment(); + if (computeEnv is not null && computeEnv != environment) + continue; + + // 1. Process workload identity annotations + // → Generate federated credentials in Bicep + // → Add ServiceAccount + labels to Helm chart + + // 2. Create KubernetesResource via inner environment + // (delegates to existing KubernetesInfrastructure) + + // 3. Add DeploymentTargetAnnotation + r.Annotations.Add(new DeploymentTargetAnnotation(environment) + { + ContainerRegistry = environment, + ComputeEnvironment = environment + }); + } + } + } +} +``` + +**Bicep output**: The `ConfigureAksInfrastructure` callback uses `Azure.Provisioning.Kubernetes` to produce: +- `ManagedCluster` with system-assigned or user-assigned identity for the control plane +- OIDC issuer enabled (required for workload identity) +- Workload identity enabled on the cluster +- Azure CNI or Kubenet network profile (based on VNet configuration) +- Container Insights add-on profile (if monitoring configured) +- Node pools with autoscaler configuration +- ACR pull role assignment for kubelet identity +- Container Registry (auto-created or explicit) + +### 2. Workload Identity Support + +Workload identity enables pods to authenticate to Azure services using federated credentials without storing secrets. This is implemented by honoring the shared `AppIdentityAnnotation` from `Aspire.Hosting.Azure` — the same mechanism used by ACA and AppService. + +**How it works**: +1. `AzureResourcePreparer` auto-creates a per-resource managed identity when a compute resource references Azure services (e.g., `WithReference(blobStorage)`) +2. It adds `AppIdentityAnnotation` with the identity to the resource +3. Users can override with `WithAzureUserAssignedIdentity(myIdentity)` to supply their own identity +4. `AzureKubernetesInfrastructure` detects `AppIdentityAnnotation` and generates: + - A K8s `ServiceAccount` with `azure.workload.identity/client-id` annotation + - `serviceAccountName` on the pod spec + - `azure.workload.identity/use: "true"` pod label + - Federated identity credential in AKS Bicep module + +**User API** (same as ACA): +```csharp +// Automatic — identity auto-created when referencing Azure resources +builder.AddProject() + .WithComputeEnvironment(aks) + .WithReference(blobStorage); // gets identity + workload identity + role assignments + +// Explicit — bring your own identity +var identity = builder.AddAzureUserAssignedIdentity("api-identity"); +builder.AddProject() + .WithComputeEnvironment(aks) + .WithAzureUserAssignedIdentity(identity); +``` + +**Generated Bicep** (federated credential): +```bicep +resource federatedCredential 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = { + parent: identity + name: '${resourceName}-fedcred' + properties: { + issuer: aksCluster.properties.oidcIssuerProfile.issuerURL + subject: 'system:serviceaccount:${namespace}:${resourceName}-sa' + audiences: ['api://AzureADTokenExchange'] + } +} +``` + +**Generated Helm chart** (ServiceAccount + pod template): +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: apiservice-sa + annotations: + azure.workload.identity/client-id: {{ .Values.parameters.apiservice.identityClientId }} + labels: + azure.workload.identity/use: "true" +--- +# In the Deployment pod template: +spec: + serviceAccountName: apiservice-sa + template: + metadata: + labels: + azure.workload.identity/use: "true" +``` + +### 3. Monitoring Integration + +**Goal**: When monitoring is enabled on the AKS environment, provision: +- Container Insights (via AKS addon profile) with Log Analytics workspace +- Azure Monitor metrics profile (managed Prometheus) +- Optional: Azure Managed Grafana dashboard +- Optional: Application Insights for application-level telemetry + +**Design** (matches `WithAzureLogAnalyticsWorkspace` pattern from Container Apps): +```csharp +// Option 1: Explicit workspace (matches Container Apps naming exactly) +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithAzureLogAnalyticsWorkspace(logAnalytics); + +// Option 2: Enable Container Insights (auto-creates workspace if not provided) +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithContainerInsights(); + +// Option 3: Both — explicit workspace + Container Insights addon +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithContainerInsights(logAnalytics); +``` + +**Bicep additions**: +- `addonProfiles.omsagent.enabled = true` with Log Analytics workspace ID +- `azureMonitorProfile.metrics.enabled = true` for managed Prometheus +- Data collection rule for container insights +- Optional: `AzureMonitorWorkspaceResource` for managed Prometheus + +**OTLP integration**: The existing Kubernetes publishing already injects `OTEL_EXPORTER_OTLP_ENDPOINT`. For AKS, we can optionally route OTLP to Application Insights via the connection string environment variable. + +### 4. VNet Integration + +AKS needs a subnet for its nodes. Unlike Container Apps, AKS does **not** use subnet delegation — it uses plain (non-delegated) subnets. The API is `WithSubnet()` (not `WithDelegatedSubnet()`). + +**Design**: +```csharp +var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); +var defaultSubnet = vnet.AddSubnet("default", "10.0.0.0/22"); +var gpuSubnet = vnet.AddSubnet("gpu-subnet", "10.0.4.0/24"); + +// Environment-level subnet (applies to all pools by default) +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithSubnet(defaultSubnet); + +// Per-pool subnet override +var gpuPool = aks.AddNodePool("gpu", AksNodeVmSizes.GpuAccelerated.StandardNC6sV3, 0, 5) + .WithSubnet(gpuSubnet); +``` + +**Bicep**: Environment-level subnet → `subnetId` parameter. Per-pool subnets → `subnetId_{poolName}` parameters. Each agent pool profile uses its own subnet if set, else the environment default. + +**Network profile**: Azure CNI is auto-configured when any subnet is set. + +**Private cluster support**: +```csharp +public static IResourceBuilder AsPrivateCluster( + this IResourceBuilder builder) +{ + // Enable private cluster (API server behind private endpoint) + // Requires a delegated subnet to be configured + // Sets apiServerAccessProfile.enablePrivateCluster = true +} +``` + +### 5. Network Perimeter Support + +AKS backing Azure services (SQL, Storage, Key Vault) should be accessible via private endpoints within the cluster's VNet. + +**This largely uses existing infrastructure**: +```csharp +// User code in AppHost +var vnet = builder.AddAzureVirtualNetwork("vnet"); +var aksSubnet = vnet.AddSubnet("aks-subnet", "10.0.0.0/22"); +var peSubnet = vnet.AddSubnet("pe-subnet", "10.0.4.0/24"); + +var aks = builder.AddAzureKubernetesService("aks") + .WithVirtualNetwork(aksSubnet); + +var sql = builder.AddAzureSqlServer("sql") + .WithPrivateEndpoint(peSubnet); // existing pattern + +// The SQL private endpoint is in the same VNet as AKS +// DNS resolution via Private DNS Zone (existing pattern) enables pod → SQL connectivity +``` + +**New consideration**: When AKS is configured with a VNet and backing services have private endpoints, the AKS infrastructure should verify or configure: +- Private DNS Zone links to the AKS VNet (so pods can resolve private endpoint DNS) +- This may need a new extension or automatic wiring + +### 6. Deployment Pipeline Integration + +Since `AzureKubernetesEnvironmentResource` unifies both Azure provisioning and K8s deployment, the pipeline is a superset of both: + +```text +[Azure Provisioning Phase] [Kubernetes Deployment Phase] +1. Generate Bicep (AKS + ACR + 4. Publish Helm chart + identity + fedcreds) 5. Get kubeconfig from AKS (az aks get-credentials) +2. Deploy Bicep via azd 6. Push images to ACR +3. Capture outputs (OIDC URL, 7. Prepare Helm values (resolve AKS outputs) + ACR endpoint, etc.) 8. helm upgrade --install + 9. Print summary + 10. (Optional) Uninstall +``` + +The Azure provisioning happens first (via `AzureEnvironmentResource` / `AzureProvisioner`), then the Kubernetes Helm deployment pipeline steps execute against the provisioned cluster. The kubeconfig step bridges the two phases — it uses the AKS cluster name from Bicep outputs to call `az aks get-credentials`. + +This is implemented by adding AKS-specific `DeploymentEngineStepsFactory` entries to the inner `KubernetesEnvironmentResource`: +```csharp +// In AddAzureKubernetesEnvironment, after AKS provisioning completes: +resource.KubernetesEnvironment.AddDeploymentEngineStep( + "get-kubeconfig", + async (context, ct) => + { + // Use AKS outputs to fetch kubeconfig + var clusterName = await resource.NameOutputReference.GetValueAsync(ct); + var resourceGroup = await resource.ResourceGroupOutput.GetValueAsync(ct); + // az aks get-credentials --resource-group {rg} --name {name} + }); +``` + +### 7. Container Registry Integration + +AKS needs a container registry for application images. Options: +1. **Auto-create ACR** when AKS is added (like Container Apps does) +2. **Bring your own ACR** via `.WithContainerRegistry()` +3. **Use existing ACR** via `AsExisting()` pattern + +```csharp +// Auto-create (default) +var aks = builder.AddAzureKubernetesService("aks"); +// → auto-creates ACR, attaches AcrPull role to kubelet identity + +// Explicit +var acr = builder.AddAzureContainerRegistry("acr"); +var aks = builder.AddAzureKubernetesService("aks") + .WithContainerRegistry(acr); +``` + +**Role assignment**: The AKS kubelet managed identity needs `AcrPull` role on the registry. + +## Open Questions + +1. **`Azure.Provisioning.Kubernetes` readiness**: The package is at v1.0.0-beta.3. We need to verify it has the types we need (`ManagedCluster`, `AgentPool`, `OidcIssuerProfile`, `WorkloadIdentity` flags, etc.) and assess stability risk. + +2. **Existing cluster support**: Should we support `AsExisting()` for AKS (reference a pre-provisioned cluster)? + - **Recommendation**: Yes, this is a common scenario. Use the established `ExistingAzureResourceAnnotation` pattern. + +3. **Managed Grafana**: Should `WithMonitoring()` also provision Azure Managed Grafana? + - Could be a separate `.WithGrafana()` extension to keep it opt-in. + +4. **Ingress controller**: Should Aspire configure an ingress controller (NGINX, Traefik, or Application Gateway Ingress Controller)? + - Application Gateway Ingress Controller (AGIC) would be the Azure-native choice. + - Could be opt-in via `.WithApplicationGatewayIngress()`. + +5. **DNS integration**: Should external endpoints auto-configure Azure DNS zones? + - Probably out of scope for v1. + +6. **Deployment mode**: For publish, should AKS support work with `aspire publish` only, or also `aspire run` (local dev with AKS)? + - Recommendation: `aspire publish` first. Local dev uses the generic K8s environment with local/kind clusters. + +7. **Multi-cluster**: Should we support multiple AKS environments in one AppHost? + - The `KubernetesEnvironmentResource` model already supports this conceptually. + +8. **Helm config delegation**: How cleanly can `WithHelm()` / `WithDashboard()` be forwarded from `AzureKubernetesEnvironmentResource` to the inner `KubernetesEnvironmentResource`? Should the inner resource be exposed or kept fully internal? + +## Implementation Status + +### ✅ Implemented + +#### Phase 1: Unified AKS Environment (Foundation) +- ✅ `Aspire.Hosting.Azure.Kubernetes` package created +- ✅ `AzureKubernetesEnvironmentResource` — extends `AzureProvisioningResource`, implements `IAzureComputeEnvironmentResource`, `IAzureNspAssociationTarget` +- ✅ `AddAzureKubernetesEnvironment()` entry point — calls `AddKubernetesEnvironment()` internally +- ✅ `AzureKubernetesInfrastructure` eventing subscriber +- ✅ Hand-crafted Bicep generation (not Azure.Provisioning SDK — `Azure.Provisioning.ContainerService` not in internal feeds) +- ✅ ACR auto-creation + AcrPull role assignment in Bicep +- ✅ Kubeconfig retrieval via `az aks get-credentials` to isolated temp file +- ✅ Multi-environment support (scoped Helm chart names, per-env kubeconfig) +- ✅ `WithVersion()`, `WithSkuTier()`, `WithContainerRegistry()` +- ✅ `AsPrivateCluster()` — sets `apiServerAccessProfile.enablePrivateCluster` +- ✅ Push step dependency wiring for container image builds + +#### Phase 2: Workload Identity +- ✅ Honors `AppIdentityAnnotation` from `Aspire.Hosting.Azure` (same mechanism as ACA/AppService) +- ✅ Auto-identity via `AzureResourcePreparer` when resources reference Azure services +- ✅ Override with `WithAzureUserAssignedIdentity(identity)` (standard API) +- ✅ ServiceAccount YAML generation with `azure.workload.identity/client-id` annotation +- ✅ Pod label `azure.workload.identity/use: "true"` on pod template +- ✅ `serviceAccountName` set on pod spec +- ✅ Federated identity credential Bicep generation per workload +- ✅ Identity `clientId` wired as deferred Helm value (resolved at deploy time) +- ✅ `ServiceAccountV1` resource added to `Aspire.Hosting.Kubernetes` +- ❌ ~~`AksWorkloadIdentityAnnotation`~~ — **Removed** (redundant with `AppIdentityAnnotation`) +- ❌ ~~`WithAzureWorkloadIdentity()`~~ — **Removed** (standard `WithAzureUserAssignedIdentity` works) + +#### Phase 3: Networking +- ✅ `WithSubnet()` (NOT `WithDelegatedSubnet` — AKS doesn't support subnet delegation) +- ✅ Per-node-pool subnet support via `WithSubnet()` on `AksNodePoolResource` +- ✅ Azure CNI network profile auto-configured when subnet is set +- ✅ `AsPrivateCluster()` for private API server +- ❌ AKS does NOT implement `IAzureDelegatedSubnetResource` (intentionally — AKS uses plain subnets) + +#### Node Pools (not in original spec) +- ✅ Base `KubernetesNodePoolResource` in `Aspire.Hosting.Kubernetes` (cloud-agnostic) +- ✅ `AksNodePoolResource` extends base with VM size, scaling, mode config +- ✅ `AddNodePool()` on both K8s and AKS environments +- ✅ `WithNodePool()` schedules workloads via `nodeSelector` on pod spec +- ✅ `AksNodeVmSizes` constants class (GeneralPurpose, ComputeOptimized, MemoryOptimized, GpuAccelerated, StorageOptimized, Burstable, Arm) +- ✅ `GenVmSizes.cs` tool + `update-azure-vm-sizes.yml` monthly workflow +- ✅ Default "workload" user pool auto-created if none configured + +#### IValueProvider Resolution (not in original spec) +- ✅ Azure resource connection strings and endpoints resolved at deploy time +- ✅ Composite expressions (e.g., `Endpoint={storage.outputs.blobEndpoint};ContainerName=photos`) handled +- ✅ Phase 4 in HelmDeploymentEngine for generic `IValueProvider` resolution + +### 🔲 Not Yet Implemented + +#### Monitoring (Phase 4) — Bicep not emitted +- 🔲 `WithContainerInsights()` and `WithAzureLogAnalyticsWorkspace()` **exist as APIs** but the Bicep generation does NOT emit: + - Container Insights addon profile (`addonProfiles.omsagent`) + - Azure Monitor metrics profile (managed Prometheus) + - Data collection rules + - Application Insights OTLP integration + +#### Helm/Dashboard delegation +- 🔲 `WithHelm()` and `WithDashboard()` are not exposed on `AzureKubernetesEnvironmentResource` + - They work on the inner `KubernetesEnvironmentResource` but users can't access them from the AKS builder + +#### AsExisting() support +- 🔲 `AsExisting()` for referencing pre-provisioned AKS clusters + +#### Private DNS Zone auto-linking +- 🔲 When backing services have private endpoints in same VNet as AKS, Private DNS Zones should be auto-linked + +#### IAzureContainerRegistry interface +- 🔲 AKS resource does not implement `IAzureContainerRegistry` (ACR outputs not exposed via standard interface) + +#### Ingress controller +- 🔲 Application Gateway Ingress Controller (AGIC) or other ingress support + +#### Managed Prometheus/Grafana +- 🔲 Azure Monitor workspace for managed Prometheus +- 🔲 Azure Managed Grafana provisioning + +## Key Design Changes from Original Spec + +1. **Bicep generation**: Uses hand-crafted `StringBuilder` via `GetBicepTemplateString()` override, NOT `Azure.Provisioning.ContainerService` SDK (package not available in internal NuGet feeds) +2. **Workload identity**: Uses shared `AppIdentityAnnotation` from `Aspire.Hosting.Azure`, not AKS-specific annotation. Same mechanism as ACA/AppService. +3. **Subnet integration**: `WithSubnet()` not `WithDelegatedSubnet()` — AKS uses plain subnets, not delegated ones +4. **Node pools**: First-class resources with `AddNodePool()` returning `IResourceBuilder`, `WithNodePool()` for scheduling, per-pool subnets, `AksNodeVmSizes` constants +5. **Multi-environment**: Full support for multiple AKS environments with scoped chart names and isolated kubeconfigs + +## Dependencies / Prerequisites + +- ~~`Azure.Provisioning.Kubernetes`~~ — Not used (hand-crafted Bicep instead) +- `Azure.Provisioning.ContainerRegistry` (for ACR resource type reference) +- `Azure.Provisioning.OperationalInsights` (for Log Analytics workspace type) +- `Aspire.Hosting.Kubernetes` (the generic K8s package) +- `Aspire.Hosting.Azure` (for `AppIdentityAnnotation`, `AzureProvisioningResource`, etc.) +- `Aspire.Hosting.Azure.Network` (for subnet, VNet, NSP types) +- `Aspire.Hosting.Azure.ContainerRegistry` (for ACR auto-creation) + +## Testing + +- 31 AKS unit tests passing (extensions + infrastructure) +- 88 K8s base tests passing +- Manual E2E validation against live Azure clusters diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksNetworkProfile.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksNetworkProfile.cs new file mode 100644 index 00000000000..ca707af77ed --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksNetworkProfile.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Network profile configuration for an AKS cluster. +/// +internal sealed class AksNetworkProfile +{ + /// + /// Gets or sets the network plugin. Defaults to "azure" for Azure CNI. + /// + public string NetworkPlugin { get; set; } = "azure"; + + /// + /// Gets or sets the network policy. Defaults to "calico". + /// + public string? NetworkPolicy { get; set; } = "calico"; + + /// + /// Gets or sets the service CIDR. + /// + public string ServiceCidr { get; set; } = "10.0.4.0/22"; + + /// + /// Gets or sets the DNS service IP address. + /// + public string DnsServiceIP { get; set; } = "10.0.4.10"; +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolConfig.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolConfig.cs new file mode 100644 index 00000000000..5fc3016f1f8 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolConfig.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Configuration for an AKS node pool. +/// +/// The name of the node pool. +/// The VM size for nodes in the pool. +/// The minimum number of nodes. +/// The maximum number of nodes. +/// The mode of the node pool. +public sealed record AksNodePoolConfig( + string Name, + string VmSize, + int MinCount, + int MaxCount, + AksNodePoolMode Mode); + +/// +/// Specifies the mode of an AKS node pool. +/// +public enum AksNodePoolMode +{ + /// + /// System node pool for hosting system pods. + /// + System, + + /// + /// User node pool for hosting application workloads. + /// + User +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolResource.cs new file mode 100644 index 00000000000..e6fb6a17999 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolResource.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.Kubernetes; + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Represents an AKS node pool with Azure-specific configuration such as VM size and autoscaling. +/// Extends the base with provisioning configuration +/// that is used to generate Azure Bicep for the AKS agent pool profile. +/// +/// The name of the node pool resource. +/// The Azure-specific node pool configuration. +/// The parent AKS environment resource. +public class AksNodePoolResource( + string name, + AksNodePoolConfig config, + AzureKubernetesEnvironmentResource parent) : KubernetesNodePoolResource(name, parent.KubernetesEnvironment) +{ + /// + /// Gets the parent AKS environment resource. + /// + public AzureKubernetesEnvironmentResource AksParent { get; } = parent ?? throw new ArgumentNullException(nameof(parent)); + + /// + /// Gets the Azure-specific node pool configuration. + /// + public AksNodePoolConfig Config { get; } = config ?? throw new ArgumentNullException(nameof(config)); +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksNodeVmSizes.Generated.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksNodeVmSizes.Generated.cs new file mode 100644 index 00000000000..40b5524311c --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksNodeVmSizes.Generated.cs @@ -0,0 +1,274 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// +// This file is generated by the GenVmSizes tool. Do not edit manually. + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Provides well-known Azure VM size constants for use with AKS node pools. +/// +/// +/// This class is auto-generated. To update, run the GenVmSizes tool: +/// dotnet run --project src/Aspire.Hosting.Azure.Kubernetes/tools GenVmSizes.cs +/// +public static partial class AksNodeVmSizes +{ + /// + /// General purpose VM sizes optimized for balanced CPU-to-memory ratio. + /// + public static class GeneralPurpose + { + /// + /// Standard_D2s_v5 — 2 vCPUs, 8 GB RAM, Premium SSD. + /// + public const string StandardD2sV5 = "Standard_D2s_v5"; + + /// + /// Standard_D4s_v5 — 4 vCPUs, 16 GB RAM, Premium SSD. + /// + public const string StandardD4sV5 = "Standard_D4s_v5"; + + /// + /// Standard_D8s_v5 — 8 vCPUs, 32 GB RAM, Premium SSD. + /// + public const string StandardD8sV5 = "Standard_D8s_v5"; + + /// + /// Standard_D16s_v5 — 16 vCPUs, 64 GB RAM, Premium SSD. + /// + public const string StandardD16sV5 = "Standard_D16s_v5"; + + /// + /// Standard_D32s_v5 — 32 vCPUs, 128 GB RAM, Premium SSD. + /// + public const string StandardD32sV5 = "Standard_D32s_v5"; + + /// + /// Standard_D2s_v6 — 2 vCPUs, 8 GB RAM, Premium SSD. + /// + public const string StandardD2sV6 = "Standard_D2s_v6"; + + /// + /// Standard_D4s_v6 — 4 vCPUs, 16 GB RAM, Premium SSD. + /// + public const string StandardD4sV6 = "Standard_D4s_v6"; + + /// + /// Standard_D8s_v6 — 8 vCPUs, 32 GB RAM, Premium SSD. + /// + public const string StandardD8sV6 = "Standard_D8s_v6"; + + /// + /// Standard_D2as_v5 — 2 vCPUs, 8 GB RAM, Premium SSD, AMD processor. + /// + public const string StandardD2asV5 = "Standard_D2as_v5"; + + /// + /// Standard_D4as_v5 — 4 vCPUs, 16 GB RAM, Premium SSD, AMD processor. + /// + public const string StandardD4asV5 = "Standard_D4as_v5"; + + /// + /// Standard_D8as_v5 — 8 vCPUs, 32 GB RAM, Premium SSD, AMD processor. + /// + public const string StandardD8asV5 = "Standard_D8as_v5"; + } + + /// + /// Compute optimized VM sizes with high CPU-to-memory ratio. + /// + public static class ComputeOptimized + { + /// + /// Standard_F2s_v2 — 2 vCPUs, 4 GB RAM, Premium SSD. + /// + public const string StandardF2sV2 = "Standard_F2s_v2"; + + /// + /// Standard_F4s_v2 — 4 vCPUs, 8 GB RAM, Premium SSD. + /// + public const string StandardF4sV2 = "Standard_F4s_v2"; + + /// + /// Standard_F8s_v2 — 8 vCPUs, 16 GB RAM, Premium SSD. + /// + public const string StandardF8sV2 = "Standard_F8s_v2"; + + /// + /// Standard_F16s_v2 — 16 vCPUs, 32 GB RAM, Premium SSD. + /// + public const string StandardF16sV2 = "Standard_F16s_v2"; + } + + /// + /// Memory optimized VM sizes with high memory-to-CPU ratio. + /// + public static class MemoryOptimized + { + /// + /// Standard_E2s_v5 — 2 vCPUs, 16 GB RAM, Premium SSD. + /// + public const string StandardE2sV5 = "Standard_E2s_v5"; + + /// + /// Standard_E4s_v5 — 4 vCPUs, 32 GB RAM, Premium SSD. + /// + public const string StandardE4sV5 = "Standard_E4s_v5"; + + /// + /// Standard_E8s_v5 — 8 vCPUs, 64 GB RAM, Premium SSD. + /// + public const string StandardE8sV5 = "Standard_E8s_v5"; + + /// + /// Standard_E16s_v5 — 16 vCPUs, 128 GB RAM, Premium SSD. + /// + public const string StandardE16sV5 = "Standard_E16s_v5"; + + /// + /// Standard_E2as_v5 — 2 vCPUs, 16 GB RAM, Premium SSD, AMD processor. + /// + public const string StandardE2asV5 = "Standard_E2as_v5"; + + /// + /// Standard_E4as_v5 — 4 vCPUs, 32 GB RAM, Premium SSD, AMD processor. + /// + public const string StandardE4asV5 = "Standard_E4as_v5"; + } + + /// + /// GPU-enabled VM sizes for compute-intensive and ML workloads. + /// + public static class GpuAccelerated + { + /// + /// Standard_NC6s_v3 — 6 vCPUs, 112 GB RAM, 1 GPU (NVIDIA V100). + /// + public const string StandardNC6sV3 = "Standard_NC6s_v3"; + + /// + /// Standard_NC12s_v3 — 12 vCPUs, 224 GB RAM, 2 GPUs (NVIDIA V100). + /// + public const string StandardNC12sV3 = "Standard_NC12s_v3"; + + /// + /// Standard_NC24s_v3 — 24 vCPUs, 448 GB RAM, 4 GPUs (NVIDIA V100). + /// + public const string StandardNC24sV3 = "Standard_NC24s_v3"; + + /// + /// Standard_NC4as_T4_v3 — 4 vCPUs, 28 GB RAM, 1 GPU (NVIDIA T4). + /// + public const string StandardNC4asT4V3 = "Standard_NC4as_T4_v3"; + + /// + /// Standard_NC8as_T4_v3 — 8 vCPUs, 56 GB RAM, 1 GPU (NVIDIA T4). + /// + public const string StandardNC8asT4V3 = "Standard_NC8as_T4_v3"; + + /// + /// Standard_NC16as_T4_v3 — 16 vCPUs, 110 GB RAM, 1 GPU (NVIDIA T4). + /// + public const string StandardNC16asT4V3 = "Standard_NC16as_T4_v3"; + + /// + /// Standard_NC24ads_A100_v4 — 24 vCPUs, 220 GB RAM, 1 GPU (NVIDIA A100 80GB). + /// + public const string StandardNC24adsA100V4 = "Standard_NC24ads_A100_v4"; + + /// + /// Standard_NC48ads_A100_v4 — 48 vCPUs, 440 GB RAM, 2 GPUs (NVIDIA A100 80GB). + /// + public const string StandardNC48adsA100V4 = "Standard_NC48ads_A100_v4"; + } + + /// + /// Storage optimized VM sizes with high disk throughput and I/O. + /// + public static class StorageOptimized + { + /// + /// Standard_L8s_v3 — 8 vCPUs, 64 GB RAM, Premium SSD, high local NVMe storage. + /// + public const string StandardL8sV3 = "Standard_L8s_v3"; + + /// + /// Standard_L16s_v3 — 16 vCPUs, 128 GB RAM, Premium SSD, high local NVMe storage. + /// + public const string StandardL16sV3 = "Standard_L16s_v3"; + + /// + /// Standard_L32s_v3 — 32 vCPUs, 256 GB RAM, Premium SSD, high local NVMe storage. + /// + public const string StandardL32sV3 = "Standard_L32s_v3"; + } + + /// + /// Burstable VM sizes for cost-effective workloads with variable CPU usage. + /// + public static class Burstable + { + /// + /// Standard_B2s — 2 vCPUs, 4 GB RAM. + /// + public const string StandardB2s = "Standard_B2s"; + + /// + /// Standard_B4ms — 4 vCPUs, 16 GB RAM. + /// + public const string StandardB4ms = "Standard_B4ms"; + + /// + /// Standard_B8ms — 8 vCPUs, 32 GB RAM. + /// + public const string StandardB8ms = "Standard_B8ms"; + + /// + /// Standard_B2s_v2 — 2 vCPUs, 8 GB RAM. + /// + public const string StandardB2sV2 = "Standard_B2s_v2"; + + /// + /// Standard_B4s_v2 — 4 vCPUs, 16 GB RAM. + /// + public const string StandardB4sV2 = "Standard_B4s_v2"; + } + + /// + /// Arm-based VM sizes with high energy efficiency and price-performance. + /// + public static class Arm + { + /// + /// Standard_D2pds_v5 — 2 vCPUs, 8 GB RAM, Ampere Altra Arm processor. + /// + public const string StandardD2pdsV5 = "Standard_D2pds_v5"; + + /// + /// Standard_D4pds_v5 — 4 vCPUs, 16 GB RAM, Ampere Altra Arm processor. + /// + public const string StandardD4pdsV5 = "Standard_D4pds_v5"; + + /// + /// Standard_D8pds_v5 — 8 vCPUs, 32 GB RAM, Ampere Altra Arm processor. + /// + public const string StandardD8pdsV5 = "Standard_D8pds_v5"; + + /// + /// Standard_D16pds_v5 — 16 vCPUs, 64 GB RAM, Ampere Altra Arm processor. + /// + public const string StandardD16pdsV5 = "Standard_D16pds_v5"; + + /// + /// Standard_E2pds_v5 — 2 vCPUs, 16 GB RAM, Ampere Altra Arm processor. + /// + public const string StandardE2pdsV5 = "Standard_E2pds_v5"; + + /// + /// Standard_E4pds_v5 — 4 vCPUs, 32 GB RAM, Ampere Altra Arm processor. + /// + public const string StandardE4pdsV5 = "Standard_E4pds_v5"; + } +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksSkuTier.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksSkuTier.cs new file mode 100644 index 00000000000..f2f10865ac4 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksSkuTier.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Specifies the SKU tier for an AKS cluster. +/// +public enum AksSkuTier +{ + /// + /// Free tier with no SLA. + /// + Free, + + /// + /// Standard tier with financially backed SLA. + /// + Standard, + + /// + /// Premium tier with mission-critical features. + /// + Premium +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksSubnetAnnotation.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksSubnetAnnotation.cs new file mode 100644 index 00000000000..9bdf97dbee3 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksSubnetAnnotation.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.ApplicationModel; + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Annotation that stores a subnet ID reference for AKS VNet integration. +/// Unlike DelegatedSubnetAnnotation, this does NOT add a service delegation +/// to the subnet — AKS uses plain (non-delegated) subnets for node pools. +/// +internal sealed class AksSubnetAnnotation(BicepOutputReference subnetId) : IResourceAnnotation +{ + /// + /// Gets the subnet ID output reference. + /// + public BicepOutputReference SubnetId { get; } = subnetId; +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj new file mode 100644 index 00000000000..1b8c765745f --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj @@ -0,0 +1,33 @@ + + + + $(DefaultTargetFramework) + true + true + false + aspire integration hosting azure kubernetes aks + Azure Kubernetes Service (AKS) resource types for Aspire. + $(SharedDir)Azure_256x.png + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs new file mode 100644 index 00000000000..fd2fe5dc43d --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -0,0 +1,580 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable ASPIREPIPELINES001 // Pipeline step types used for push/deploy dependency wiring +#pragma warning disable ASPIREAZURE001 // AzureEnvironmentResource.ProvisionInfrastructureStepName for pipeline ordering +#pragma warning disable ASPIREAZURE003 // AzureSubnetResource used in WithSubnet extensions + +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Azure; +using Aspire.Hosting.Azure.Kubernetes; +using Aspire.Hosting.Kubernetes; +using Aspire.Hosting.Kubernetes.Extensions; +using Aspire.Hosting.Lifecycle; +using Aspire.Hosting.Pipelines; +using Azure.Provisioning; +using Azure.Provisioning.Authorization; +using Azure.Provisioning.ContainerRegistry; +using Azure.Provisioning.ContainerService; +using Azure.Provisioning.Expressions; +using Azure.Provisioning.Resources; +using Azure.Provisioning.Roles; +using Microsoft.Extensions.DependencyInjection; + +namespace Aspire.Hosting; + +/// +/// Provides extension methods for adding Azure Kubernetes Service (AKS) environments to the application model. +/// +public static class AzureKubernetesEnvironmentExtensions +{ + /// + /// Adds an Azure Kubernetes Service (AKS) environment to the distributed application. + /// This provisions an AKS cluster and configures it as a Kubernetes compute environment. + /// + /// The . + /// The name of the AKS environment resource. + /// A reference to the . + /// + /// This method internally creates a Kubernetes environment for Helm-based deployment + /// and provisions an AKS cluster via Azure Bicep. It combines the functionality of + /// AddKubernetesEnvironment with Azure-specific provisioning. + /// + /// + /// + /// var aks = builder.AddAzureKubernetesEnvironment("aks"); + /// + /// + [AspireExport(Description = "Adds an Azure Kubernetes Service environment resource")] + public static IResourceBuilder AddAzureKubernetesEnvironment( + this IDistributedApplicationBuilder builder, + [ResourceName] string name) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrEmpty(name); + + // Set up Azure provisioning infrastructure + builder.AddAzureProvisioning(); + builder.Services.Configure( + o => o.SupportsTargetedRoleAssignments = true); + + // Register the AKS-specific infrastructure eventing subscriber + builder.Services.TryAddEventingSubscriber(); + + // Create the inner KubernetesEnvironmentResource via the public API. + // This registers KubernetesInfrastructure, creates the resource with + // Helm chart name/dashboard, adds it to the model, and sets up the + // default Helm deployment engine. + var k8sEnvBuilder = builder.AddKubernetesEnvironment($"{name}-k8s"); + + // Scope the Helm chart name to this AKS environment to avoid + // conflicts when multiple environments deploy to the same cluster + // or when re-deploying with different environment names. + k8sEnvBuilder.Resource.HelmChartName = $"{builder.Environment.ApplicationName}-{name}".ToHelmChartName(); + + // Create the unified AKS environment resource + var resource = new AzureKubernetesEnvironmentResource(name, ConfigureAksInfrastructure); + resource.KubernetesEnvironment = k8sEnvBuilder.Resource; + + // Set the parent so KubernetesInfrastructure matches resources that use + // WithComputeEnvironment(aksEnv) — the inner K8s env checks both itself + // and its parent when filtering compute resources. + k8sEnvBuilder.Resource.ParentComputeEnvironment = resource; + + if (builder.ExecutionContext.IsRunMode) + { + return builder.CreateResourceBuilder(resource); + } + + // Auto-create a default Azure Container Registry for image push/pull. + // Wire it to the inner K8s environment immediately so KubernetesInfrastructure + // can discover it during BeforeStartEvent (both subscribers run during the same + // event, so we can't rely on annotation ordering during the event). + var defaultRegistry = builder.AddAzureContainerRegistry($"{name}-acr"); + resource.DefaultContainerRegistry = defaultRegistry.Resource; + k8sEnvBuilder.WithAnnotation(new ContainerRegistryReferenceAnnotation(defaultRegistry.Resource)); + + // Wire ACR name as a parameter on the AKS resource so the Bicep module + // can create an AcrPull role assignment for the kubelet identity. + // The publishing context will wire this as a parameter in main.bicep. + resource.Parameters["acrName"] = defaultRegistry.Resource.NameOutputReference; + + // Ensure push steps wait for ALL Azure provisioning to complete. Push steps + // call registry.Endpoint.GetValueAsync() which awaits the BicepOutputReference + // for loginServer — if the ACR hasn't been provisioned yet, this blocks. + // + // NOTE: The standard push step dependency wiring (pushSteps.DependsOn(buildSteps) + // and pushSteps.DependsOn(push-prereq)) from ProjectResource's PipelineConfigurationAnnotation + // may not resolve correctly when using Kubernetes compute environments, because + // context.GetSteps(resource, tag) may return empty if the resource reference doesn't + // match. We explicitly wire the dependencies here as a workaround. + k8sEnvBuilder.WithAnnotation(new PipelineConfigurationAnnotation(context => + { + var pushSteps = context.Steps + .Where(s => s.Tags.Contains(WellKnownPipelineTags.PushContainerImage)) + .ToList(); + + foreach (var pushStep in pushSteps) + { + // Ensure push waits for Azure provisioning (ACR endpoint resolution) + pushStep.DependsOn(AzureEnvironmentResource.ProvisionInfrastructureStepName); + + // Ensure push waits for push-prereq (ACR login) + pushStep.DependsOn(WellKnownPipelineSteps.PushPrereq); + + // Ensure push waits for its corresponding build step + var resourceName = pushStep.Resource?.Name; + if (resourceName is not null) + { + pushStep.DependsOn($"build-{resourceName}"); + } + } + })); + + return builder.AddResource(resource); + } + + /// + /// Adds a node pool to the AKS cluster. + /// + /// The AKS environment resource builder. + /// The name of the node pool. + /// The VM size for nodes. + /// The minimum node count for autoscaling. + /// The maximum node count for autoscaling. + /// A reference to the for the new node pool. + /// + /// The returned node pool resource can be passed to + /// on compute resources to schedule workloads on this pool. + /// + /// + /// + /// var aks = builder.AddAzureKubernetesEnvironment("aks"); + /// var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); + /// + /// builder.AddProject<MyApi>() + /// .WithNodePool(gpuPool); + /// + /// + [AspireExport(Description = "Adds a node pool to the AKS cluster")] + public static IResourceBuilder AddNodePool( + this IResourceBuilder builder, + [ResourceName] string name, + string vmSize, + int minCount, + int maxCount) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrEmpty(name); + ArgumentException.ThrowIfNullOrEmpty(vmSize); + ArgumentOutOfRangeException.ThrowIfNegative(minCount); + ArgumentOutOfRangeException.ThrowIfNegative(maxCount); + ArgumentOutOfRangeException.ThrowIfGreaterThan(minCount, maxCount); + + var config = new AksNodePoolConfig(name, vmSize, minCount, maxCount, AksNodePoolMode.User); + builder.Resource.NodePools.Add(config); + + var nodePool = new AksNodePoolResource(name, config, builder.Resource); + + if (builder.ApplicationBuilder.ExecutionContext.IsRunMode) + { + return builder.ApplicationBuilder.CreateResourceBuilder(nodePool); + } + + return builder.ApplicationBuilder.AddResource(nodePool) + .ExcludeFromManifest(); + } + + /// + /// Configures the AKS cluster to use a VNet subnet for node pool networking. + /// Unlike , this does NOT + /// add a service delegation to the subnet — AKS uses plain (non-delegated) subnets. + /// + /// The AKS environment resource builder. + /// The subnet to use for AKS node pools. + /// A reference to the for chaining. + /// + /// + /// var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + /// var subnet = vnet.AddSubnet("aks-subnet", "10.0.0.0/22"); + /// var aks = builder.AddAzureKubernetesEnvironment("aks") + /// .WithSubnet(subnet); + /// + /// + [AspireExport(Description = "Configures the AKS cluster to use a VNet subnet")] + public static IResourceBuilder WithSubnet( + this IResourceBuilder builder, + IResourceBuilder subnet) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(subnet); + + builder.WithAnnotation(new AksSubnetAnnotation(subnet.Resource.Id), ResourceAnnotationMutationBehavior.Replace); + return builder; + } + + /// + /// Configures a specific AKS node pool to use its own VNet subnet. + /// When applied, this node pool's subnet overrides the environment-level subnet + /// set via . + /// + /// The node pool resource builder. + /// The subnet to use for this node pool. + /// A reference to the for chaining. + /// + /// + /// var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + /// var defaultSubnet = vnet.AddSubnet("default", "10.0.0.0/22"); + /// var gpuSubnet = vnet.AddSubnet("gpu-subnet", "10.0.4.0/24"); + /// + /// var aks = builder.AddAzureKubernetesEnvironment("aks") + /// .WithSubnet(defaultSubnet); + /// + /// var gpuPool = aks.AddNodePool("gpu", AksNodeVmSizes.GpuAccelerated.StandardNC6sV3, 0, 5) + /// .WithSubnet(gpuSubnet); + /// + /// + [AspireExport("withAksNodePoolSubnet", Description = "Configures an AKS node pool to use a specific VNet subnet")] + public static IResourceBuilder WithSubnet( + this IResourceBuilder builder, + IResourceBuilder subnet) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(subnet); + + // Store the subnet on the node pool annotation for Bicep resolution + builder.WithAnnotation(new AksSubnetAnnotation(subnet.Resource.Id)); + + // Also register in the parent AKS environment's per-pool subnet dictionary + // so Bicep generation can emit the correct parameter per pool. + builder.Resource.AksParent.NodePoolSubnets[builder.Resource.Name] = subnet.Resource.Id; + + return builder; + } + + /// + /// Configures the AKS environment to use a specific Azure Container Registry for image storage. + /// When set, this replaces the auto-created default container registry. + /// + /// The AKS environment resource builder. + /// The Azure Container Registry resource builder. + /// A reference to the for chaining. + /// + /// If not called, a default Azure Container Registry is automatically created. + /// The registry endpoint is flowed to the inner Kubernetes environment so that + /// Helm deployments can push and pull images. + /// + [AspireExport(Description = "Configures the AKS environment to use a specific container registry")] + public static IResourceBuilder WithContainerRegistry( + this IResourceBuilder builder, + IResourceBuilder registry) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(registry); + + // Remove the default registry from the model if one was auto-created + if (builder.Resource.DefaultContainerRegistry is not null) + { + builder.ApplicationBuilder.Resources.Remove(builder.Resource.DefaultContainerRegistry); + builder.Resource.DefaultContainerRegistry = null; + } + + // Set the explicit registry via annotation on both the AKS environment + // and the inner K8s environment (so KubernetesInfrastructure finds it) + builder.WithAnnotation(new ContainerRegistryReferenceAnnotation(registry.Resource)); + builder.Resource.KubernetesEnvironment.Annotations.Add( + new ContainerRegistryReferenceAnnotation(registry.Resource)); + + // Update the acrName parameter to reference the explicit registry's output + // (replaces the default ACR reference set during AddAzureKubernetesEnvironment) + builder.Resource.Parameters["acrName"] = registry.Resource.NameOutputReference; + + return builder; + } + + /// + /// Enables workload identity on the AKS environment, allowing pods to authenticate + /// to Azure services using federated credentials. + /// + /// The resource builder. + /// A reference to the for chaining. + /// + /// This ensures the AKS cluster is configured with OIDC issuer and workload identity enabled. + /// Workload identity is automatically wired when compute resources have an , + /// which is added by WithAzureUserAssignedIdentity or auto-created by AzureResourcePreparer. + /// + [AspireExport(Description = "Enables workload identity on the AKS cluster")] + public static IResourceBuilder WithWorkloadIdentity( + this IResourceBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Resource.OidcIssuerEnabled = true; + builder.Resource.WorkloadIdentityEnabled = true; + return builder; + } + + private static void ConfigureAksInfrastructure(AzureResourceInfrastructure infrastructure) + { + var aksResource = (AzureKubernetesEnvironmentResource)infrastructure.AspireResource; + + var skuTier = aksResource.SkuTier switch + { + AksSkuTier.Free => ManagedClusterSkuTier.Free, + AksSkuTier.Standard => ManagedClusterSkuTier.Standard, + AksSkuTier.Premium => ManagedClusterSkuTier.Premium, + _ => ManagedClusterSkuTier.Free + }; + + // Create the AKS managed cluster + var aks = new ContainerServiceManagedCluster(aksResource.GetBicepIdentifier(), + ContainerServiceManagedCluster.ResourceVersions.V2025_03_01) + { + ClusterIdentity = new ManagedClusterIdentity + { + ResourceIdentityType = ManagedServiceIdentityType.SystemAssigned + }, + Sku = new ManagedClusterSku + { + Name = ManagedClusterSkuName.Base, + Tier = skuTier + }, + DnsPrefix = $"{aksResource.Name}-dns", + Tags = { { "aspire-resource-name", aksResource.Name } } + }; + + if (aksResource.KubernetesVersion is not null) + { + aks.KubernetesVersion = aksResource.KubernetesVersion; + } + + // Agent pool profiles + var hasDefaultSubnet = aksResource.TryGetLastAnnotation(out var subnetAnnotation); + ProvisioningParameter? defaultSubnetParam = null; + + if (hasDefaultSubnet) + { + defaultSubnetParam = new ProvisioningParameter("subnetId", typeof(string)); + infrastructure.Add(defaultSubnetParam); + aksResource.Parameters["subnetId"] = subnetAnnotation!.SubnetId; + } + + // Per-pool subnet parameters + var poolSubnetParams = new Dictionary(); + foreach (var (poolName, poolSubnetRef) in aksResource.NodePoolSubnets) + { + var paramName = $"subnetId_{poolName}"; + var param = new ProvisioningParameter(paramName, typeof(string)); + infrastructure.Add(param); + poolSubnetParams[poolName] = param; + aksResource.Parameters[paramName] = poolSubnetRef; + } + + foreach (var pool in aksResource.NodePools) + { + var mode = pool.Mode switch + { + AksNodePoolMode.System => AgentPoolMode.System, + AksNodePoolMode.User => AgentPoolMode.User, + _ => AgentPoolMode.User + }; + + var agentPool = new ManagedClusterAgentPoolProfile + { + Name = pool.Name, + VmSize = pool.VmSize, + MinCount = pool.MinCount, + MaxCount = pool.MaxCount, + Count = pool.MinCount, + EnableAutoScaling = true, + Mode = mode, + OSType = ContainerServiceOSType.Linux, + }; + + // Per-pool subnet override, else environment default + if (poolSubnetParams.TryGetValue(pool.Name, out var poolSubnetParam)) + { + agentPool.VnetSubnetId = poolSubnetParam; + } + else if (defaultSubnetParam is not null) + { + agentPool.VnetSubnetId = defaultSubnetParam; + } + + aks.AgentPoolProfiles.Add(agentPool); + } + + // OIDC issuer + if (aksResource.OidcIssuerEnabled) + { + aks.OidcIssuerProfile = new ManagedClusterOidcIssuerProfile + { + IsEnabled = true + }; + } + + // Workload identity + if (aksResource.WorkloadIdentityEnabled) + { + aks.SecurityProfile = new ManagedClusterSecurityProfile + { + IsWorkloadIdentityEnabled = true + }; + } + + // Private cluster + if (aksResource.IsPrivateCluster) + { + aks.ApiServerAccessProfile = new ManagedClusterApiServerAccessProfile + { + EnablePrivateCluster = true + }; + } + + // Network profile + var hasSubnetConfig = hasDefaultSubnet || aksResource.NodePoolSubnets.Count > 0; + if (aksResource.NetworkProfile is not null) + { + aks.NetworkProfile = new ContainerServiceNetworkProfile + { + NetworkPlugin = aksResource.NetworkProfile.NetworkPlugin switch + { + "azure" => ContainerServiceNetworkPlugin.Azure, + "kubenet" => ContainerServiceNetworkPlugin.Kubenet, + _ => ContainerServiceNetworkPlugin.Azure + }, + ServiceCidr = aksResource.NetworkProfile.ServiceCidr, + DnsServiceIP = aksResource.NetworkProfile.DnsServiceIP + }; + if (aksResource.NetworkProfile.NetworkPolicy is not null) + { + aks.NetworkProfile.NetworkPolicy = aksResource.NetworkProfile.NetworkPolicy switch + { + "calico" => ContainerServiceNetworkPolicy.Calico, + "azure" => ContainerServiceNetworkPolicy.Azure, + _ => ContainerServiceNetworkPolicy.Calico + }; + } + } + else if (hasSubnetConfig) + { + aks.NetworkProfile = new ContainerServiceNetworkProfile + { + NetworkPlugin = ContainerServiceNetworkPlugin.Azure, + }; + } + + infrastructure.Add(aks); + + // ACR pull role assignment for kubelet identity + if (aksResource.DefaultContainerRegistry is not null || aksResource.TryGetLastAnnotation(out _)) + { + var acrNameParam = new ProvisioningParameter("acrName", typeof(string)); + infrastructure.Add(acrNameParam); + + var acr = ContainerRegistryService.FromExisting("acr"); + acr.Name = acrNameParam; + infrastructure.Add(acr); + + // AcrPull role: 7f951dda-4ed3-4680-a7ca-43fe172d538d + var acrPullRoleId = BicepFunction.GetSubscriptionResourceId( + "Microsoft.Authorization/roleDefinitions", + "7f951dda-4ed3-4680-a7ca-43fe172d538d"); + + // Access kubelet identity objectId via property path + var kubeletObjectId = new MemberExpression( + new MemberExpression( + new MemberExpression( + new MemberExpression( + new IdentifierExpression(aks.BicepIdentifier), + "properties"), + "identityProfile"), + "kubeletidentity"), + "objectId"); + + var roleAssignment = new RoleAssignment("acrPullRole") + { + Name = BicepFunction.CreateGuid(acr.Id, aks.Id, acrPullRoleId), + Scope = new IdentifierExpression(acr.BicepIdentifier), + RoleDefinitionId = acrPullRoleId, + PrincipalId = kubeletObjectId, + PrincipalType = RoleManagementPrincipalType.ServicePrincipal + }; + infrastructure.Add(roleAssignment); + } + + // Outputs + infrastructure.Add(new ProvisioningOutput("id", typeof(string)) { Value = aks.Id }); + infrastructure.Add(new ProvisioningOutput("name", typeof(string)) { Value = aks.Name }); + + // OIDC issuer URL and kubelet identity require property path expressions + var aksId = new IdentifierExpression(aks.BicepIdentifier); + infrastructure.Add(new ProvisioningOutput("clusterFqdn", typeof(string)) + { + Value = new MemberExpression(new MemberExpression(aksId, "properties"), "fqdn") + }); + infrastructure.Add(new ProvisioningOutput("oidcIssuerUrl", typeof(string)) + { + Value = new MemberExpression( + new MemberExpression(new MemberExpression(aksId, "properties"), "oidcIssuerProfile"), + "issuerURL") + }); + infrastructure.Add(new ProvisioningOutput("kubeletIdentityObjectId", typeof(string)) + { + Value = new MemberExpression( + new MemberExpression( + new MemberExpression(new MemberExpression(aksId, "properties"), "identityProfile"), + "kubeletidentity"), + "objectId") + }); + infrastructure.Add(new ProvisioningOutput("nodeResourceGroup", typeof(string)) + { + Value = new MemberExpression(new MemberExpression(aksId, "properties"), "nodeResourceGroup") + }); + + // Federated identity credentials for workload identity + // Resolve the K8s namespace for the service account subject. + // If not explicitly configured, defaults to "default". + var k8sNamespace = "default"; + if (aksResource.KubernetesEnvironment.TryGetLastAnnotation(out var nsAnnotation)) + { + // Use the namespace expression's format string as the literal value. + // Dynamic (parameter-based) namespaces are not supported for federated + // credentials since Azure AD needs a fixed subject at provision time. + var nsFormat = nsAnnotation.Namespace.Format; + if (!string.IsNullOrEmpty(nsFormat) && !nsFormat.Contains('{')) + { + k8sNamespace = nsFormat; + } + } + + foreach (var (resourceName, identityResource) in aksResource.WorkloadIdentities) + { + var saName = $"{resourceName}-sa"; + var sanitizedName = Infrastructure.NormalizeBicepIdentifier(resourceName); + var identityParamName = $"identityName_{sanitizedName}"; + + var identityNameParam = new ProvisioningParameter(identityParamName, typeof(string)); + infrastructure.Add(identityNameParam); + aksResource.Parameters[identityParamName] = identityResource.PrincipalName; + + var existingIdentity = UserAssignedIdentity.FromExisting($"identity_{sanitizedName}"); + existingIdentity.Name = identityNameParam; + infrastructure.Add(existingIdentity); + + var fedCred = new FederatedIdentityCredential($"fedcred_{sanitizedName}") + { + Parent = existingIdentity, + Name = $"{resourceName}-fedcred", + IssuerUri = new MemberExpression( + new MemberExpression( + new MemberExpression(new IdentifierExpression(aks.BicepIdentifier), "properties"), + "oidcIssuerProfile"), + "issuerURL"), + Subject = $"system:serviceaccount:{k8sNamespace}:{saName}", + Audiences = { "api://AzureADTokenExchange" } + }; + infrastructure.Add(fedCred); + } + } +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs new file mode 100644 index 00000000000..0cc80dd8fd5 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -0,0 +1,123 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable ASPIREAZURE003 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + +using Aspire.Hosting.Azure.Kubernetes; +using Aspire.Hosting.Kubernetes; + +namespace Aspire.Hosting.Azure; + +/// +/// Represents an Azure Kubernetes Service (AKS) environment resource that provisions +/// an AKS cluster and serves as a compute environment for Kubernetes workloads. +/// +/// The name of the resource. +/// Callback to configure the Azure infrastructure. +public class AzureKubernetesEnvironmentResource( + string name, + Action configureInfrastructure) + : AzureProvisioningResource(name, configureInfrastructure), + IAzureComputeEnvironmentResource, + IAzureNspAssociationTarget +{ + + /// + /// Gets the underlying Kubernetes environment resource used for Helm-based deployment. + /// + internal KubernetesEnvironmentResource KubernetesEnvironment { get; set; } = default!; + + /// + /// Gets the resource ID of the AKS cluster. + /// + public BicepOutputReference Id => new("id", this); + + /// + /// Gets the fully qualified domain name of the AKS cluster. + /// + public BicepOutputReference ClusterFqdn => new("clusterFqdn", this); + + /// + /// Gets the OIDC issuer URL for the AKS cluster, used for workload identity federation. + /// + public BicepOutputReference OidcIssuerUrl => new("oidcIssuerUrl", this); + + /// + /// Gets the object ID of the kubelet managed identity. + /// + public BicepOutputReference KubeletIdentityObjectId => new("kubeletIdentityObjectId", this); + + /// + /// Gets the name of the node resource group. + /// + public BicepOutputReference NodeResourceGroup => new("nodeResourceGroup", this); + + /// + /// Gets the name output reference for the AKS cluster. + /// + public BicepOutputReference NameOutputReference => new("name", this); + + /// + /// Gets or sets the Kubernetes version for the AKS cluster. + /// + internal string? KubernetesVersion { get; set; } + + /// + /// Gets or sets the SKU tier for the AKS cluster. + /// + internal AksSkuTier SkuTier { get; set; } = AksSkuTier.Free; + + /// + /// Gets or sets whether OIDC issuer is enabled on the cluster. + /// + internal bool OidcIssuerEnabled { get; set; } = true; + + /// + /// Gets or sets whether workload identity is enabled on the cluster. + /// + internal bool WorkloadIdentityEnabled { get; set; } = true; + + /// + /// Gets or sets the Log Analytics workspace resource for monitoring. + /// + internal AzureLogAnalyticsWorkspaceResource? LogAnalyticsWorkspace { get; set; } + + /// + /// Gets or sets whether Container Insights is enabled. + /// + internal bool ContainerInsightsEnabled { get; set; } + + /// + /// Gets the node pool configurations. + /// + internal List NodePools { get; } = + [ + new AksNodePoolConfig("system", "Standard_D2s_v5", 1, 3, AksNodePoolMode.System) + ]; + + /// + /// Gets the per-node-pool subnet overrides. Key is the pool name. + /// + internal Dictionary NodePoolSubnets { get; } = []; + + /// + /// Gets the workload identity mappings. Key is the resource name, value is the identity resource. + /// Used to generate federated identity credentials in Bicep. + /// + internal Dictionary WorkloadIdentities { get; } = []; + + /// + /// Gets or sets the network profile for the AKS cluster. + /// + internal AksNetworkProfile? NetworkProfile { get; set; } + + /// + /// Gets or sets whether the cluster should be private. + /// + internal bool IsPrivateCluster { get; set; } + + /// + /// Gets or sets the default container registry auto-created for this AKS environment. + /// + internal AzureContainerRegistryResource? DefaultContainerRegistry { get; set; } +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs new file mode 100644 index 00000000000..f732048f21d --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -0,0 +1,419 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable ASPIREPIPELINES001 // Pipeline step types used for push/deploy dependency wiring +#pragma warning disable ASPIREAZURE001 // AzureEnvironmentResource.ProvisionInfrastructureStepName for pipeline ordering +#pragma warning disable ASPIREFILESYSTEM001 // IFileSystemService/TempDirectory are experimental + +using System.Text; +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Dcp.Process; +using Aspire.Hosting.Eventing; +using Aspire.Hosting.Kubernetes; +using Aspire.Hosting.Kubernetes.Resources; +using Aspire.Hosting.Lifecycle; +using Aspire.Hosting.Pipelines; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Infrastructure eventing subscriber that processes compute resources +/// targeting an AKS environment. +/// +internal sealed class AzureKubernetesInfrastructure( + ILogger logger) + : IDistributedApplicationEventingSubscriber +{ + /// + public Task SubscribeAsync(IDistributedApplicationEventing eventing, DistributedApplicationExecutionContext executionContext, CancellationToken cancellationToken) + { + if (!executionContext.IsRunMode) + { + eventing.Subscribe(OnBeforeStartAsync); + } + + return Task.CompletedTask; + } + + private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cancellationToken = default) + { + var aksEnvironments = @event.Model.Resources + .OfType() + .ToArray(); + + if (aksEnvironments.Length == 0) + { + return Task.CompletedTask; + } + + foreach (var environment in aksEnvironments) + { + logger.LogInformation("Processing AKS environment '{Name}'", environment.Name); + + // Add a pipeline step to fetch AKS credentials into an isolated kubeconfig + // file. This runs after AKS is provisioned and before the Helm deploy. + AddGetCredentialsStep(environment); + + // Ensure a default user node pool exists for workload scheduling. + // The system pool should only run system pods; application workloads + // need a user pool. + var defaultUserPool = EnsureDefaultUserNodePool(environment, @event.Model); + + foreach (var r in @event.Model.GetComputeResources()) + { + var resourceComputeEnvironment = r.GetComputeEnvironment(); + + // Check if this resource targets THIS AKS environment + if (resourceComputeEnvironment is not null && resourceComputeEnvironment != environment) + { + continue; + } + + // If the resource has no explicit node pool affinity, assign it + // to the default user pool. + if (!r.TryGetLastAnnotation(out _) && defaultUserPool is not null) + { + r.Annotations.Add(new KubernetesNodePoolAnnotation(defaultUserPool)); + } + + // Wire workload identity: if the resource has an AppIdentityAnnotation + // (auto-created by AzureResourcePreparer or explicit via WithAzureUserAssignedIdentity), + // generate a ServiceAccount and wire the pod spec. + if (r.TryGetLastAnnotation(out var appIdentity)) + { + // Ensure OIDC + workload identity are enabled on the cluster + environment.OidcIssuerEnabled = true; + environment.WorkloadIdentityEnabled = true; + + var saName = $"{r.Name}-sa"; + var identityClientId = appIdentity.IdentityResource.ClientId; + + // Use KubernetesServiceCustomizationAnnotation to inject SA + pod spec changes + // during Helm chart generation. + r.Annotations.Add(new KubernetesServiceCustomizationAnnotation(kubeResource => + { + // Create ServiceAccount with workload identity annotations + var serviceAccount = new ServiceAccountV1(); + serviceAccount.Metadata.Name = saName; + serviceAccount.Metadata.Annotations["azure.workload.identity/client-id"] = + $"{{{{ .Values.parameters.{r.Name}.identityClientId }}}}"; + serviceAccount.Metadata.Labels["azure.workload.identity/use"] = "true"; + kubeResource.AdditionalResources.Add(serviceAccount); + + // Add a placeholder parameter for the identity clientId + // so it appears in values.yaml under parameters..identityClientId. + // The actual value is resolved at deploy time via CapturedHelmValueProviders. + kubeResource.Parameters["identityClientId"] = new KubernetesResource.HelmValue( + $"{{{{ .Values.parameters.{r.Name}.identityClientId }}}}", + string.Empty); + + // Set serviceAccountName on pod spec and add workload identity label + if (kubeResource.Workload?.PodTemplate is { } podTemplate) + { + if (podTemplate.Spec is { } podSpec) + { + podSpec.ServiceAccountName = saName; + } + + // The workload identity webhook requires this label on the POD + // to inject AZURE_CLIENT_ID, token volume mounts, etc. + podTemplate.Metadata.Labels["azure.workload.identity/use"] = "true"; + } + })); + + // Wire the identity clientId as a deferred Helm value so it gets + // resolved from the Bicep output at deploy time. The SA annotation + // references {{ .Values.parameters..identityClientId }}. + if (identityClientId is IValueProvider clientIdProvider) + { + environment.KubernetesEnvironment.CapturedHelmValueProviders.Add( + new KubernetesEnvironmentResource.CapturedHelmValueProvider( + "parameters", + r.Name, + "identityClientId", + clientIdProvider)); + } + + // Store the identity reference for federated credential Bicep generation + environment.WorkloadIdentities[r.Name] = appIdentity.IdentityResource; + } + } + } + + return Task.CompletedTask; + } + + /// + /// Ensures the AKS environment has at least one user node pool. If none exists, + /// creates a default "workload" user pool and adds it to the app model. + /// + private static AksNodePoolResource? EnsureDefaultUserNodePool( + AzureKubernetesEnvironmentResource environment, + DistributedApplicationModel appModel) + { + var hasUserPool = environment.NodePools.Any(p => p.Mode is AksNodePoolMode.User); + + if (hasUserPool) + { + // Return the first user pool. Search the app model for the existing + // AksNodePoolResource so we use the same object identity as AddNodePool created. + var firstUserConfig = environment.NodePools.First(p => p.Mode is AksNodePoolMode.User); + return FindNodePoolResource(appModel, environment, firstUserConfig.Name); + } + + // No user pool configured — create a default one and add it to the app model. + var defaultConfig = new AksNodePoolConfig("workload", "Standard_D2s_v5", 1, 3, AksNodePoolMode.User); + environment.NodePools.Add(defaultConfig); + + var defaultPool = new AksNodePoolResource("workload", defaultConfig, environment); + defaultPool.Annotations.Add(ManifestPublishingCallbackAnnotation.Ignore); + appModel.Resources.Add(defaultPool); + return defaultPool; + } + + /// + /// Finds an existing AksNodePoolResource in the app model by name, + /// or creates one if not found (for pools added via config but not via AddNodePool). + /// + private static AksNodePoolResource FindNodePoolResource( + DistributedApplicationModel appModel, + AzureKubernetesEnvironmentResource environment, + string poolName) + { + // Search the app model for an existing pool resource with matching name and parent + var existing = appModel.Resources + .OfType() + .FirstOrDefault(p => p.Name == poolName && p.AksParent == environment); + + if (existing is not null) + { + return existing; + } + + // Pool was added via NodePools config but not via AddNodePool — create the resource + var config = environment.NodePools.First(p => p.Name == poolName); + var pool = new AksNodePoolResource(poolName, config, environment); + appModel.Resources.Add(pool); + return pool; + } + + /// + /// Adds a pipeline step to the inner KubernetesEnvironmentResource that fetches + /// AKS cluster credentials into an isolated kubeconfig file after the AKS cluster + /// is provisioned via Bicep. + /// + private static void AddGetCredentialsStep(AzureKubernetesEnvironmentResource environment) + { + var k8sEnv = environment.KubernetesEnvironment; + + k8sEnv.Annotations.Add(new PipelineStepAnnotation((_) => + { + var step = new PipelineStep + { + Name = $"aks-get-credentials-{environment.Name}", + Description = $"Fetches AKS credentials for {environment.Name}", + Action = ctx => GetAksCredentialsAsync(ctx, environment) + }; + + // Run after ALL Azure infrastructure is provisioned (including the AKS cluster). + // This depends on the aggregation step that gates on all individual provision-* steps. + step.DependsOn(AzureEnvironmentResource.ProvisionInfrastructureStepName); + + // Must complete before Helm prepare step + step.RequiredBy($"prepare-{k8sEnv.Name}"); + + return new[] { step }; + })); + } + + /// + /// Fetches AKS credentials into an isolated kubeconfig file using az aks get-credentials, + /// then sets the KubeConfigPath on the inner KubernetesEnvironmentResource so that + /// subsequent Helm and kubectl commands target the AKS cluster. + /// + private static async Task GetAksCredentialsAsync( + PipelineStepContext context, + AzureKubernetesEnvironmentResource environment) + { + var getCredsTask = await context.ReportingStep.CreateTaskAsync( + $"Fetching AKS credentials for {environment.Name}", + context.CancellationToken).ConfigureAwait(false); + + await using (getCredsTask.ConfigureAwait(false)) + { + try + { + // Get the actual provisioned cluster name from the Bicep output. + // The Azure.Provisioning SDK may add a unique suffix to the name + // (e.g., take('aks-${uniqueString(resourceGroup().id)}', 63)). + var clusterName = await environment.NameOutputReference.GetValueAsync(context.CancellationToken).ConfigureAwait(false) + ?? environment.Name; + + var azPath = FindAzCli(); + var resourceGroup = await GetResourceGroupAsync(azPath, clusterName, context) + .ConfigureAwait(false); + + // Fetch kubeconfig content to stdout using --file - to avoid az CLI + // writing credentials with potentially permissive file permissions. + // We then write the content ourselves to a temp file with controlled access. + var fileSystemService = context.Services.GetRequiredService(); + var kubeConfigDir = fileSystemService.TempDirectory.CreateTempSubdirectory("aspire-aks"); + var kubeConfigPath = Path.Combine(kubeConfigDir.Path, "kubeconfig"); + + context.Logger.LogInformation( + "Fetching AKS credentials: cluster={ClusterName}, resourceGroup={ResourceGroup}", + clusterName, resourceGroup); + + var result = await RunAzCommandAsync( + azPath, + $"aks get-credentials --resource-group \"{resourceGroup}\" --name \"{clusterName}\" --file -", + context.Logger, + context.Services).ConfigureAwait(false); + + if (result.ExitCode != 0) + { + throw new InvalidOperationException( + $"az aks get-credentials failed (exit code {result.ExitCode}): {result.StandardError}"); + } + + // Write kubeconfig content to a temp file we control. + // The IFileSystemService temp directory is auto-cleaned on dispose. + await File.WriteAllTextAsync(kubeConfigPath, result.StandardOutput, context.CancellationToken).ConfigureAwait(false); + + // On Unix, restrict file permissions to owner-only (0600) + if (!OperatingSystem.IsWindows()) + { + File.SetUnixFileMode(kubeConfigPath, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + + // Set the kubeconfig path on the inner K8s environment so + // Helm and kubectl commands use --kubeconfig to target this cluster + environment.KubernetesEnvironment.KubeConfigPath = kubeConfigPath; + + context.Logger.LogInformation( + "AKS credentials written to {KubeConfigPath}", kubeConfigPath); + + // Add AKS connection info to the pipeline summary + context.Summary.Add( + "☸ AKS Cluster", + new MarkdownString($"**{clusterName}** in resource group **{resourceGroup}**")); + + context.Summary.Add( + "🔑 Connect to cluster", + new MarkdownString($"`az aks get-credentials --resource-group {resourceGroup} --name {clusterName}`")); + + await getCredsTask.SucceedAsync( + $"AKS credentials fetched for cluster {clusterName}", + context.CancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + await getCredsTask.FailAsync( + $"Failed to fetch AKS credentials: {ex.Message}", + context.CancellationToken).ConfigureAwait(false); + throw; + } + } + } + + private static string FindAzCli() + { + var azPath = PathLookupHelper.FindFullPathFromPath("az"); + if (azPath is null) + { + throw new InvalidOperationException( + "Azure CLI (az) not found. Install it from https://learn.microsoft.com/cli/azure/install-azure-cli"); + } + return azPath; + } + + /// + /// Gets the resource group, trying deployment state first, falling back to az CLI query. + /// On first deploy, the deployment state may not be loaded into IConfiguration yet + /// because it's written during the pipeline run (after create-provisioning-context). + /// + private static async Task GetResourceGroupAsync( + string azPath, + string clusterName, + PipelineStepContext context) + { + // Try deployment state first (works on re-deploys) + var configuration = context.Services.GetRequiredService(); + var resourceGroup = configuration["Azure:ResourceGroup"]; + + if (!string.IsNullOrEmpty(resourceGroup)) + { + return resourceGroup; + } + + // Fallback for first deploy: query Azure directly + context.Logger.LogDebug( + "Resource group not in deployment state, querying Azure for cluster '{ClusterName}'", + clusterName); + + var result = await RunAzCommandAsync( + azPath, + $"resource list --resource-type Microsoft.ContainerService/managedClusters --name \"{clusterName}\" --query [0].resourceGroup -o tsv", + context.Logger, + context.Services).ConfigureAwait(false); + + if (result.ExitCode != 0) + { + throw new InvalidOperationException( + $"az resource list failed (exit code {result.ExitCode}): {result.StandardError}"); + } + + resourceGroup = result.StandardOutput.Trim().ReplaceLineEndings("").Trim(); + + if (string.IsNullOrEmpty(resourceGroup)) + { + throw new InvalidOperationException( + $"Could not resolve resource group for AKS cluster '{clusterName}'. " + + "Ensure Azure provisioning has completed."); + } + + return resourceGroup; + } + + /// + /// Runs an az CLI command using IProcessRunner from DI. + /// Returns the captured stdout, stderr, and exit code. + /// + private static async Task RunAzCommandAsync( + string azPath, + string arguments, + ILogger logger, + IServiceProvider services) + { + var stdout = new StringBuilder(); + var stderr = new StringBuilder(); + + var spec = new ProcessSpec(azPath) + { + Arguments = arguments, + OnOutputData = data => stdout.AppendLine(data), + OnErrorData = data => stderr.AppendLine(data), + ThrowOnNonZeroReturnCode = false + }; + + logger.LogDebug("Running: {AzPath} {Arguments}", azPath, arguments); + + var processRunner = services.GetRequiredService(); + var (task, disposable) = processRunner.Run(spec); + + try + { + var result = await task.ConfigureAwait(false); + return new AzCommandResult(result.ExitCode, stdout.ToString(), stderr.ToString()); + } + finally + { + await disposable.DisposeAsync().ConfigureAwait(false); + } + } + + private sealed record AzCommandResult(int ExitCode, string StandardOutput, string StandardError); +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/README.md b/src/Aspire.Hosting.Azure.Kubernetes/README.md new file mode 100644 index 00000000000..c31a66af4d0 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/README.md @@ -0,0 +1,36 @@ +# Aspire.Hosting.Azure.Kubernetes library + +Provides extension methods and resource definitions for an Aspire AppHost to configure an Azure Kubernetes Service (AKS) environment. + +## Getting started + +### Prerequisites + +- An Azure subscription - [create one for free](https://azure.microsoft.com/free/) + +### Install the package + +In your AppHost project, install the Aspire Azure Kubernetes Hosting library with [NuGet](https://www.nuget.org): + +```dotnetcli +dotnet add package Aspire.Hosting.Azure.Kubernetes +``` + +## Usage example + +Then, in the _AppHost.cs_ file of `AppHost`, add an AKS environment and deploy services to it: + +```csharp +var aks = builder.AddAzureKubernetesEnvironment("aks"); + +var myService = builder.AddProject() + .WithComputeEnvironment(aks); +``` + +## Additional documentation + +* https://learn.microsoft.com/azure/aks/ + +## Feedback & contributing + +https://github.com/microsoft/aspire diff --git a/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs b/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs new file mode 100644 index 00000000000..fb45d20a0c3 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs @@ -0,0 +1,344 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +#:property PublishAot=false + +using System.Diagnostics; +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; + +// Fetch VM sizes from Azure REST API using the 'az' CLI +var subscriptionId = Environment.GetEnvironmentVariable("AZURE_SUBSCRIPTION_ID"); +if (string.IsNullOrWhiteSpace(subscriptionId)) +{ + // Try to get default subscription from az CLI + subscriptionId = await RunAzCommand("account show --query id -o tsv").ConfigureAwait(false); + subscriptionId = subscriptionId?.Trim(); +} + +if (string.IsNullOrWhiteSpace(subscriptionId)) +{ + Console.Error.WriteLine("Error: No Azure subscription found. Set AZURE_SUBSCRIPTION_ID or run 'az login'."); + return 1; +} + +Console.WriteLine($"Using subscription: {subscriptionId}"); + +// Query all US regions for VM SKUs to build a comprehensive unified list. +// Different regions offer different VM sizes, so querying multiple regions +// ensures we capture the full set available across the US. +string[] usRegions = [ + "eastus", "eastus2", "centralus", "northcentralus", "southcentralus", + "westus", "westus2", "westus3", "westcentralus" +]; + +var allSkus = new List(); +foreach (var region in usRegions) +{ + Console.WriteLine($"Querying VM SKUs for {region}..."); + var url = $"https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.Compute/skus?api-version=2021-07-01&$filter=location eq '{region}'"; + var json = await RunAzCommand($"rest --method get --url \"{url}\"").ConfigureAwait(false); + + if (string.IsNullOrWhiteSpace(json)) + { + Console.Error.WriteLine($"Warning: Failed to fetch VM SKUs for {region}, skipping."); + continue; + } + + var skuResponse = JsonSerializer.Deserialize(json, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + if (skuResponse?.Value is not null) + { + allSkus.AddRange(skuResponse.Value); + Console.WriteLine($" Found {skuResponse.Value.Count} SKUs in {region}"); + } +} + +if (allSkus.Count == 0) +{ + Console.Error.WriteLine("Error: Failed to fetch VM SKUs from any US region."); + return 1; +} + +// Filter to virtualMachines, deduplicate by name (keep first occurrence for capability data) +var vmSkus = allSkus + .Where(s => s.ResourceType == "virtualMachines" && !string.IsNullOrEmpty(s.Name)) + .GroupBy(s => s.Name) + .Select(g => g.First()) + .Select(s => new VmSizeInfo + { + Name = s.Name!, + Family = s.Family ?? "Other", + VCpus = s.GetCapabilityValue("vCPUs"), + MemoryGB = s.GetCapabilityValue("MemoryGB"), + MaxDataDiskCount = s.GetCapabilityValue("MaxDataDiskCount"), + PremiumIO = s.GetCapabilityBool("PremiumIO"), + AcceleratedNetworking = s.GetCapabilityBool("AcceleratedNetworkingEnabled"), + GpuCount = s.GetCapabilityValue("GPUs"), + }) + .DistinctBy(s => s.Name) + .OrderBy(s => s.Family, StringComparer.OrdinalIgnoreCase) + .ThenBy(s => s.Name, StringComparer.OrdinalIgnoreCase) + .ToList(); + +Console.WriteLine($"Found {vmSkus.Count} VM sizes"); + +var code = VmSizeClassGenerator.GenerateCode("Aspire.Hosting.Azure.Kubernetes", vmSkus); +File.WriteAllText(Path.Combine("..", "AksNodeVmSizes.Generated.cs"), code); +Console.WriteLine($"Generated AksNodeVmSizes.Generated.cs with {vmSkus.Count} VM sizes"); + +return 0; + +static async Task RunAzCommand(string arguments) +{ + var psi = new ProcessStartInfo + { + FileName = "az", + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + using var process = Process.Start(psi); + if (process is null) + { + return null; + } + + // Read stdout and stderr concurrently to avoid deadlock when + // the process fills the stderr pipe buffer. + var stdoutTask = process.StandardOutput.ReadToEndAsync(); + var stderrTask = process.StandardError.ReadToEndAsync(); + + await process.WaitForExitAsync().ConfigureAwait(false); + + var output = await stdoutTask.ConfigureAwait(false); + var stderr = await stderrTask.ConfigureAwait(false); + + if (process.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) + { + Console.Error.WriteLine($"az {arguments}: {stderr.Trim()}"); + } + + return process.ExitCode == 0 ? output : null; +} + +public sealed class SkuResponse +{ + [JsonPropertyName("value")] + public List? Value { get; set; } +} + +public sealed class ResourceSku +{ + [JsonPropertyName("resourceType")] + public string? ResourceType { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("tier")] + public string? Tier { get; set; } + + [JsonPropertyName("size")] + public string? Size { get; set; } + + [JsonPropertyName("family")] + public string? Family { get; set; } + + [JsonPropertyName("capabilities")] + public List? Capabilities { get; set; } + + public string? GetCapabilityValue(string name) + { + return Capabilities?.FirstOrDefault(c => c.Name == name)?.Value; + } + + public bool GetCapabilityBool(string name) + { + var value = GetCapabilityValue(name); + return string.Equals(value, "True", StringComparison.OrdinalIgnoreCase); + } +} + +public sealed class SkuCapability +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("value")] + public string? Value { get; set; } +} + +public sealed class VmSizeInfo +{ + public string Name { get; set; } = ""; + public string Family { get; set; } = ""; + public string? VCpus { get; set; } + public string? MemoryGB { get; set; } + public string? MaxDataDiskCount { get; set; } + public bool PremiumIO { get; set; } + public bool AcceleratedNetworking { get; set; } + public string? GpuCount { get; set; } +} + +internal static partial class VmSizeClassGenerator +{ + public static string GenerateCode(string ns, List sizes) + { + var sb = new StringBuilder(); + sb.AppendLine("// Licensed to the .NET Foundation under one or more agreements."); + sb.AppendLine("// The .NET Foundation licenses this file to you under the MIT license."); + sb.AppendLine(); + sb.AppendLine("// "); + sb.AppendLine("// This file is generated by the GenVmSizes tool. Do not edit manually."); + sb.AppendLine(); + sb.AppendLine(CultureInfo.InvariantCulture, $"namespace {ns};"); + sb.AppendLine(); + sb.AppendLine("/// "); + sb.AppendLine("/// Provides well-known Azure VM size constants for use with AKS node pools."); + sb.AppendLine("/// "); + sb.AppendLine("/// "); + sb.AppendLine("/// This class is auto-generated from Azure Resource SKUs across all US regions."); + sb.AppendLine("/// To update, run the GenVmSizes tool:"); + sb.AppendLine("/// dotnet run --project src/Aspire.Hosting.Azure.Kubernetes/tools GenVmSizes.cs"); + sb.AppendLine("/// VM size availability varies by region. This list is a union of sizes available"); + sb.AppendLine("/// across eastus, eastus2, centralus, northcentralus, southcentralus, westus, westus2,"); + sb.AppendLine("/// westus3, and westcentralus. Not all sizes may be available in every region."); + sb.AppendLine("/// "); + sb.AppendLine("public static partial class AksNodeVmSizes"); + sb.AppendLine("{"); + + var groups = sizes.GroupBy(s => s.Family) + .OrderBy(g => g.Key, StringComparer.OrdinalIgnoreCase); + + var firstClass = true; + foreach (var group in groups) + { + if (!firstClass) + { + sb.AppendLine(); + } + firstClass = false; + + var className = FamilyToClassName(group.Key); + + sb.AppendLine(" /// "); + sb.AppendLine(CultureInfo.InvariantCulture, $" /// VM sizes in the {EscapeXml(group.Key)} family."); + sb.AppendLine(" /// "); + sb.AppendLine(CultureInfo.InvariantCulture, $" public static class {className}"); + sb.AppendLine(" {"); + + var firstField = true; + foreach (var size in group.OrderBy(s => s.Name, StringComparer.OrdinalIgnoreCase)) + { + if (!firstField) + { + sb.AppendLine(); + } + firstField = false; + + var fieldName = VmSizeToFieldName(size.Name); + var description = BuildDescription(size); + + sb.AppendLine(" /// "); + sb.AppendLine(CultureInfo.InvariantCulture, $" /// {EscapeXml(description)}"); + sb.AppendLine(" /// "); + sb.AppendLine(CultureInfo.InvariantCulture, $" public const string {fieldName} = \"{size.Name}\";"); + } + + sb.AppendLine(" }"); + } + + sb.AppendLine("}"); + return sb.ToString(); + } + + private static string BuildDescription(VmSizeInfo size) + { + var parts = new List { size.Name }; + if (size.VCpus is not null) + { + parts.Add($"{size.VCpus} vCPUs"); + } + if (size.MemoryGB is not null) + { + parts.Add($"{size.MemoryGB} GB RAM"); + } + if (size.GpuCount is not null && size.GpuCount != "0") + { + parts.Add($"{size.GpuCount} GPU(s)"); + } + if (size.PremiumIO) + { + parts.Add("Premium SSD"); + } + return string.Join(" — ", parts); + } + + private static string FamilyToClassName(string family) + { + // Convert family names like "standardDSv2Family" to "StandardDSv2" + var name = family.Replace("Family", "", StringComparison.OrdinalIgnoreCase) + .Replace("_", ""); + + if (name.Length > 0) + { + name = char.ToUpperInvariant(name[0]) + name[1..]; + } + + // Clean non-identifier chars + return CleanIdentifier(name); + } + + private static string VmSizeToFieldName(string vmSize) + { + // "Standard_D4s_v5" → "StandardD4sV5" + var parts = vmSize.Split('_'); + var sb = new StringBuilder(); + foreach (var part in parts) + { + if (part.Length > 0) + { + sb.Append(char.ToUpperInvariant(part[0])); + if (part.Length > 1) + { + sb.Append(part[1..]); + } + } + } + return CleanIdentifier(sb.ToString()); + } + + private static string CleanIdentifier(string name) + { + var sb = new StringBuilder(); + foreach (var c in name) + { + if (char.IsLetterOrDigit(c)) + { + sb.Append(c); + } + } + + var result = sb.ToString(); + + // Ensure doesn't start with a digit + if (result.Length > 0 && char.IsDigit(result[0])) + { + result = "_" + result; + } + + return result; + } + + private static string EscapeXml(string s) => + s.Replace("&", "&") + .Replace("<", "<") + .Replace(">", ">") + .Replace("\"", """) + .Replace("'", "'"); +} diff --git a/src/Aspire.Hosting.Azure/Aspire.Hosting.Azure.csproj b/src/Aspire.Hosting.Azure/Aspire.Hosting.Azure.csproj index 87a93b17cf2..c0278dc6228 100644 --- a/src/Aspire.Hosting.Azure/Aspire.Hosting.Azure.csproj +++ b/src/Aspire.Hosting.Azure/Aspire.Hosting.Azure.csproj @@ -41,6 +41,7 @@ + diff --git a/src/Aspire.Hosting.Kubernetes/Aspire.Hosting.Kubernetes.csproj b/src/Aspire.Hosting.Kubernetes/Aspire.Hosting.Kubernetes.csproj index de2d534e056..04e87db475a 100644 --- a/src/Aspire.Hosting.Kubernetes/Aspire.Hosting.Kubernetes.csproj +++ b/src/Aspire.Hosting.Kubernetes/Aspire.Hosting.Kubernetes.csproj @@ -19,6 +19,8 @@ + + diff --git a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs index ed04194be43..08a35388b58 100644 --- a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs +++ b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs @@ -180,7 +180,7 @@ await ctx.ReportingStep.CompleteAsync( // Use saved state for the confirmation message (more accurate than recomputing) var @namespace = savedNamespace ?? "default"; await ConfirmDestroyAsync(ctx, $"Uninstall Helm release '{savedReleaseName}' from namespace '{@namespace}'? This action cannot be undone.").ConfigureAwait(false); - await HelmUninstallAsync(ctx, savedReleaseName, @namespace).ConfigureAwait(false); + await HelmUninstallAsync(ctx, environment, savedReleaseName, @namespace).ConfigureAwait(false); ctx.Summary.Add("🗑️ Helm Release", savedReleaseName); ctx.Summary.Add("☸️ Namespace", @namespace); @@ -283,7 +283,8 @@ internal static async Task ResolveAndWriteDeployValuesAsync( { if (environment.CapturedHelmValues.Count == 0 && environment.CapturedHelmCrossReferences.Count == 0 - && environment.CapturedHelmImageReferences.Count == 0) + && environment.CapturedHelmImageReferences.Count == 0 + && environment.CapturedHelmValueProviders.Count == 0) { return; } @@ -329,6 +330,20 @@ internal static async Task ResolveAndWriteDeployValuesAsync( } } + // Phase 4: Resolve generic IValueProvider references. + // During publish, values backed by IValueProvider (e.g., Bicep output references, + // connection strings) are written as empty placeholders. At deploy time, we call + // GetValueAsync() to resolve the actual values from external sources. + // This is cloud-provider agnostic — any IValueProvider implementation works. + foreach (var valueProviderRef in environment.CapturedHelmValueProviders) + { + var resolvedValue = await valueProviderRef.ValueProvider.GetValueAsync(cancellationToken).ConfigureAwait(false); + if (resolvedValue is not null) + { + SetOverrideValue(overrideValues, valueProviderRef.Section, valueProviderRef.ResourceKey, valueProviderRef.ValueKey, resolvedValue); + } + } + if (overrideValues.Count > 0) { var serializer = new YamlDotNet.Serialization.SerializerBuilder() @@ -424,6 +439,11 @@ private static async Task HelmDeployAsync(PipelineStepContext context, Kubernete arguments.Append(" --create-namespace"); arguments.Append(" --wait"); + if (environment.KubeConfigPath is not null) + { + arguments.Append(CultureInfo.InvariantCulture, $" --kubeconfig \"{environment.KubeConfigPath}\""); + } + if (File.Exists(valuesFilePath)) { arguments.Append(CultureInfo.InvariantCulture, $" -f \"{valuesFilePath}\""); @@ -501,7 +521,7 @@ internal static async Task PrintResourceSummaryAsync( try { - var endpoints = await GetServiceEndpointsAsync(computeResource.Name.ToServiceName(), @namespace, context.Logger, context.CancellationToken).ConfigureAwait(false); + var endpoints = await GetServiceEndpointsAsync(computeResource.Name.ToServiceName(), @namespace, environment.KubeConfigPath, context.Logger, context.CancellationToken).ConfigureAwait(false); if (endpoints.Count > 0) { @@ -555,10 +575,10 @@ private static async Task HelmUninstallAsync(PipelineStepContext context, Kubern { var @namespace = await ResolveNamespaceAsync(context, environment).ConfigureAwait(false); var releaseName = await ResolveReleaseNameAsync(context, environment).ConfigureAwait(false); - await HelmUninstallAsync(context, releaseName, @namespace).ConfigureAwait(false); + await HelmUninstallAsync(context, environment, releaseName, @namespace).ConfigureAwait(false); } - private static async Task HelmUninstallAsync(PipelineStepContext context, string releaseName, string @namespace) + private static async Task HelmUninstallAsync(PipelineStepContext context, KubernetesEnvironmentResource environment, string releaseName, string @namespace) { var uninstallTask = await context.ReportingStep.CreateTaskAsync( new MarkdownString($"Uninstalling Helm release **{releaseName}** from namespace **{@namespace}**"), @@ -571,6 +591,11 @@ private static async Task HelmUninstallAsync(PipelineStepContext context, string var helmRunner = context.Services.GetRequiredService(); var arguments = $"uninstall {releaseName} --namespace {@namespace}"; + if (environment.KubeConfigPath is not null) + { + arguments += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + context.Logger.LogDebug("Running helm {Arguments}", arguments); var exitCode = await helmRunner.RunAsync( @@ -640,12 +665,18 @@ private static async Task ConfirmDestroyAsync(PipelineStepContext context, strin private static async Task> GetServiceEndpointsAsync( string serviceName, string @namespace, + string? kubeConfigPath, ILogger logger, CancellationToken cancellationToken) { var endpoints = new List(); var arguments = $"get service {serviceName} --namespace {@namespace} -o json"; + + if (kubeConfigPath is not null) + { + arguments += $" --kubeconfig \"{kubeConfigPath}\""; + } var stdoutBuilder = new StringBuilder(); var spec = new ProcessSpec("kubectl") diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentExtensions.cs index f12456f6473..8122a274cbc 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentExtensions.cs @@ -161,7 +161,80 @@ public static IResourceBuilder WithDashboard(this return builder; } - private static void EnsureDefaultHelmEngine(IResourceBuilder builder) + /// + /// Adds a named node pool to the Kubernetes environment. + /// + /// The Kubernetes environment resource builder. + /// The name of the node pool. This value is used as the nodeSelector value. + /// A reference to the for the new node pool. + /// + /// For vanilla Kubernetes, this creates a named reference to an existing node pool. + /// For managed Kubernetes services (e.g., AKS), the cloud-specific AddNodePool overload + /// provisions the pool with additional configuration such as VM size and autoscaling. + /// Use to schedule workloads on the returned node pool. + /// + /// + /// + /// var k8s = builder.AddKubernetesEnvironment("k8s"); + /// var gpuPool = k8s.AddNodePool("gpu"); + /// + /// builder.AddProject<MyApi>() + /// .WithComputeEnvironment(k8s) + /// .WithNodePool(gpuPool); + /// + /// + [AspireExport(Description = "Adds a named node pool to a Kubernetes environment")] + public static IResourceBuilder AddNodePool( + this IResourceBuilder builder, + [ResourceName] string name) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrEmpty(name); + + var nodePool = new KubernetesNodePoolResource(name, builder.Resource); + + if (builder.ApplicationBuilder.ExecutionContext.IsRunMode) + { + return builder.ApplicationBuilder.CreateResourceBuilder(nodePool); + } + + return builder.ApplicationBuilder.AddResource(nodePool) + .ExcludeFromManifest(); + } + + /// + /// Schedules a compute resource's workload on the specified Kubernetes node pool. + /// This translates to a Kubernetes nodeSelector in the pod specification + /// targeting the named node pool. + /// + /// The type of the compute resource. + /// The resource builder. + /// The node pool to schedule the workload on. + /// A reference to the for chaining. + /// + /// + /// var k8s = builder.AddKubernetesEnvironment("k8s"); + /// var gpuPool = k8s.AddNodePool("gpu"); + /// + /// builder.AddProject<MyApi>() + /// .WithComputeEnvironment(k8s) + /// .WithNodePool(gpuPool); + /// + /// + [AspireExport("withKubernetesNodePool", Description = "Schedules a workload on a specific Kubernetes node pool")] + public static IResourceBuilder WithNodePool( + this IResourceBuilder builder, + IResourceBuilder nodePool) + where T : IResource + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(nodePool); + + builder.WithAnnotation(new KubernetesNodePoolAnnotation(nodePool.Resource)); + return builder; + } + + internal static void EnsureDefaultHelmEngine(IResourceBuilder builder) { builder.Resource.DeploymentEngineStepsFactory ??= HelmDeploymentEngine.CreateStepsAsync; } diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index 68c03a816e0..3522184e938 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -92,6 +92,29 @@ public sealed class KubernetesEnvironmentResource : Resource, IComputeEnvironmen /// public string DefaultServiceType { get; set; } = "ClusterIP"; + /// + /// Gets or sets the path to an explicit kubeconfig file for Helm and kubectl commands. + /// When set, all Helm and kubectl commands will use --kubeconfig to target + /// this file instead of the default ~/.kube/config. + /// + /// + /// This is used by Azure Kubernetes Service (AKS) integration to isolate credentials + /// fetched via az aks get-credentials from the user's default kubectl context. + /// + public string? KubeConfigPath { get; set; } + + /// + /// Gets or sets the parent compute environment resource that owns this Kubernetes environment. + /// When set, resources with WithComputeEnvironment targeting the parent will also + /// be processed by this Kubernetes environment. + /// + /// + /// This is used by Azure Kubernetes Service (AKS) integration where the user calls + /// WithComputeEnvironment(aksEnv) but the inner KubernetesEnvironmentResource + /// needs to process the resource. + /// + public IComputeEnvironmentResource? ParentComputeEnvironment { get; set; } + internal IPortAllocator PortAllocator { get; } = new PortAllocator(); /// @@ -131,6 +154,19 @@ internal sealed record CapturedHelmCrossReference(string Section, string Resourc /// internal sealed record CapturedHelmImageReference(string Section, string ResourceKey, string ValueKey, IResource Resource); + /// + /// Represents a captured value provider reference that needs deploy-time resolution. + /// This handles any implementation (e.g., Bicep output references, + /// connection strings) that can't be resolved at publish time. + /// + internal sealed record CapturedHelmValueProvider(string Section, string ResourceKey, string ValueKey, IValueProvider ValueProvider); + + /// + /// Captured value provider references populated during publish, consumed during deploy + /// to resolve values from external sources (e.g., Azure Bicep outputs). + /// + internal List CapturedHelmValueProviders { get; } = []; + /// /// Gets or sets the delegate that creates deployment pipeline steps for the configured engine. /// @@ -173,7 +209,8 @@ public KubernetesEnvironmentResource(string name) : base(name) foreach (var computeResource in resources) { - var deploymentTarget = computeResource.GetDeploymentTargetAnnotation(environment)?.DeploymentTarget; + var targetEnv = (IComputeEnvironmentResource?)environment.ParentComputeEnvironment ?? environment; + var deploymentTarget = computeResource.GetDeploymentTargetAnnotation(targetEnv)?.DeploymentTarget; if (deploymentTarget is not null && deploymentTarget.TryGetAnnotationsOfType(out var annotations)) { @@ -209,7 +246,8 @@ public KubernetesEnvironmentResource(string name) : base(name) foreach (var computeResource in resources) { - var deploymentTarget = computeResource.GetDeploymentTargetAnnotation(this)?.DeploymentTarget; + var targetEnv = (IComputeEnvironmentResource?)ParentComputeEnvironment ?? this; + var deploymentTarget = computeResource.GetDeploymentTargetAnnotation(targetEnv)?.DeploymentTarget; if (deploymentTarget is null) { continue; diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs b/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs index 46c1003398c..b23afc41219 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs @@ -52,9 +52,13 @@ private async Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken foreach (var r in @event.Model.GetComputeResources()) { - // Skip resources that are explicitly targeted to a different compute environment + // Skip resources that are explicitly targeted to a different compute environment. + // Also match if the resource targets a parent compute environment (e.g., AKS) + // that owns this Kubernetes environment. var resourceComputeEnvironment = r.GetComputeEnvironment(); - if (resourceComputeEnvironment is not null && resourceComputeEnvironment != environment) + if (resourceComputeEnvironment is not null && + resourceComputeEnvironment != environment && + resourceComputeEnvironment != environment.ParentComputeEnvironment) { continue; } @@ -69,10 +73,14 @@ private async Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken var serviceResource = await environmentContext.CreateKubernetesResourceAsync(r, executionContext, cancellationToken).ConfigureAwait(false); serviceResource.AddPrintSummaryStep(); - // Add deployment target annotation to the resource + // Add deployment target annotation to the resource. + // Use the resource's actual compute environment (which may be a parent + // like AzureKubernetesEnvironmentResource) so that GetDeploymentTargetAnnotation + // can match it correctly during publish. + var computeEnvForAnnotation = resourceComputeEnvironment ?? (IComputeEnvironmentResource)environment; r.Annotations.Add(new DeploymentTargetAnnotation(serviceResource) { - ComputeEnvironment = environment, + ComputeEnvironment = computeEnvForAnnotation, ContainerRegistry = containerRegistry }); } diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesNodePoolAnnotation.cs b/src/Aspire.Hosting.Kubernetes/KubernetesNodePoolAnnotation.cs new file mode 100644 index 00000000000..df2f82b78b9 --- /dev/null +++ b/src/Aspire.Hosting.Kubernetes/KubernetesNodePoolAnnotation.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.ApplicationModel; + +namespace Aspire.Hosting.Kubernetes; + +/// +/// Annotation that associates a compute resource with a specific Kubernetes node pool. +/// When present, the Kubernetes deployment will include a nodeSelector targeting +/// the specified node pool. +/// +internal sealed class KubernetesNodePoolAnnotation(KubernetesNodePoolResource nodePool) : IResourceAnnotation +{ + /// + /// Gets the node pool to schedule the workload on. + /// + public KubernetesNodePoolResource NodePool { get; } = nodePool; +} diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesNodePoolResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesNodePoolResource.cs new file mode 100644 index 00000000000..94c1b41fad8 --- /dev/null +++ b/src/Aspire.Hosting.Kubernetes/KubernetesNodePoolResource.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.ApplicationModel; + +namespace Aspire.Hosting.Kubernetes; + +/// +/// Represents a Kubernetes node pool as a child resource of a . +/// Node pools can be referenced by compute resources to schedule workloads on specific node pools +/// using . +/// +/// The name of the node pool resource. +/// The parent Kubernetes environment resource. +[AspireExport] +public class KubernetesNodePoolResource( + string name, + KubernetesEnvironmentResource environment) : Resource(name), IResourceWithParent +{ + /// + /// Gets the parent Kubernetes environment resource. + /// + public KubernetesEnvironmentResource Parent { get; } = environment ?? throw new ArgumentNullException(nameof(environment)); + + /// + /// Gets the label key used to identify the node pool in the Kubernetes cluster. + /// Defaults to agentpool which is the standard label used by AKS and many managed Kubernetes services. + /// + public string NodeSelectorLabelKey { get; init; } = "agentpool"; +} diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs b/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs index f29e9f65f0e..7bc9bc5fc48 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs @@ -75,7 +75,9 @@ private async Task WriteKubernetesOutputAsync(DistributedApplicationModel model, foreach (var resource in resources) { - if (resource.GetDeploymentTargetAnnotation(environment)?.DeploymentTarget is KubernetesResource serviceResource) + // Check for deployment target matching this environment or its parent (e.g., AKS) + var targetEnv = (IComputeEnvironmentResource?)environment.ParentComputeEnvironment ?? environment; + if (resource.GetDeploymentTargetAnnotation(targetEnv)?.DeploymentTarget is KubernetesResource serviceResource) { // Materialize Dockerfile factory if present if (serviceResource.TargetResource.TryGetLastAnnotation(out var dockerfileBuildAnnotation) && @@ -103,6 +105,12 @@ private async Task WriteKubernetesOutputAsync(DistributedApplicationModel model, } } + // Apply node pool nodeSelector if the resource has a node pool annotation + if (serviceResource.TargetResource.TryGetLastAnnotation(out var nodePoolAnnotation)) + { + ApplyNodePoolSelector(serviceResource, nodePoolAnnotation.NodePool); + } + await WriteKubernetesTemplatesForResource(resource, serviceResource.GetTemplatedResources()).ConfigureAwait(false); await AppendResourceContextToHelmValuesAsync(resource, serviceResource).ConfigureAwait(false); } @@ -188,6 +196,21 @@ private async Task AddValuesToHelmSectionAsync( else { value = helmExpressionWithValue.Value; + + // If the value has an IValueProvider source, capture it for deploy-time + // resolution. Write an empty placeholder now and resolve at deploy time. + // This handles Bicep output references, connection strings, and any other + // deferred value source without requiring Azure-specific knowledge. + if (helmExpressionWithValue.ValueProviderSource is { } valueProvider) + { + value = string.Empty; + environment?.CapturedHelmValueProviders.Add( + new KubernetesEnvironmentResource.CapturedHelmValueProvider( + helmKey, + resource.Name.ToHelmValuesSectionName(), + valuesKey, + valueProvider)); + } } paramValues[valuesKey] = value ?? string.Empty; @@ -273,4 +296,15 @@ private async Task WriteKubernetesHelmChartAsync(KubernetesEnvironmentResource e Directory.CreateDirectory(OutputPath); await File.WriteAllTextAsync(outputFile, chartYaml, cancellationToken).ConfigureAwait(false); } + + private static void ApplyNodePoolSelector(KubernetesResource serviceResource, KubernetesNodePoolResource nodePool) + { + var podSpec = serviceResource.Workload?.PodTemplate?.Spec; + if (podSpec is null) + { + return; + } + + podSpec.NodeSelector[nodePool.NodeSelectorLabelKey] = nodePool.Name; + } } diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs index 3015e3438e3..04e9d21348d 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs @@ -365,6 +365,18 @@ private async Task ProcessEnvironmentAsync(KubernetesEnvironmentContext environm foreach (var environmentVariable in context.EnvironmentVariables) { var key = environmentVariable.Key; + + // Check if the value contains deferred providers (e.g., Bicep outputs) + // that can only be resolved after infrastructure provisioning. + // If so, create a deferred HelmValue with the env var key name. + if (IsUnresolvedAtPublishTime(environmentVariable.Value) && + environmentVariable.Value is IValueProvider deferredVp) + { + var deferredHelmValue = CreateDeferredHelmValue(key, deferredVp); + ProcessEnvironmentHelmExpression(deferredHelmValue, key); + continue; + } + var value = await ProcessValueAsync(environmentContext, executionContext, environmentVariable.Value).ConfigureAwait(false); switch (value) @@ -660,6 +672,8 @@ private static HelmValue ResolveUnknownValue(IManifestExpressionProvider paramet { var formattedName = parameter.ValueExpression.Replace(HelmExtensions.StartDelimiter, string.Empty) .Replace(HelmExtensions.EndDelimiter, string.Empty) + .Replace("{", string.Empty) + .Replace("}", string.Empty) .Replace(".", "_") .ToHelmValuesSectionName(); @@ -667,7 +681,52 @@ private static HelmValue ResolveUnknownValue(IManifestExpressionProvider paramet formattedName.ToHelmSecretExpression(resource.Name) : formattedName.ToHelmConfigExpression(resource.Name); - return new(helmExpression, parameter.ValueExpression); + var helmValue = new HelmValue(helmExpression, parameter.ValueExpression) + { + // If the expression provider also implements IValueProvider, attach it + // for deploy-time resolution. This handles Bicep output references, + // connection strings, and any other deferred value source. + ValueProviderSource = parameter as IValueProvider + }; + + return helmValue; + } + + /// + /// Checks if a value contains sub-expressions that cannot be resolved + /// at publish time and need deploy-time resolution via IValueProvider. + /// Recursively checks ReferenceExpression value providers. + /// + private static bool IsUnresolvedAtPublishTime(object value) + { + return value switch + { + string => false, + EndpointReference => false, + EndpointReferenceExpression => false, + ParameterResource => false, + ConnectionStringReference cs => IsUnresolvedAtPublishTime(cs.Resource.ConnectionStringExpression), + IResourceWithConnectionString csrs => IsUnresolvedAtPublishTime(csrs.ConnectionStringExpression), + ReferenceExpression expr => expr.ValueProviders.Any(IsUnresolvedAtPublishTime), + // Any other IManifestExpressionProvider that also implements IValueProvider + // is a deferred source (e.g., BicepOutputReference) + IManifestExpressionProvider when value is IValueProvider => true, + _ => false + }; + } + + /// + /// Creates a HelmValue that defers resolution to deploy time via IValueProvider. + /// + /// The environment variable or config key name to use as the Helm values path. + /// The value provider for deploy-time resolution. + private HelmValue CreateDeferredHelmValue(string key, IValueProvider valueProvider) + { + var helmExpression = key.ToHelmConfigExpression(TargetResource.Name); + return new HelmValue(helmExpression, string.Empty) + { + ValueProviderSource = valueProvider + }; } private static string GetKubernetesProtocolName(ProtocolType type) @@ -743,6 +802,14 @@ public HelmValue(string expression, ParameterResource parameterSource) /// public IResource? ImageResource { get; init; } + /// + /// Gets the value provider for deferred resolution at deploy time. + /// When set, the value is resolved via + /// during the prepare step, replacing the placeholder in values.yaml. + /// This handles any value provider (e.g., Bicep output references, connection strings). + /// + public IValueProvider? ValueProviderSource { get; init; } + /// /// Gets the key to use when writing this value to the Helm values.yaml file. /// When set, this overrides the dictionary key to ensure the values.yaml key matches diff --git a/src/Aspire.Hosting.Kubernetes/Resources/ServiceAccountV1.cs b/src/Aspire.Hosting.Kubernetes/Resources/ServiceAccountV1.cs new file mode 100644 index 00000000000..891262ab49a --- /dev/null +++ b/src/Aspire.Hosting.Kubernetes/Resources/ServiceAccountV1.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using YamlDotNet.Serialization; + +namespace Aspire.Hosting.Kubernetes.Resources; + +/// +/// Represents a Kubernetes ServiceAccount resource. +/// +[YamlSerializable] +public sealed class ServiceAccountV1() : BaseKubernetesResource("v1", "ServiceAccount") +{ +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs new file mode 100644 index 00000000000..410f4e831b3 --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs @@ -0,0 +1,279 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to Azure Kubernetes Service (AKS) +/// using Azure.Provisioning (Bicep-based) via aspire deploy --clear-cache. +/// This test uses the ACA-style deploy pipeline rather than manual AKS/ACR/Helm setup. +/// +public sealed class AksAzureProvisioningDeploymentTests(ITestOutputHelper output) +{ + // Timeout set to 45 minutes to allow for AKS provisioning (~10-15 min) plus container build and Helm deploy. + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithAzureProvisioning() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithAzureProvisioningCore(cancellationToken); + } + + private async Task DeployStarterToAksWithAzureProvisioningCore(CancellationToken cancellationToken) + { + // Validate prerequisites + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-provisioning"); + var projectName = "AksProvisioned"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithAzureProvisioning)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + // In CI, aspire add shows a version selection prompt + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); // select first version (PR build) + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Write complete AppHost.cs with AKS environment (full rewrite to avoid + // string-replacement failures caused by line-ending or whitespace differences) + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); + + var appHostContent = $""" + #pragma warning disable ASPIREPIPELINES001 + + var builder = DistributedApplication.CreateBuilder(args); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health"); + + builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WaitFor(apiService); + + builder.Build().Run(); + """; + + File.WriteAllText(appHostFilePath, appHostContent); + + output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy (Bicep provisioning + container build + ACR push + Helm deploy) + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + // Wait for pipeline to complete - AKS provisioning can take up to 30 minutes + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + // The cluster name is dynamic (Azure.Provisioning adds a suffix), so discover it + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running + output.WriteLine("Step 12: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 13: Verify apiservice endpoint via port-forward + output.WriteLine("Step 13: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 14: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 14: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 15: Clean up port-forwards + output.WriteLine("Step 15: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 16: Destroy Azure deployment + output.WriteLine("Step 16: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 17: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS provisioned deployment completed in {duration}"); + + // Report success + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithAzureProvisioning), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS via Azure Provisioning!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithAzureProvisioning), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + // Clean up the resource group we created + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + /// + /// Triggers cleanup of a specific resource group. + /// This is fire-and-forget - the hourly cleanup workflow handles any missed resources. + /// + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs new file mode 100644 index 00000000000..2e25d2e190d --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs @@ -0,0 +1,271 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to AKS with an explicit Azure Container Registry. +/// Verifies that AddAzureContainerRegistry and WithContainerRegistry correctly +/// provision a dedicated ACR and attach it to the AKS cluster. +/// +public sealed class AksExplicitRegistryDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithExplicitRegistry() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithExplicitRegistryCore(cancellationToken); + } + + private async Task DeployStarterToAksWithExplicitRegistryCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-acr"); + var projectName = "AksExplicitAcr"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithExplicitRegistry)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + // (Aspire.Hosting.Azure.Kubernetes already depends on Aspire.Hosting.Azure.ContainerRegistry) + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Write complete AppHost.cs with AKS environment and explicit ACR + // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); + + var appHostContent = $""" + #pragma warning disable ASPIREPIPELINES001 + + var builder = DistributedApplication.CreateBuilder(args); + + var acr = builder.AddAzureContainerRegistry("myacr"); + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithContainerRegistry(acr); + + var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health"); + + builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WaitFor(apiService); + + builder.Build().Run(); + """; + + File.WriteAllText(appHostFilePath, appHostContent); + + output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and explicit ACR"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running + output.WriteLine("Step 12: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 13: Verify apiservice endpoint via port-forward + output.WriteLine("Step 13: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 14: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 14: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 15: Clean up port-forwards + output.WriteLine("Step 15: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 16: Destroy Azure deployment + output.WriteLine("Step 16: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 17: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS explicit ACR deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithExplicitRegistry), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS with explicit ACR!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithExplicitRegistry), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs new file mode 100644 index 00000000000..607e8e266e8 --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs @@ -0,0 +1,283 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to AKS with a custom node pool. +/// Verifies that AddNodePool creates additional node pools and that pods are +/// scheduled to the correct pool via WithNodePool. +/// +public sealed class AksNodePoolDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithCustomNodePool() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithCustomNodePoolCore(cancellationToken); + } + + private async Task DeployStarterToAksWithCustomNodePoolCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-nodepool"); + var projectName = "AksNodePool"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithCustomNodePool)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Write complete AppHost.cs with AKS environment and custom node pool + // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); + + var appHostContent = $""" + #pragma warning disable ASPIREPIPELINES001 + + var builder = DistributedApplication.CreateBuilder(args); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var computePool = aks.AddNodePool("compute", "Standard_D2s_v5", 1, 3); + + var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithNodePool(computePool); + + builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WaitFor(apiService); + + builder.Build().Run(); + """; + + File.WriteAllText(appHostFilePath, appHostContent); + + output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and custom node pool"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running + output.WriteLine("Step 12: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 13: Verify two node pools exist (system + compute) + output.WriteLine("Step 13: Verifying node pools..."); + await auto.TypeAsync($"az aks nodepool list --resource-group {resourceGroupName} --cluster-name $AKS_NAME --query '[].{{name:name, mode:mode}}' -o table"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("compute", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 14: Verify apiservice pod has nodeSelector for the compute pool + output.WriteLine("Step 14: Verifying apiservice pod nodeSelector..."); + await auto.TypeAsync("kubectl get pod -l app.kubernetes.io/component=apiservice -o jsonpath='{.items[0].spec.nodeSelector}'"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 15: Verify apiservice endpoint via port-forward + output.WriteLine("Step 15: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 16: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 16: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 17: Clean up port-forwards + output.WriteLine("Step 17: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 18: Destroy Azure deployment + output.WriteLine("Step 18: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 19: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS node pool deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithCustomNodePool), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS with custom node pool!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithCustomNodePool), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs new file mode 100644 index 00000000000..4ffc173f91a --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs @@ -0,0 +1,284 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to AKS with per-pool subnet assignments. +/// Verifies that the default system pool and a user node pool can each be assigned to +/// different subnets within the same VNet. +/// +public sealed class AksPerPoolSubnetDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithPerPoolSubnets() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithPerPoolSubnetsCore(cancellationToken); + } + + private async Task DeployStarterToAksWithPerPoolSubnetsCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-poolsubnet"); + var projectName = "AksPoolSubnet"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithPerPoolSubnets)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Write complete AppHost.cs with AKS environment and per-pool subnets + // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); + + var appHostContent = $""" + #pragma warning disable ASPIREPIPELINES001 + + var builder = DistributedApplication.CreateBuilder(args); + + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.1.0.0/16"); + var defaultSubnet = vnet.AddSubnet("defaultsubnet", "10.1.0.0/22"); + var auxSubnet = vnet.AddSubnet("auxsubnet", "10.1.4.0/24"); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithSubnet(defaultSubnet); + + var auxPool = aks.AddNodePool("aux", "Standard_D2s_v5", 1, 3) + .WithSubnet(auxSubnet); + + var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithNodePool(auxPool); + + builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WaitFor(apiService); + + builder.Build().Run(); + """; + + File.WriteAllText(appHostFilePath, appHostContent); + + output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and per-pool subnets"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running + output.WriteLine("Step 12: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 13: Verify two node pools with different subnets exist + output.WriteLine("Step 13: Verifying node pools with different subnets..."); + await auto.TypeAsync($"az aks nodepool list --resource-group {resourceGroupName} --cluster-name $AKS_NAME --query '[].{{name:name, mode:mode, vnetSubnetId:vnetSubnetId}}' -o table"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("aux", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 14: Verify apiservice endpoint via port-forward + output.WriteLine("Step 14: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 15: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 15: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 16: Clean up port-forwards + output.WriteLine("Step 16: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 17: Destroy Azure deployment + output.WriteLine("Step 17: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 18: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS per-pool subnet deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithPerPoolSubnets), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS with per-pool subnet assignments!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithPerPoolSubnets), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs new file mode 100644 index 00000000000..a4507647092 --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs @@ -0,0 +1,273 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to AKS with a custom VNet/subnet. +/// Verifies that AddAzureVirtualNetwork and WithSubnet correctly integrate +/// AKS networking so that pods receive VNet IPs rather than default overlay IPs. +/// +public sealed class AksVnetDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithVnet() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithVnetCore(cancellationToken); + } + + private async Task DeployStarterToAksWithVnetCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-vnet"); + var projectName = "AksVnet"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithVnet)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Write complete AppHost.cs with AKS environment and VNet/subnet + // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); + + var appHostContent = $""" + #pragma warning disable ASPIREPIPELINES001 + + var builder = DistributedApplication.CreateBuilder(args); + + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.1.0.0/16"); + var subnet = vnet.AddSubnet("akssubnet", "10.1.0.0/22"); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithSubnet(subnet); + + var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health"); + + builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WaitFor(apiService); + + builder.Build().Run(); + """; + + File.WriteAllText(appHostFilePath, appHostContent); + + output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and VNet/subnet"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running and have VNet IPs (10.1.x.x, not 10.244.x.x overlay) + output.WriteLine("Step 12: Verifying pods have VNet IPs..."); + await auto.TypeAsync("kubectl get pods -o wide -n default"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("10.1.", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 13: Verify apiservice endpoint via port-forward + output.WriteLine("Step 13: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 14: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 14: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 15: Clean up port-forwards + output.WriteLine("Step 15: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 16: Destroy Azure deployment + output.WriteLine("Step 16: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 17: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS VNet deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithVnet), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS with VNet integration!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithVnet), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs new file mode 100644 index 00000000000..ddfa85d43f8 --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs @@ -0,0 +1,298 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to AKS with workload identity. +/// Verifies that referencing an Azure resource (blob storage) from an AKS-hosted service +/// correctly provisions workload identity (service account annotation and pod label). +/// +public sealed class AksWorkloadIdentityDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithWorkloadIdentity() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithWorkloadIdentityCore(cancellationToken); + } + + private async Task DeployStarterToAksWithWorkloadIdentityCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-wi"); + var projectName = "AksWorkloadId"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithWorkloadIdentity)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add required packages + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + output.WriteLine("Step 5b: Adding Azure Storage hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Storage"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Write complete AppHost.cs with AKS environment and workload identity via Azure Storage + // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); + + var appHostContent = $""" + #pragma warning disable ASPIREPIPELINES001 + + var builder = DistributedApplication.CreateBuilder(args); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + var storage = builder.AddAzureStorage("storage"); + var blobs = storage.AddBlobs("blobs"); + var photos = blobs.AddBlobContainer("photos"); + + var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithReference(photos); + + builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WaitFor(apiService); + + builder.Build().Run(); + """; + + File.WriteAllText(appHostFilePath, appHostContent); + + output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and workload identity via Azure Storage"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running + output.WriteLine("Step 12: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 13: Verify service account has workload identity annotation + output.WriteLine("Step 13: Verifying workload identity service account..."); + await auto.TypeAsync("kubectl get sa apiservice-sa -o yaml"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("azure.workload.identity/client-id", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 14: Verify pod has workload identity label + output.WriteLine("Step 14: Verifying pod workload identity label..."); + await auto.TypeAsync("kubectl get pod -l app.kubernetes.io/component=apiservice -o jsonpath='{.items[0].metadata.labels}' | grep azure.workload.identity"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 15: Verify apiservice endpoint via port-forward + output.WriteLine("Step 15: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 16: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 16: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 17: Clean up port-forwards + output.WriteLine("Step 17: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 18: Destroy Azure deployment + output.WriteLine("Step 18: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 19: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS workload identity deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithWorkloadIdentity), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS with workload identity!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithWorkloadIdentity), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs new file mode 100644 index 00000000000..e119f535b0c --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs @@ -0,0 +1,264 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying a TypeScript Express/React Aspire application to Azure Kubernetes Service (AKS). +/// Uses the same Azure.Provisioning pipeline as the C# variant but from a TypeScript AppHost created via +/// aspire new --template express-react. +/// +public sealed class TypeScriptAksDeploymentTests(ITestOutputHelper output) +{ + // Timeout set to 45 minutes to allow for AKS provisioning (~10-15 min) plus npm install and container build. + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployTypeScriptExpressToAksWithAzureProvisioning() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployTypeScriptExpressToAksWithAzureProvisioningCore(cancellationToken); + } + + private async Task DeployTypeScriptExpressToAksWithAzureProvisioningCore(CancellationToken cancellationToken) + { + // Validate prerequisites + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("ts-aks"); + var projectName = "TsAksApp"; + + output.WriteLine($"Test: {nameof(DeployTypeScriptExpressToAksWithAzureProvisioning)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + // TypeScript apphosts need the full bundle (not just the CLI binary) because + // the prebuilt AppHost server is required for aspire add to regenerate SDK code. + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + var prNumber = DeploymentE2ETestHelpers.GetPrNumber(); + if (prNumber > 0) + { + output.WriteLine($"Step 2: Installing Aspire bundle from PR #{prNumber}..."); + await auto.InstallAspireBundleFromPullRequestAsync(prNumber, counter); + } + await auto.SourceAspireBundleEnvironmentAsync(counter); + } + + // Step 3: Create TypeScript Express/React project using aspire new + output.WriteLine("Step 3: Creating TypeScript Express/React project..."); + await auto.AspireNewAsync(projectName, counter, template: AspireTemplate.ExpressReact); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitForAspireAddCompletionAsync(counter); + } + else + { + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + } + + // Step 6: Modify apphost.ts to add AKS environment for deployment + { + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostFilePath = Path.Combine(projectDir, "apphost.ts"); + + output.WriteLine($"Looking for apphost.ts at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + var originalContent = content; + + // Add Azure Kubernetes Environment before build().run() + // When there's exactly one compute environment, all resources auto-target it. + content = content.Replace( + "await builder.build().run();", + """ +// Add Azure Kubernetes Environment for deployment +await builder.addAzureKubernetesEnvironment("aks"); + +await builder.build().run(); +"""); + + if (content == originalContent) + { + throw new InvalidOperationException("apphost.ts was not modified. Template may have changed."); + } + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine($"Modified apphost.ts at: {appHostFilePath}"); + } + + // Step 7: Set environment for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Deploy to AKS using aspire deploy + output.WriteLine("Step 8: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + // Wait for pipeline to complete - AKS provisioning can take up to 30 minutes + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 9: Get AKS credentials for kubectl verification + output.WriteLine("Step 9: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 10: Wait for pods to be ready + output.WriteLine("Step 10: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 11: Verify pods are running + output.WriteLine("Step 11: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 12: Verify service endpoints via port-forward + output.WriteLine("Step 12: Verifying service endpoints..."); + await auto.TypeAsync("kubectl port-forward svc/api-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 13: Clean up port-forwards + output.WriteLine("Step 13: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 14: Destroy Azure deployment + output.WriteLine("Step 14: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 15: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"TypeScript AKS deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployTypeScriptExpressToAksWithAzureProvisioning), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - TypeScript Express app deployed to AKS via Azure Provisioning!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployTypeScriptExpressToAksWithAzureProvisioning), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs new file mode 100644 index 00000000000..e8219126224 --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs @@ -0,0 +1,268 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying a TypeScript Express/React Aspire application to AKS with a custom node pool. +/// Verifies that addNodePool from a TypeScript AppHost creates additional node pools in the AKS cluster. +/// +public sealed class TypeScriptAksNodePoolDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployTypeScriptExpressToAksWithNodePool() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployTypeScriptExpressToAksWithNodePoolCore(cancellationToken); + } + + private async Task DeployTypeScriptExpressToAksWithNodePoolCore(CancellationToken cancellationToken) + { + // Validate prerequisites + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("ts-aks-np"); + var projectName = "TsAksNodePool"; + + output.WriteLine($"Test: {nameof(DeployTypeScriptExpressToAksWithNodePool)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + // TypeScript apphosts need the full bundle (not just the CLI binary) because + // the prebuilt AppHost server is required for aspire add to regenerate SDK code. + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + var prNumber = DeploymentE2ETestHelpers.GetPrNumber(); + if (prNumber > 0) + { + output.WriteLine($"Step 2: Installing Aspire bundle from PR #{prNumber}..."); + await auto.InstallAspireBundleFromPullRequestAsync(prNumber, counter); + } + await auto.SourceAspireBundleEnvironmentAsync(counter); + } + + // Step 3: Create TypeScript Express/React project using aspire new + output.WriteLine("Step 3: Creating TypeScript Express/React project..."); + await auto.AspireNewAsync(projectName, counter, template: AspireTemplate.ExpressReact); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitForAspireAddCompletionAsync(counter); + } + else + { + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + } + + // Step 6: Modify apphost.ts to add AKS environment with custom node pool + { + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostFilePath = Path.Combine(projectDir, "apphost.ts"); + + output.WriteLine($"Looking for apphost.ts at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + var originalContent = content; + + // Add Azure Kubernetes Environment with a custom node pool before build().run() + content = content.Replace( + "await builder.build().run();", + """ +// Add Azure Kubernetes Environment with a custom node pool +const aks = await builder.addAzureKubernetesEnvironment("aks"); +await aks.addNodePool("compute", "Standard_D2s_v5", 1, 3); + +await builder.build().run(); +"""); + + if (content == originalContent) + { + throw new InvalidOperationException("apphost.ts was not modified. Template may have changed."); + } + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine($"Modified apphost.ts at: {appHostFilePath}"); + } + + // Step 7: Set environment for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Deploy to AKS using aspire deploy + output.WriteLine("Step 8: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 9: Get AKS credentials for kubectl verification + output.WriteLine("Step 9: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 10: Wait for pods to be ready + output.WriteLine("Step 10: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 11: Verify pods are running + output.WriteLine("Step 11: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 12: Verify two node pools exist (system + compute) + output.WriteLine("Step 12: Verifying node pools..."); + await auto.TypeAsync($"az aks nodepool list --resource-group {resourceGroupName} --cluster-name $AKS_NAME --query '[].{{name:name, mode:mode}}' -o table"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("compute", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 13: Verify service endpoints via port-forward + output.WriteLine("Step 13: Verifying service endpoints..."); + await auto.TypeAsync("kubectl port-forward svc/api-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 14: Clean up port-forwards + output.WriteLine("Step 14: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 15: Destroy Azure deployment + output.WriteLine("Step 15: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 16: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"TypeScript AKS node pool deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployTypeScriptExpressToAksWithNodePool), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - TypeScript Express app deployed to AKS with custom node pool!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployTypeScriptExpressToAksWithNodePool), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs new file mode 100644 index 00000000000..8d441d41fce --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs @@ -0,0 +1,282 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying a TypeScript Express/React Aspire application to AKS with a custom VNet/subnet. +/// Verifies that addAzureVirtualNetwork and withSubnet from a TypeScript AppHost correctly integrate +/// AKS networking so that pods receive VNet IPs rather than default overlay IPs. +/// +public sealed class TypeScriptAksVnetDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployTypeScriptExpressToAksWithVnet() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployTypeScriptExpressToAksWithVnetCore(cancellationToken); + } + + private async Task DeployTypeScriptExpressToAksWithVnetCore(CancellationToken cancellationToken) + { + // Validate prerequisites + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("ts-aks-vnet"); + var projectName = "TsAksVnet"; + + output.WriteLine($"Test: {nameof(DeployTypeScriptExpressToAksWithVnet)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + // TypeScript apphosts need the full bundle (not just the CLI binary) because + // the prebuilt AppHost server is required for aspire add to regenerate SDK code. + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + var prNumber = DeploymentE2ETestHelpers.GetPrNumber(); + if (prNumber > 0) + { + output.WriteLine($"Step 2: Installing Aspire bundle from PR #{prNumber}..."); + await auto.InstallAspireBundleFromPullRequestAsync(prNumber, counter); + } + await auto.SourceAspireBundleEnvironmentAsync(counter); + } + + // Step 3: Create TypeScript Express/React project using aspire new + output.WriteLine("Step 3: Creating TypeScript Express/React project..."); + await auto.AspireNewAsync(projectName, counter, template: AspireTemplate.ExpressReact); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5a: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5a: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitForAspireAddCompletionAsync(counter); + } + else + { + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + } + + // Step 5b: Add Aspire.Hosting.Azure.Network package (for VNet/subnet support) + output.WriteLine("Step 5b: Adding Azure Network hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Network"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitForAspireAddCompletionAsync(counter); + } + else + { + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + } + + // Step 6: Modify apphost.ts to add AKS environment with VNet/subnet + { + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostFilePath = Path.Combine(projectDir, "apphost.ts"); + + output.WriteLine($"Looking for apphost.ts at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + var originalContent = content; + + // Add VNet, subnet, and AKS environment with subnet integration before build().run() + content = content.Replace( + "await builder.build().run();", + """ +// Add VNet and subnet for AKS networking +const vnet = await builder.addAzureVirtualNetwork("vnet", "10.1.0.0/16"); +const subnet = await vnet.addSubnet("akssubnet", "10.1.0.0/22"); + +// Add Azure Kubernetes Environment with VNet integration +const aks = await builder.addAzureKubernetesEnvironment("aks"); +await aks.withSubnet(subnet); + +await builder.build().run(); +"""); + + if (content == originalContent) + { + throw new InvalidOperationException("apphost.ts was not modified. Template may have changed."); + } + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine($"Modified apphost.ts at: {appHostFilePath}"); + } + + // Step 7: Set environment for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Deploy to AKS using aspire deploy + output.WriteLine("Step 8: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + // Wait for pipeline to complete - AKS provisioning can take up to 30 minutes + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 9: Get AKS credentials for kubectl verification + output.WriteLine("Step 9: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 10: Wait for pods to be ready + output.WriteLine("Step 10: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 11: Verify pods are running and have VNet IPs (10.1.x.x, not 10.244.x.x overlay) + output.WriteLine("Step 11: Verifying pods have VNet IPs..."); + await auto.TypeAsync("kubectl get pods -o wide -n default"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("10.1.", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 12: Verify service endpoints via port-forward + output.WriteLine("Step 12: Verifying service endpoints..."); + await auto.TypeAsync("kubectl port-forward svc/api-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 13: Clean up port-forwards + output.WriteLine("Step 13: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 14: Destroy Azure deployment + output.WriteLine("Step 14: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 15: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"TypeScript AKS VNet deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployTypeScriptExpressToAksWithVnet), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - TypeScript Express app deployed to AKS with VNet integration!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployTypeScriptExpressToAksWithVnet), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Aspire.Hosting.Azure.Kubernetes.Tests.csproj b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Aspire.Hosting.Azure.Kubernetes.Tests.csproj new file mode 100644 index 00000000000..eb9779ffcbd --- /dev/null +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Aspire.Hosting.Azure.Kubernetes.Tests.csproj @@ -0,0 +1,22 @@ + + + + $(DefaultTargetFramework) + + + + + + + + + + + + + + + + + + diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs new file mode 100644 index 00000000000..e06ae52d338 --- /dev/null +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -0,0 +1,303 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable ASPIREAZURE003 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +#pragma warning disable ASPIRECOMPUTE003 // Type is for evaluation purposes only + +using System.Runtime.CompilerServices; +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Azure.Kubernetes; +using Aspire.Hosting.Kubernetes; +using Aspire.Hosting.Utils; + +namespace Aspire.Hosting.Azure.Tests; + +public class AzureKubernetesEnvironmentExtensionsTests +{ + [Fact] + public async Task AddAzureKubernetesEnvironment_BasicConfiguration() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + Assert.Equal("aks", aks.Resource.Name); + Assert.Equal("{aks.outputs.id}", aks.Resource.Id.ValueExpression); + Assert.Equal("{aks.outputs.clusterFqdn}", aks.Resource.ClusterFqdn.ValueExpression); + Assert.Equal("{aks.outputs.oidcIssuerUrl}", aks.Resource.OidcIssuerUrl.ValueExpression); + + var manifest = await AzureManifestUtils.GetManifestWithBicep(aks.Resource); + await Verify(manifest.BicepText, extension: "bicep"); + } + + [Fact] + public void AddNodePool_ReturnsNodePoolResource() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); + + // Default system pool + added user pool + Assert.Equal(2, aks.Resource.NodePools.Count); + + Assert.Equal("gpu", gpuPool.Resource.Name); + Assert.Equal("gpu", gpuPool.Resource.Config.Name); + Assert.Equal("Standard_NC6s_v3", gpuPool.Resource.Config.VmSize); + Assert.Equal(0, gpuPool.Resource.Config.MinCount); + Assert.Equal(5, gpuPool.Resource.Config.MaxCount); + Assert.Equal(AksNodePoolMode.User, gpuPool.Resource.Config.Mode); + Assert.Same(aks.Resource, gpuPool.Resource.AksParent); + } + + [Fact] + public void AddAzureKubernetesEnvironment_DefaultNodePool() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + Assert.Single(aks.Resource.NodePools); + var defaultPool = aks.Resource.NodePools[0]; + Assert.Equal("system", defaultPool.Name); + Assert.Equal("Standard_D2s_v5", defaultPool.VmSize); + Assert.Equal(1, defaultPool.MinCount); + Assert.Equal(3, defaultPool.MaxCount); + Assert.Equal(AksNodePoolMode.System, defaultPool.Mode); + } + + [Fact] + public void AddAzureKubernetesEnvironment_DefaultConfiguration() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + Assert.True(aks.Resource.OidcIssuerEnabled); + Assert.True(aks.Resource.WorkloadIdentityEnabled); + Assert.Equal(AksSkuTier.Free, aks.Resource.SkuTier); + Assert.Null(aks.Resource.KubernetesVersion); + Assert.False(aks.Resource.IsPrivateCluster); + Assert.False(aks.Resource.ContainerInsightsEnabled); + Assert.Null(aks.Resource.LogAnalyticsWorkspace); + } + + [Fact] + public void AddAzureKubernetesEnvironment_HasInternalKubernetesEnvironment() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + Assert.NotNull(aks.Resource.KubernetesEnvironment); + Assert.Equal("aks-k8s", aks.Resource.KubernetesEnvironment.Name); + } + + [Fact] + public void AddAzureKubernetesEnvironment_ThrowsOnNullBuilder() + { + IDistributedApplicationBuilder builder = null!; + + Assert.Throws(() => + builder.AddAzureKubernetesEnvironment("aks")); + } + + [Fact] + public void AddAzureKubernetesEnvironment_ThrowsOnEmptyName() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + Assert.Throws(() => + builder.AddAzureKubernetesEnvironment("")); + } + + [Fact] + public void WithWorkloadIdentity_EnablesOidcAndWorkloadIdentity() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithWorkloadIdentity(); + + Assert.True(aks.Resource.OidcIssuerEnabled); + Assert.True(aks.Resource.WorkloadIdentityEnabled); + } + + [Fact] + public void WithAzureUserAssignedIdentity_WorksWithAks() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var identity = builder.AddAzureUserAssignedIdentity("myIdentity"); + + var project = builder.AddContainer("myapi", "myimage") + .WithAzureUserAssignedIdentity(identity); + + Assert.True(project.Resource.TryGetLastAnnotation(out var appIdentity)); + Assert.Same(identity.Resource, appIdentity.IdentityResource); + } + + [Fact] + public void AzureKubernetesEnvironment_ImplementsIAzureComputeEnvironmentResource() + { + using var builder = TestDistributedApplicationBuilder.Create(); + var aks = builder.AddAzureKubernetesEnvironment("aks"); + Assert.IsAssignableFrom(aks.Resource); + } + + [Fact] + public void AzureKubernetesEnvironment_ImplementsIAzureNspAssociationTarget() + { + using var builder = TestDistributedApplicationBuilder.Create(); + var aks = builder.AddAzureKubernetesEnvironment("aks"); + Assert.IsAssignableFrom(aks.Resource); + } + + [Fact] + public void AsExisting_WorksOnAksResource() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var nameParam = builder.AddParameter("aks-name"); + var rgParam = builder.AddParameter("aks-rg"); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .AsExisting(nameParam, rgParam); + + Assert.NotNull(aks); + } + + [Fact] + public void WithSubnet_OnNodePool_StoresPerPoolSubnet() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + var defaultSubnet = vnet.AddSubnet("default-subnet", "10.0.0.0/22"); + var gpuSubnet = vnet.AddSubnet("gpu-subnet", "10.0.4.0/24"); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithSubnet(defaultSubnet); + + var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5) + .WithSubnet(gpuSubnet); + + // Environment-level subnet should be set via annotation + Assert.True(aks.Resource.TryGetLastAnnotation(out _)); + + // Per-pool subnet should be stored in NodePoolSubnets dictionary + Assert.Single(aks.Resource.NodePoolSubnets); + Assert.True(aks.Resource.NodePoolSubnets.ContainsKey("gpu")); + + // Node pool should also have its own subnet annotation + Assert.True(gpuPool.Resource.TryGetLastAnnotation(out _)); + } + + [Fact] + public void WithSubnet_OnNodePool_WithoutEnvironmentSubnet() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + var gpuSubnet = vnet.AddSubnet("gpu-subnet", "10.0.4.0/24"); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + // Only set subnet on the pool, not the environment + var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5) + .WithSubnet(gpuSubnet); + + // No environment-level subnet + Assert.False(aks.Resource.TryGetLastAnnotation(out _)); + + // Per-pool subnet should still work + Assert.Single(aks.Resource.NodePoolSubnets); + Assert.True(aks.Resource.NodePoolSubnets.ContainsKey("gpu")); + } + + [Fact] + public void WithNodePool_AddsAnnotation() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); + + var container = builder.AddContainer("myapi", "myimage") + .WithNodePool(gpuPool); + + Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); + Assert.Same(gpuPool.Resource, affinity.NodePool); + Assert.Equal("gpu", affinity.NodePool.Name); + } + + [Fact] + public void AddNodePool_MultiplePoolsSupported() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var pool1 = aks.AddNodePool("cpu", "Standard_D2s_v5", 1, 10); + var pool2 = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); + + // Default system pool + 2 user pools + Assert.Equal(3, aks.Resource.NodePools.Count); + Assert.Equal("cpu", pool1.Resource.Name); + Assert.Equal("gpu", pool2.Resource.Name); + } + + [Fact] + public void AddAzureKubernetesEnvironment_AutoCreatesDefaultRegistry() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + Assert.NotNull(aks.Resource.DefaultContainerRegistry); + Assert.Equal("aks-acr", aks.Resource.DefaultContainerRegistry.Name); + } + + [Fact] + public void WithContainerRegistry_ReplacesDefault() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var explicitAcr = builder.AddAzureContainerRegistry("my-acr"); + + aks.WithContainerRegistry(explicitAcr); + + // Default registry should be removed + Assert.Null(aks.Resource.DefaultContainerRegistry); + + // Explicit registry should be set via annotation + Assert.True(aks.Resource.TryGetLastAnnotation(out var annotation)); + Assert.Same(explicitAcr.Resource, annotation.Registry); + } + + [Fact] + public async Task ContainerRegistry_FlowsToInnerKubernetesEnvironment() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var container = builder.AddContainer("myapi", "myimage"); + + await using var app = builder.Build(); + await ExecuteBeforeStartHooksAsync(app, default); + + // The inner K8s environment should have the registry annotation + Assert.True(aks.Resource.KubernetesEnvironment + .TryGetLastAnnotation(out var annotation)); + Assert.Same(aks.Resource.DefaultContainerRegistry, annotation.Registry); + } + + [UnsafeAccessor(UnsafeAccessorKind.Method, Name = "ExecuteBeforeStartHooksAsync")] + private static extern Task ExecuteBeforeStartHooksAsync(DistributedApplication app, CancellationToken cancellationToken); +} diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs new file mode 100644 index 00000000000..c3c9bba17db --- /dev/null +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs @@ -0,0 +1,156 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable ASPIREAZURE003 + +using System.Runtime.CompilerServices; +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Azure.Kubernetes; +using Aspire.Hosting.Kubernetes; +using Aspire.Hosting.Utils; + +namespace Aspire.Hosting.Azure.Tests; + +public class AzureKubernetesInfrastructureTests +{ + [Fact] + public async Task NoUserPool_CreatesDefaultWorkloadPool() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + // No AddNodePool call — only the default system pool exists + Assert.Single(aks.Resource.NodePools); + Assert.Equal(AksNodePoolMode.System, aks.Resource.NodePools[0].Mode); + + var container = builder.AddContainer("myapi", "myimage"); + + await using var app = builder.Build(); + await ExecuteBeforeStartHooksAsync(app, default); + + // Infrastructure should have added a default "workload" user pool + Assert.Equal(2, aks.Resource.NodePools.Count); + var workloadPool = aks.Resource.NodePools.First(p => p.Mode is AksNodePoolMode.User); + Assert.Equal("workload", workloadPool.Name); + + // Compute resource should have been auto-assigned to the workload pool + Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); + Assert.Equal("workload", affinity.NodePool.Name); + } + + [Fact] + public async Task ExplicitUserPool_NoDefaultCreated() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); + + var container = builder.AddContainer("myapi", "myimage"); + + await using var app = builder.Build(); + await ExecuteBeforeStartHooksAsync(app, default); + + // Should NOT create a default pool since one already exists + Assert.Equal(2, aks.Resource.NodePools.Count); // system + gpu + Assert.DoesNotContain(aks.Resource.NodePools, p => p.Name == "workload"); + + // Unaffinitized compute resource should get assigned to the first user pool + Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); + Assert.Equal("gpu", affinity.NodePool.Name); + } + + [Fact] + public async Task ExplicitAffinity_NotOverridden() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); + var cpuPool = aks.AddNodePool("cpu", "Standard_D4s_v5", 1, 10); + + var container = builder.AddContainer("myapi", "myimage") + .WithNodePool(cpuPool); + + await using var app = builder.Build(); + await ExecuteBeforeStartHooksAsync(app, default); + + // Explicit affinity should be preserved, not overridden + Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); + Assert.Equal("cpu", affinity.NodePool.Name); + } + + [Fact] + public async Task ComputeResource_GetsDeploymentTargetFromKubernetesInfrastructure() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var container = builder.AddContainer("myapi", "myimage"); + + await using var app = builder.Build(); + await ExecuteBeforeStartHooksAsync(app, default); + + // DeploymentTargetAnnotation comes from KubernetesInfrastructure (via the inner + // KubernetesEnvironmentResource), not from AzureKubernetesInfrastructure. + Assert.True(container.Resource.TryGetLastAnnotation(out var target)); + Assert.NotNull(target.DeploymentTarget); + + // The compute environment should be the inner K8s environment + Assert.Same(aks.Resource.KubernetesEnvironment, target.ComputeEnvironment); + + // CRITICAL: ContainerRegistry must be set on the DeploymentTargetAnnotation + // so that push steps can resolve the registry endpoint + Assert.NotNull(target.ContainerRegistry); + Assert.IsType(target.ContainerRegistry); + } + + [UnsafeAccessor(UnsafeAccessorKind.Method, Name = "ExecuteBeforeStartHooksAsync")] + private static extern Task ExecuteBeforeStartHooksAsync(DistributedApplication app, CancellationToken cancellationToken); + + [Fact] + public async Task MultiEnv_ResourcesMatchCorrectEnvironment() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var registry = builder.AddAzureContainerRegistry("registry"); + var enva = builder.AddAzureKubernetesEnvironment("enva") + .WithContainerRegistry(registry); + var envb = builder.AddAzureKubernetesEnvironment("envb") + .WithContainerRegistry(registry); + + var cache = builder.AddContainer("cache", "redis") + .WithComputeEnvironment(enva); + var api = builder.AddContainer("api", "myapi") + .WithComputeEnvironment(enva); + var other = builder.AddContainer("other", "myother") + .WithComputeEnvironment(envb); + + // ParentComputeEnvironment should be set + Assert.Same(enva.Resource, enva.Resource.KubernetesEnvironment.ParentComputeEnvironment); + Assert.Same(envb.Resource, envb.Resource.KubernetesEnvironment.ParentComputeEnvironment); + + await using var app = builder.Build(); + await ExecuteBeforeStartHooksAsync(app, default); + + // cache and api should get DeploymentTargetAnnotation targeting enva + Assert.True(cache.Resource.TryGetLastAnnotation(out var cacheTarget), + "cache should have DeploymentTargetAnnotation"); + Assert.Same(enva.Resource, cacheTarget.ComputeEnvironment); + + Assert.True(api.Resource.TryGetLastAnnotation(out var apiTarget), + "api should have DeploymentTargetAnnotation"); + Assert.Same(enva.Resource, apiTarget.ComputeEnvironment); + + // other should get DeploymentTargetAnnotation targeting envb + Assert.True(other.Resource.TryGetLastAnnotation(out var otherTarget), + "other should have DeploymentTargetAnnotation"); + Assert.Same(envb.Resource, otherTarget.ComputeEnvironment); + } +} diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep new file mode 100644 index 00000000000..d48ffe64157 --- /dev/null +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep @@ -0,0 +1,52 @@ +@description('The location for the resource(s) to be deployed.') +param location string = resourceGroup().location + +resource aks 'Microsoft.ContainerService/managedClusters@2025-03-01' = { + name: take('aks-${uniqueString(resourceGroup().id)}', 63) + location: location + properties: { + agentPoolProfiles: [ + { + name: 'system' + count: 1 + vmSize: 'Standard_D2s_v5' + osType: 'Linux' + maxCount: 3 + minCount: 1 + enableAutoScaling: true + mode: 'System' + } + ] + dnsPrefix: 'aks-dns' + oidcIssuerProfile: { + enabled: true + } + securityProfile: { + workloadIdentity: { + enabled: true + } + } + } + identity: { + type: 'SystemAssigned' + } + sku: { + name: 'Base' + tier: 'Free' + } + tags: { + 'aspire-resource-name': 'aks' + } +} + +output id string = aks.id + +output name string = aks.name + +output clusterFqdn string = aks.properties.fqdn + +output oidcIssuerUrl string = aks.properties.oidcIssuerProfile.issuerURL + +output kubeletIdentityObjectId string = aks.properties.identityProfile.kubeletidentity.objectId + +output nodeResourceGroup string = aks.properties.nodeResourceGroup \ No newline at end of file