Adding out-of-tree azure credentail provider set up to CAPZ templates

capz/run-capz-e2e.sh

@@ -61,6 +61,7 @@ main() {
     if [[ "${GMSA}" == "true" ]]; then create_gmsa_domain; fi
 
     install_tools
+    prepare_cloud_provider_azure
     create_cluster
     apply_workload_configuration
     apply_cloud_provider_azure
@@ -414,12 +415,7 @@ apply_cloud_provider_azure() {
     log "entering apply_cloud_provider_azure"
     echo "KUBERNETES_VERSION = ${KUBERNETES_VERSION}"
 
-    echo "Building cloud provider images"
-    # shellcheck disable=SC1091
-    "${CAPZ_DIR}/hack/ensure-acr-login.sh"
-    # shellcheck disable=SC1091
-    source "${CAPZ_DIR}/scripts/ci-build-azure-ccm.sh" || false
-    trap run_capz_e2e_cleanup EXIT # reset the EXIT trap since ci-build-azure-ccm.sh also sets it.
+    prepare_cloud_provider_azure
     echo "Will use the ${IMAGE_REGISTRY}/${CCM_IMAGE_NAME}:${IMAGE_TAG_CCM} cloud-controller-manager image for external cloud-provider-cluster"
     echo "Will use the ${IMAGE_REGISTRY}/${CNM_IMAGE_NAME}:${IMAGE_TAG_CNM} cloud-node-manager image for external cloud-provider-azure cluster"
 
@@ -434,6 +430,27 @@ apply_cloud_provider_azure() {
     "$TOOLS_BIN_DIR"/helm upgrade cloud-provider-azure --install --namespace kube-system --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure "${CCM_IMG_ARGS[@]}"
 }
 
+prepare_cloud_provider_azure() {
+    if [[ "${CLOUD_PROVIDER_AZURE_ARTIFACTS_READY:-}" == "true" ]]; then
+        return
+    fi
+
+    log "preparing cloud-provider-azure images and credential-provider artifacts"
+    # shellcheck disable=SC1091
+    "${CAPZ_DIR}/hack/ensure-acr-login.sh"
+
+    local previous_dir
+    previous_dir="$(pwd)"
+    # shellcheck disable=SC1091
+    source "${CAPZ_DIR}/scripts/ci-build-azure-ccm.sh" || false
+    cd "${previous_dir}"
+    trap run_capz_e2e_cleanup EXIT # reset the EXIT trap since ci-build-azure-ccm.sh also sets it.
+
+    : "${AZURE_BLOB_CONTAINER_NAME:?Environment variable empty or not defined.}"
+    : "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER:?Environment variable empty or not defined.}"
+    export CLOUD_PROVIDER_AZURE_ARTIFACTS_READY="true"
+}
+
 apply_hpc_webhook(){
     log "applying configuration for HPC webhook"
 

capz/templates/ci/kustomization.yaml

@@ -12,13 +12,27 @@ patches:
     name: .*-control-plane
     namespace: default
   path: patches/kubeadm-bootstrap-control-plane-ci.yaml
+- target:
+    group: controlplane.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmControlPlane
+    name: .*-control-plane
+    namespace: default
+  path: ../patches/oot-credential-provider-kcp.yaml
 - target:
     group: bootstrap.cluster.x-k8s.io
     version: v1beta1
     kind: KubeadmConfigTemplate
     name: .*-md-win
     namespace: default
   path: patches/kubeadm-bootstrap-windows-ci.yaml
+- target:
+    group: bootstrap.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmConfigTemplate
+    name: .*-md-win
+    namespace: default
+  path: ../patches/oot-credential-provider-win.yaml
 generatorOptions:
   disableNameSuffixHash: true
   labels:

capz/templates/gmsa-ci.yaml

@@ -240,6 +240,27 @@ spec:
           kube-proxy.exe --version
         path: C:/replace-ci-binaries.ps1
         permissions: "0744"
+      - content: |
+          $ErrorActionPreference = 'Stop'
+
+          Write-Host "Attempting to log in to Azure with managed identity"
+          az login --identity > $null 2>&1
+          if ($LASTEXITCODE -eq 0) {
+            Write-Host "Logged in Azure with managed identity"
+            Write-Host "Use OOT credential provider"
+            mkdir C:\var\lib\kubelet\credential-provider
+            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider --auth-mode login
+            cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe
+            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login
+          } else {
+            Write-Host "Using curl to download the OOT credential provider"
+            mkdir C:\var\lib\kubelet\credential-provider
+            curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider
+            cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe
+            curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml
+          }
+        path: C:/oot-cred-provider.ps1
+        permissions: "0744"
       joinConfiguration:
         nodeRegistration:
           criSocket: npipe:////./pipe/containerd-containerd
@@ -261,6 +282,7 @@ spec:
       - powershell C:/replace-containerd.ps1
       - powershell C:/collect-hns-crashes.ps1
       - powershell C:/replace-ci-binaries.ps1
+      - powershell C:/oot-cred-provider.ps1
       users:
       - groups: Administrators
         name: capi
@@ -442,17 +464,49 @@ spec:
       owner: root:root
       path: /tmp/kubeadm-bootstrap.sh
       permissions: "0744"
+    - content: |
+        #!/bin/bash
+
+        set -o nounset
+        set -o pipefail
+        set -o errexit
+        [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
+
+        # Run the az login command with managed identity
+        if az login --identity > /dev/null 2>&1; then
+          echo "Logged in Azure with managed identity"
+          echo "Use OOT credential provider"
+          mkdir -p /var/lib/kubelet/credential-provider
+          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
+          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
+          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
+          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
+        else
+          echo "Using curl to download the OOT credential provider"
+          mkdir -p /var/lib/kubelet/credential-provider
+          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
+          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
+          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
+          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
+        fi
+      owner: root:root
+      path: /tmp/oot-cred-provider.sh
+      permissions: "0744"
     initConfiguration:
       nodeRegistration:
         kubeletExtraArgs:
           cloud-provider: external
           feature-gates: ${NODE_FEATURE_GATES:-""}
+          image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
+          image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
         name: '{{ ds.meta_data["local_hostname"] }}'
     joinConfiguration:
       nodeRegistration:
         kubeletExtraArgs:
           cloud-provider: external
           feature-gates: ${NODE_FEATURE_GATES:-""}
+          image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
+          image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
         name: '{{ ds.meta_data["local_hostname"] }}'
     mounts:
     - - LABEL=etcd_disk
@@ -461,6 +515,7 @@ spec:
     - bash -c /tmp/node-log-query-kubelet-config.sh
     preKubeadmCommands:
     - bash -c /tmp/kubeadm-bootstrap.sh
+    - bash -c /tmp/oot-cred-provider.sh
     useExperimentalRetryJoin: true
   machineTemplate:
     infrastructureRef:

capz/templates/gmsa-pr.yaml

@@ -235,6 +235,27 @@ spec:
           kube-proxy.exe --version
         path: C:/replace-pr-binaries.ps1
         permissions: "0744"
+      - content: |
+          $ErrorActionPreference = 'Stop'
+
+          Write-Host "Attempting to log in to Azure with managed identity"
+          az login --identity > $null 2>&1
+          if ($LASTEXITCODE -eq 0) {
+            Write-Host "Logged in Azure with managed identity"
+            Write-Host "Use OOT credential provider"
+            mkdir C:\var\lib\kubelet\credential-provider
+            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider --auth-mode login
+            cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe
+            az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login
+          } else {
+            Write-Host "Using curl to download the OOT credential provider"
+            mkdir C:\var\lib\kubelet\credential-provider
+            curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider
+            cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe
+            curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml
+          }
+        path: C:/oot-cred-provider.ps1
+        permissions: "0744"
       joinConfiguration:
         nodeRegistration:
           criSocket: npipe:////./pipe/containerd-containerd
@@ -256,6 +277,7 @@ spec:
       - powershell C:/replace-containerd.ps1
       - powershell C:/collect-hns-crashes.ps1
       - powershell C:/replace-pr-binaries.ps1
+      - powershell C:/oot-cred-provider.ps1
       users:
       - groups: Administrators
         name: capi
@@ -415,17 +437,49 @@ spec:
       owner: root:root
       path: /tmp/replace-k8s-binaries.sh
       permissions: "0744"
+    - content: |
+        #!/bin/bash
+
+        set -o nounset
+        set -o pipefail
+        set -o errexit
+        [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
+
+        # Run the az login command with managed identity
+        if az login --identity > /dev/null 2>&1; then
+          echo "Logged in Azure with managed identity"
+          echo "Use OOT credential provider"
+          mkdir -p /var/lib/kubelet/credential-provider
+          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
+          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
+          az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
+          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
+        else
+          echo "Using curl to download the OOT credential provider"
+          mkdir -p /var/lib/kubelet/credential-provider
+          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
+          chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
+          curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
+          chmod 644 /var/lib/kubelet/credential-provider-config.yaml
+        fi
+      owner: root:root
+      path: /tmp/oot-cred-provider.sh
+      permissions: "0744"
     initConfiguration:
       nodeRegistration:
         kubeletExtraArgs:
           cloud-provider: external
           feature-gates: ${NODE_FEATURE_GATES:-""}
+          image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
+          image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
         name: '{{ ds.meta_data["local_hostname"] }}'
     joinConfiguration:
       nodeRegistration:
         kubeletExtraArgs:
           cloud-provider: external
           feature-gates: ${NODE_FEATURE_GATES:-""}
+          image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
+          image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
         name: '{{ ds.meta_data["local_hostname"] }}'
     mounts:
     - - LABEL=etcd_disk
@@ -434,6 +488,7 @@ spec:
     - bash -c /tmp/node-log-query-kubelet-config.sh
     preKubeadmCommands:
     - bash -c /tmp/replace-k8s-binaries.sh
+    - bash -c /tmp/oot-cred-provider.sh
     useExperimentalRetryJoin: true
   machineTemplate:
     infrastructureRef:

capz/templates/patches/oot-credential-provider-kcp.yaml

@@ -0,0 +1,53 @@
+# Source: https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/e8b9ce372d09fdf3e6b91300eec23715e1ad5a6f/templates/test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml
+# Keep this patch in sync with the upstream CAPZ prow-ci-version patch when updating.
+- op: add
+  path: /spec/kubeadmConfigSpec/files/-
+  value:
+    content: |
+      #!/bin/bash
+
+      set -o nounset
+      set -o pipefail
+      set -o errexit
+      [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
+
+      # Run the az login command with managed identity
+      if az login --identity > /dev/null 2>&1; then
+        echo "Logged in Azure with managed identity"
+        echo "Use OOT credential provider"
+        mkdir -p /var/lib/kubelet/credential-provider
+        az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
+        chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
+        az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
+        chmod 644 /var/lib/kubelet/credential-provider-config.yaml
+      else
+        echo "Using curl to download the OOT credential provider"
+        mkdir -p /var/lib/kubelet/credential-provider
+        curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
+        chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
+        curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
+        chmod 644 /var/lib/kubelet/credential-provider-config.yaml
+      fi
+    path: /tmp/oot-cred-provider.sh
+    owner: "root:root"
+    permissions: "0744"
+- op: add
+  path: /spec/kubeadmConfigSpec/preKubeadmCommands/-
+  value:
+    bash -c /tmp/oot-cred-provider.sh
+- op: add
+  path: /spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir
+  value:
+    /var/lib/kubelet/credential-provider
+- op: add
+  path: /spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config
+  value:
+    /var/lib/kubelet/credential-provider-config.yaml
+- op: add
+  path: /spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir
+  value:
+    /var/lib/kubelet/credential-provider
+- op: add
+  path: /spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config
+  value:
+    /var/lib/kubelet/credential-provider-config.yaml