feat(linux): network isolated cluster install package via cached first

e2e/scenario_test.go

@@ -1035,6 +1035,7 @@ func Test_Ubuntu2204Gen2_Containerd_NetworkIsolatedCluster_NoneCached(t *testing
 					PrivateEgress: &datamodel.PrivateEgress{
 						Enabled:                 true,
 						ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRName(config.Config.DefaultLocation)),
+						TestMode:                true,
 					},
 				}
 				nbc.AgentPoolProfile.LocalDNSProfile = nil

parts/linux/cloud-init/artifacts/cse_cmd.sh

@@ -178,6 +178,7 @@ SYSCTL_CONTENT="{{GetSysctlContent}}"
 PRIVATE_EGRESS_PROXY_ADDRESS="{{GetPrivateEgressProxyAddress}}"
 BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="{{GetBootstrapProfileContainerRegistryServer}}"
 MCR_REPOSITORY_BASE="{{GetMCRRepositoryBase}}"
+NETWORK_ISOLATED_CLUSTER_TEST_MODE="{{GetNetworkIsolatedClusterTestMode}}" # For E2E/local AB testing only; force downloads from ACR even when cached packages exist.
 ENABLE_IMDS_RESTRICTION="{{EnableIMDSRestriction}}"
 INSERT_IMDS_RESTRICTION_RULE_TO_MANGLE_TABLE="{{InsertIMDSRestrictionRuleToMangleTable}}"
 SHOULD_ENABLE_LOCALDNS="{{ShouldEnableLocalDNS}}"

parts/linux/cloud-init/artifacts/cse_install.sh

@@ -524,6 +524,18 @@ installToolFromBootstrapProfileRegistry() {
     # Try to pull distro-specific packages (e.g., .deb for Ubuntu) from registry
     local download_root="/tmp/kubernetes/downloads" # /opt folder will return permission error
 
+    if [ "${NETWORK_ISOLATED_CLUSTER_TEST_MODE}" = "true" ]; then
+        echo "NETWORK_ISOLATED_CLUSTER_TEST_MODE=true, skipping installPackageFromCache for ${tool_name}"
+    else
+        if installPackageFromCache "$tool_name" "$version"; then
+            if [ -n "$install_path" ]; then
+                mv "$(which "$tool_name")" "$install_path"
+            fi
+            return 0
+        fi
+    fi
+    echo "install from cache failed for ${tool_name}, start to pull from registry"
+
     version_tag="${version}"
     if [ "${version}" != "v*" ]; then
         version_tag="v${version_tag}"

parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh

@@ -403,6 +403,49 @@ installNvidiaManagedExpPkgFromCache() {
   done
 }
 
+
+findCachedRpmFileName() {
+    local packageName="${1}"
+    local desiredVersion="${2}"
+    local downloadDir="${3}"
+
+    local rpmFile=""
+    rpmFile=$(ls "${downloadDir}" | grep "${packageName}" | grep "${desiredVersion}" | sort -V | tail -n 1) || rpmFile=""
+    echo "${rpmFile}"
+}
+
+buildRpmInstallArgsFromCache() {
+    local packageName="${1}"
+    local desiredVersion="${2}"
+    local rpmFile="${3}"
+    local downloadDir="${4}"
+
+    local -a rpmArgs=("${rpmFile}")
+    local -a cachedRpmFiles=()
+    mapfile -t cachedRpmFiles < <(find "${downloadDir}" -maxdepth 1 -type f -name "*.rpm" -print 2>/dev/null | sort)
+
+    # selecting the correct version of dependency rpms from the cache
+    for cachedRpm in "${cachedRpmFiles[@]}"; do
+      if [ "${cachedRpm}" = "${rpmFile}" ]; then
+        continue
+      fi
+
+      local cachedBaseName
+      cachedBaseName=$(basename "${cachedRpm}")
+
+      case "${cachedBaseName}" in
+        *${packageName}*)
+          echo "Skipping cached ${packageName} rpm ${cachedBaseName} because it does not match desired version ${desiredVersion}" >&2
+          continue
+          ;;
+      esac
+
+      rpmArgs+=("${cachedRpm}")
+    done
+
+    printf '%s\n' "${rpmArgs[@]}"
+}
+
 installRPMPackageFromFile() {
     local packageName="${1}"
     local desiredVersion="${2}"
@@ -411,7 +454,9 @@ installRPMPackageFromFile() {
     downloadDir="$(getPackageDownloadDir "${packageName}")"
 
     # check cached rpms for matching filename
-    rpmFile=$(ls "${downloadDir}" | grep "${packageName}" | grep "${desiredVersion}" | sort -V | tail -n 1) || rpmFile=""
+    local rpmFile
+    local -a rpmArgs=()
+    rpmFile="$(findCachedRpmFileName "${packageName}" "${desiredVersion}" "${downloadDir}")"
     if [ -z "${rpmFile}" ]; then
         if fallbackToKubeBinaryInstall "${packageName}" "${desiredVersion}"; then
             echo "Successfully installed ${packageName} version ${desiredVersion} from binary fallback"
@@ -428,36 +473,15 @@ installRPMPackageFromFile() {
         fi
         echo "Did not find cached rpm file, downloading ${packageName} version ${fullPackageVersion}"
         downloadPkgFromVersion "${packageName}" ${fullPackageVersion} "${downloadDir}"
-        rpmFile=$(ls "${downloadDir}" | grep "${packageName}" | grep "${desiredVersion}" | sort -V | tail -n 1) || rpmFile=""
+        rpmFile="$(findCachedRpmFileName "${packageName}" "${desiredVersion}" "${downloadDir}")"
     fi
     if [ -z "${rpmFile}" ]; then
         echo "Failed to locate ${packageName} rpm"
         return 1
     fi
 
     rpmFile="${downloadDir}/${rpmFile}"
-    local rpmArgs=("${rpmFile}")
-    local -a cachedRpmFiles=()
-    mapfile -t cachedRpmFiles < <(find "${downloadDir}" -maxdepth 1 -type f -name "*.rpm" -print 2>/dev/null | sort)
-
-    # selecting the correct version of dependency rpms from the cache
-    for cachedRpm in "${cachedRpmFiles[@]}"; do
-      if [ "${cachedRpm}" = "${rpmFile}" ]; then
-        continue
-      fi
-
-      local cachedBaseName
-      cachedBaseName=$(basename "${cachedRpm}")
-
-      case "${cachedBaseName}" in
-        *${packageName}*)
-          echo "Skipping cached ${packageName} rpm ${cachedBaseName} because it does not match desired version ${desiredVersion}"
-          continue
-          ;;
-      esac
-
-      rpmArgs+=("${cachedRpm}")
-    done
+    mapfile -t rpmArgs < <(buildRpmInstallArgsFromCache "${packageName}" "${desiredVersion}" "${rpmFile}" "${downloadDir}")
 
     if [ ${#rpmArgs[@]} -gt 1 ]; then
         echo "Installing ${packageName} with cached dependency RPMs: ${rpmArgs[*]}"
@@ -474,6 +498,56 @@ installRPMPackageFromFile() {
     rm -rf "${downloadDir}"
 }
 
+installPackageFromCache() {
+    local packageName="${1}"
+    local desiredVersion="${2}"
+    echo "installing ${packageName} version ${desiredVersion} (cache only)"
+    local downloadDir
+    downloadDir="$(getPackageDownloadDir "${packageName}")"
+
+    # check cached rpms for matching filename
+    local rpmFile=""
+    local -a rpmArgs=()
+    rpmFile="$(findCachedRpmFileName "${packageName}" "${desiredVersion}" "${downloadDir}")"
+    if [ -z "${rpmFile}" ]; then
+        echo "Failed to locate cached ${packageName} rpm for version ${desiredVersion}"
+        return 1
+    fi
+
+    rpmFile="${downloadDir}/${rpmFile}"
+    mapfile -t rpmArgs < <(buildRpmInstallArgsFromCache "${packageName}" "${desiredVersion}" "${rpmFile}" "${downloadDir}")
+
+    if [ ${#rpmArgs[@]} -gt 1 ]; then
+        echo "Installing ${packageName} with cached dependency RPMs: ${rpmArgs[*]}"
+    fi
+
+    # check if additional dependencies are required
+    local dnfPlanOutput=""
+    # Use --assumeno to inspect transaction plan without installing.
+    # dnf can still return non-zero on --assumeno, so rely on output parsing.
+    dnfPlanOutput=$(dnf install -y --assumeno --disablerepo='*' "${rpmArgs[@]}" 2>&1 || true)
+    if echo "${dnfPlanOutput}" | grep -q "Installing dependencies"; then
+      echo "Additional dependencies are required for ${packageName}; cache-only install is not allowed"
+      return 1
+    fi
+    if echo "${dnfPlanOutput}" | grep -Eq "nothing provides|requires .* but none of the providers can be installed|Problem:"; then
+      echo "Dependency resolution failed during precheck for ${packageName}"
+      return 1
+    fi
+
+    # Cache-only install: if required dependencies are not available in cache/repo, return 1.
+    # Cannot implement dnf_install because it will attempt to run makecache
+    if ! dnf install -y --disablerepo='*' "${rpmArgs[@]}"; then
+        echo "Failed to install ${packageName} from cache-only RPMs"
+        return 1
+    fi
+
+    mkdir -p /opt/bin
+    ln -snf "/usr/bin/${packageName}" "/opt/bin/${packageName}"
+    rm -rf "${downloadDir}"
+    return 0
+}
+
 downloadPkgFromVersion() {
     packageName="${1:-}"
     packageVersion="${2:-}"

parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh

@@ -328,6 +328,55 @@ installPkgWithAptGet() {
     rm -rf ${downloadDir}
 }
 
+installPackageFromCache() {
+    local packageName="${1:-}"
+    local packageVersion="${2}"
+    local downloadDir="/opt/${packageName}/downloads"
+    local debFile=""
+    local aptPlanOutput=""
+    local plannedPkg=""
+    local hasAdditionalDependencies="false"
+
+    echo "installing ${packageName} version ${packageVersion} (cache only)"
+
+    debFile=$(ls "${downloadDir}" | grep "${packageName}" | grep "${packageVersion}" | sort -V | tail -n 1) || debFile=""
+    if [ -z "${debFile}" ]; then
+        echo "Failed to locate cached ${packageName} deb for version ${packageVersion}"
+        return 1
+    fi
+
+    debFile="${downloadDir}/${debFile}"
+
+    # Simulate install first to detect whether apt would pull additional dependencies.
+    aptPlanOutput=$(apt-get -s install "${debFile}" 2>&1 || true)
+    if echo "${aptPlanOutput}" | grep -Eqi "unmet dependencies|depends:|unable to correct problems|but it is not installable"; then
+        echo "Dependency resolution failed during precheck for ${packageName}"
+        return 1
+    fi
+
+    while read -r plannedPkg; do
+        if [ -n "${plannedPkg}" ] && [ "${plannedPkg}" != "${packageName}" ]; then
+            hasAdditionalDependencies="true"
+            break
+        fi
+    done < <(echo "${aptPlanOutput}" | awk '/^Inst / {print $2}')
+
+    if [ "${hasAdditionalDependencies}" = "true" ]; then
+        echo "Additional dependencies are required for ${packageName}; cache-only install is not allowed"
+        return 1
+    fi
+
+    if ! logs_to_events "AKS.CSE.install${packageName}.installDebPackageFromFile" "installDebPackageFromFile ${debFile}"; then
+        echo "Failed to install ${packageName} from cache-only deb ${debFile}"
+        return 1
+    fi
+
+    mkdir -p /opt/bin
+    ln -snf "/usr/bin/${packageName}" "/opt/bin/${packageName}"
+    rm -rf "${downloadDir}"
+    return 0
+}
+
 downloadPkgFromVersion() {
     packageName="${1:-}"
     packageVersion="${2:-}"

spec/parts/linux/cloud-init/artifacts/cse_install_mariner_spec.sh

@@ -62,7 +62,7 @@ Describe 'cse_install_mariner.sh'
             touch "$dependencyRpm"
             touch "$conflictRpm"
             When call installRPMPackageFromFile kubelet "$desiredVersion"
-            The output should include "Skipping cached kubelet rpm $(basename "$conflictRpm") because it does not match desired version $desiredVersion"
+            The stderr should include "Skipping cached kubelet rpm $(basename "$conflictRpm") because it does not match desired version $desiredVersion"
             The output should include "Installing kubelet with cached dependency RPMs"
             The output should include "$dependencyRpm"
             The output should include "$kubeletRpm"
@@ -91,7 +91,7 @@ Describe 'cse_install_mariner.sh'
             # sort -V | tail -n 1 should pick the latest release as the primary RPM
             The output should include "dnf install 30 1 600 $release2"
             # the older release should be skipped, not added as a dependency
-            The output should include "Skipping cached kubelet rpm"
+            The stderr should include "Skipping cached kubelet rpm"
             The output should not include "$release1"
         End
 
@@ -105,6 +105,86 @@ Describe 'cse_install_mariner.sh'
         End
     End
 
+    Describe 'installPackageFromCache'
+        rpm_cache_root="$PWD/spec/tmp/rpm-cache-cacheonly"
+        DNF_PLAN_MODE="success"
+
+        setup_rpm_cache_cacheonly() {
+            RPM_PACKAGE_CACHE_BASE_DIR="$rpm_cache_root"
+            mkdir -p "$RPM_PACKAGE_CACHE_BASE_DIR/kubelet/downloads"
+        }
+
+        cleanup_rpm_cache_cacheonly() {
+            rm -rf "$rpm_cache_root"
+        }
+
+        dnf() {
+            local args="$*"
+
+            if echo "$args" | grep -q -- "--assumeno"; then
+                if [ "$DNF_PLAN_MODE" = "needs-deps" ]; then
+                    cat <<EOF
+Dependencies resolved.
+Installing dependencies:
+ container-selinux noarch 2.200.0-1.azl3 @baseos
+EOF
+                    return 1
+                fi
+
+                if [ "$DNF_PLAN_MODE" = "resolution-fail" ]; then
+                    echo "Problem: package kubelet requires container-selinux, but none of the providers can be installed"
+                    return 1
+                fi
+
+                cat <<EOF
+Dependencies resolved.
+Installing:
+ kubelet x86_64 1.34.0-5.azl3 @commandline
+EOF
+                return 1
+            fi
+
+            echo "dnf $args"
+            return 0
+        }
+
+        BeforeEach 'setup_rpm_cache_cacheonly'
+        AfterEach 'cleanup_rpm_cache_cacheonly'
+
+        It 'installs successfully from cache when no additional dependencies are required'
+            DNF_PLAN_MODE="success"
+            desiredVersion="1.34.0-5.azl3"
+            rpmDir="$RPM_PACKAGE_CACHE_BASE_DIR/kubelet/downloads"
+            kubeletRpm="$rpmDir/kubelet-${desiredVersion}.x86_64.rpm"
+            touch "$kubeletRpm"
+
+            When call installPackageFromCache kubelet "$desiredVersion"
+            The status should equal 0
+            The output should include "dnf install -y --disablerepo=* $kubeletRpm"
+        End
+
+        It 'returns failure when precheck detects additional dependencies'
+            DNF_PLAN_MODE="needs-deps"
+            desiredVersion="1.34.0-5.azl3"
+            rpmDir="$RPM_PACKAGE_CACHE_BASE_DIR/kubelet/downloads"
+            kubeletRpm="$rpmDir/kubelet-${desiredVersion}.x86_64.rpm"
+            touch "$kubeletRpm"
+
+            When call installPackageFromCache kubelet "$desiredVersion"
+            The status should equal 1
+            The output should include "Additional dependencies are required for kubelet; cache-only install is not allowed"
+        End
+
+        It 'returns failure when cached rpm is not found'
+            DNF_PLAN_MODE="success"
+            desiredVersion="1.34.0-5.azl3"
+
+            When call installPackageFromCache kubelet "$desiredVersion"
+            The status should equal 1
+            The output should include "Failed to locate cached kubelet rpm for version $desiredVersion"
+        End
+    End
+
     Describe 'should_use_nvidia_open_drivers'
         # Tests for the GPU driver selection logic
         # Returns 0 (true) for open driver (A100+, H100, H200, etc.)