diff --git a/mlir/utils/jenkins/Jenkinsfile b/mlir/utils/jenkins/Jenkinsfile index 72d5423d0574..431b08d1b000 100644 --- a/mlir/utils/jenkins/Jenkinsfile +++ b/mlir/utils/jenkins/Jenkinsfile @@ -10,6 +10,11 @@ import java.util.concurrent.ConcurrentHashMap // one instance for the whole run @Field ConcurrentHashMap DOCKER_ARGS_BY_NODE = new ConcurrentHashMap<>() +// Physical Kubernetes nodes rejected by any withHealthyNode invocation in this build. +// Jenkins agent names are ephemeral pod names, so tracking only those can let a +// replacement pod land on the same unhealthy host. +@Field +ConcurrentHashMap BLACKLISTED_K8S_NODES = new ConcurrentHashMap<>() // Jenkins Git plugin defaults to 10 minutes per command. Use 2h for fetches and // checkouts that can exceed that on a slow network. @Field @@ -1312,11 +1317,28 @@ void produceCoverageHtml(String cov, String cpath) { """ } +String currentKubernetesNode() { + // Jenkins Kubernetes templates expose this through a Pod Environment + // Variable whose field path is spec.nodeName. + return env.K8S_NODE_NAME?.trim() +} + +void blacklistKubernetesNode(String k8sNode, String agentName) { + if (!k8sNode) return + + String firstRejectedAgent = BLACKLISTED_K8S_NODES.putIfAbsent(k8sNode, agentName) + if (firstRejectedAgent) { + echo "[withHealthyNode] Kubernetes node ${k8sNode} was already blacklisted by ${firstRejectedAgent}." + } else { + echo "[withHealthyNode] Blacklisted Kubernetes node ${k8sNode} for the remainder of this build." + } +} + // Run the body on a node that passes the supplied healthChecks() block // The health check is retried on fresh executors; the body is not retried. // This function also retries the main 'body' if it fails due to a recoverable node-related issue (e.g., agent disconnect). def withHealthyNode(String baseLabel, Closure healthChecks, Closure body, int maxAttempts = 3) { - def blacklist = [] // nodes and pods that already failed the check + def blacklist = [] // Jenkins nodes and pods that already failed the check int attempt = 0 boolean done = false @@ -1329,9 +1351,23 @@ def withHealthyNode(String baseLabel, Closure healthChecks, Closure body, echo "[withHealthyNode] attempt #${attempt}: looking for '${expr}'" node(expr.toString()) { + String agentName = env.NODE_NAME + String k8sNode = currentKubernetesNode() + + if (agentName?.startsWith('k8s-') && !k8sNode) { + echo "[withHealthyNode] WARNING: ${agentName} does not expose K8S_NODE_NAME; physical-node blacklisting is disabled for this agent." + } + + String rejectedByAgent = k8sNode ? BLACKLISTED_K8S_NODES[k8sNode] : null + if (rejectedByAgent) { + echo "[withHealthyNode] Rejecting ${agentName}: Kubernetes node ${k8sNode} was blacklisted by ${rejectedByAgent}." + blacklist << agentName + return + } + // Retry ONLY the health-check. We don't want to retry the actual stages try { - stage("Health checks on ${env.NODE_NAME}") { + stage("Health checks on ${agentName}") { echo 'Cleaning up old Docker images...' def pruneStatus = sh(script: 'docker image prune -af --filter "until=720h"', returnStatus: true) if (pruneStatus != 0) { @@ -1341,14 +1377,16 @@ def withHealthyNode(String baseLabel, Closure healthChecks, Closure body, gitHealthCheck() } } catch (Exception err) { - echo "[withHealthyNode] ❌ ${env.NODE_NAME} rejected: ${err}" - blacklist << env.NODE_NAME + echo "[withHealthyNode] ❌ ${agentName} rejected: ${err}" + blacklist << agentName + blacklistKubernetesNode(k8sNode, agentName) // return exits the node {} block here, not the whole function. Some groovy magic return } stage("Node selected") { // Health-check passed. Do real work - echo "[withHealthyNode] ✅ using ${env.NODE_NAME}" + String k8sNodeSuffix = k8sNode ? " on Kubernetes node ${k8sNode}" : '' + echo "[withHealthyNode] ✅ using ${agentName}${k8sNodeSuffix}" } // Per-row console log: shStrict mirrors output here so we can classify transient // failures (e.g. GPU hang) that only appear in stdout, not in the thrown exception. @@ -1374,13 +1412,14 @@ def withHealthyNode(String baseLabel, Closure healthChecks, Closure body, // Group-1 transient on this node: blacklist it and retry the same arch on a // fresh node. The while loop continues (done still false); if attempts run out // this becomes "no healthy node found", which the post-block re-kicks whole-job. - echo "[withHealthyNode] Per-server transient on ${env.NODE_NAME}. Blacklisting the node and retrying.." + echo "[withHealthyNode] Per-server transient on ${agentName}. Blacklisting the node and retrying.." echo "[withHealthyNode] Error was: ${err}" - blacklist << env.NODE_NAME + blacklist << agentName + blacklistKubernetesNode(k8sNode, agentName) return } // Real failure (or a whole-job transient like no-healthy-node): fail immediately. - echo "[withHealthyNode] Execution failed with a non-recoverable error on ${env.NODE_NAME}" + echo "[withHealthyNode] Execution failed with a non-recoverable error on ${agentName}" echo "[withHealthyNode] Error was: ${err}" throw err } finally {