From 3f99f5b2acb9484d1c95d37455b5574fd338cb9a Mon Sep 17 00:00:00 2001 From: Mateusz Kowalski Date: Tue, 7 Jul 2026 17:15:02 +0200 Subject: [PATCH 1/2] Bug 94072: add startup probes and DNS egress to network policies Add startupProbe to nmstate-webhook and kube-rbac-proxy containers to allow sufficient time for initialization before liveness/readiness probes begin checking. Without a startupProbe, pods that take longer than ~30s to start are killed by the liveness probe and enter CrashLoopBackOff. The startupProbe uses TCP socket checks with initialDelaySeconds=10, periodSeconds=10, and failureThreshold=18, giving containers up to 3 minutes to start. Once the startupProbe succeeds, regular liveness/readiness probes take over (no initialDelaySeconds needed since startup was already verified). Additionally, add DNS egress network policies (UDP/TCP 53) for webhook, metrics, and operator pods. The existing default-deny policies block all egress except TCP 6443 (API server), which prevents DNS resolution. While in-cluster API server access uses the KUBERNETES_SERVICE_HOST IP directly, various libraries and sidecars (kube-rbac-proxy) may attempt DNS lookups during initialization, causing startup delays that compound with aggressive probe timing. Signed-off-by: Mateusz Kowalski Generated-by: AI Signed-off-by: Mateusz Kowalski (cherry picked from commit 0fd6985f52b0b361879d556cec42d015398c6fc0) --- deploy/handler/network_policy.yaml | 56 ++++++++++++++++++++++++++++++ deploy/handler/operator.yaml | 16 ++++++--- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/deploy/handler/network_policy.yaml b/deploy/handler/network_policy.yaml index 47cc0e8fb6..edfbd68a83 100644 --- a/deploy/handler/network_policy.yaml +++ b/deploy/handler/network_policy.yaml @@ -121,6 +121,25 @@ spec: --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy +metadata: + name: allow-webhook-egress-dns + namespace: {{ .HandlerNamespace }} +spec: + podSelector: + matchLabels: + app: kubernetes-nmstate + component: kubernetes-nmstate-webhook + egress: + - ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + policyTypes: + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy metadata: name: allow-metrics-egress-api-6443 namespace: {{ .HandlerNamespace }} @@ -138,6 +157,43 @@ spec: --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy +metadata: + name: allow-metrics-egress-dns + namespace: {{ .HandlerNamespace }} +spec: + podSelector: + matchLabels: + app: kubernetes-nmstate + component: kubernetes-nmstate-metrics + egress: + - ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + policyTypes: + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-operator-egress-dns + namespace: {{ .OperatorNamespace }} +spec: + podSelector: + matchLabels: + app: kubernetes-nmstate-operator + egress: + - ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + policyTypes: + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy metadata: name: default-deny-labelled-operator namespace: {{ .OperatorNamespace }} diff --git a/deploy/handler/operator.yaml b/deploy/handler/operator.yaml index 9edeb28b9e..890b13d0ab 100644 --- a/deploy/handler/operator.yaml +++ b/deploy/handler/operator.yaml @@ -92,15 +92,19 @@ spec: - containerPort: 8443 name: metrics protocol: TCP - readinessProbe: + startupProbe: tcpSocket: port: metrics initialDelaySeconds: 10 periodSeconds: 10 + failureThreshold: 18 + readinessProbe: + tcpSocket: + port: metrics + periodSeconds: 10 livenessProbe: tcpSocket: port: metrics - initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 1 successThreshold: 1 @@ -192,6 +196,12 @@ spec: - containerPort: 9443 name: webhook-server protocol: TCP + startupProbe: + tcpSocket: + port: webhook-server + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 18 readinessProbe: httpGet: path: /readyz @@ -200,7 +210,6 @@ spec: httpHeaders: - name: Content-Type value: application/json - initialDelaySeconds: 10 periodSeconds: 10 livenessProbe: httpGet: @@ -210,7 +219,6 @@ spec: httpHeaders: - name: Content-Type value: application/json - initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 1 successThreshold: 1 From 97f1ad90c05a0769a7ff546de5c2339dce2080e1 Mon Sep 17 00:00:00 2001 From: Mateusz Kowalski Date: Wed, 12 Aug 2026 17:11:40 +0200 Subject: [PATCH 2/2] OCPBUGS-94072: netpol: allow DNS egress to openshift-dns on port 5353 The allow-{webhook,metrics,operator}-egress-dns NetworkPolicies only allow egress on port 53. On OpenShift the dns-default service maps port 53 to CoreDNS pods listening on port 5353, and OVN-Kubernetes evaluates egress NetworkPolicy rules after service DNAT, so the port-53 rule never matches the actual DNS traffic and lookups from the selected pods are silently dropped. This is the documented OpenShift NetworkPolicy pattern for DNS: allow UDP/TCP 5353 towards the openshift-dns namespace. Blocked DNS is the main contributor to the 50-75s startup stall of the nmstate pods observed in OCPBUGS-94072 on a cluster with a cluster-wide proxy, which together with tight probe timing results in CrashLoopBackOff. Keep the port-53 rule for vanilla Kubernetes deployments and gate the 5353 rule on IsOpenShift, which is already provided as render data to this template. Assisted-By: Claude Fable 5 Signed-off-by: Mateusz Kowalski --- deploy/handler/network_policy.yaml | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/deploy/handler/network_policy.yaml b/deploy/handler/network_policy.yaml index edfbd68a83..09177fb328 100644 --- a/deploy/handler/network_policy.yaml +++ b/deploy/handler/network_policy.yaml @@ -135,6 +135,17 @@ spec: port: 53 - protocol: TCP port: 53 +{{- if .IsOpenShift }} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + ports: + - protocol: UDP + port: 5353 + - protocol: TCP + port: 5353 +{{- end }} policyTypes: - Egress --- @@ -171,6 +182,17 @@ spec: port: 53 - protocol: TCP port: 53 +{{- if .IsOpenShift }} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + ports: + - protocol: UDP + port: 5353 + - protocol: TCP + port: 5353 +{{- end }} policyTypes: - Egress --- @@ -189,6 +211,17 @@ spec: port: 53 - protocol: TCP port: 53 +{{- if .IsOpenShift }} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + ports: + - protocol: UDP + port: 5353 + - protocol: TCP + port: 5353 +{{- end }} policyTypes: - Egress ---