From 30d73d6c9080428dc160971568f3a325b8cc39a3 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Mon, 8 Dec 2025 15:44:46 +0800 Subject: [PATCH 01/30] make sure x ip crd exist and then update subnet status v4|6 using range Signed-off-by: zbb88888 --- pkg/controller/ovn_eip.go | 6 ++++++ pkg/controller/vip.go | 16 +++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index 388ba3d5b77..c9afe116a67 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -51,6 +51,12 @@ func (c *Controller) enqueueUpdateOvnEip(oldObj, newObj any) { klog.Infof("enqueue update ovn eip %s", key) c.updateOvnEipQueue.Add(key) } + + // Trigger subnet status update when EIP status is updated + // This ensures both CR and IPAM state are synced before status calculation + if oldEip.Status.V4Ip != newEip.Status.V4Ip || oldEip.Status.V6Ip != newEip.Status.V6Ip || oldEip.Status.Ready != newEip.Status.Ready { + c.updateSubnetStatusQueue.Add(newEip.Spec.ExternalSubnet) + } } func (c *Controller) enqueueDelOvnEip(obj any) { diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index 722114975d1..1f12d8993a3 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -42,6 +42,12 @@ func (c *Controller) enqueueUpdateVirtualIP(oldObj, newObj any) { klog.Infof("enqueue update virtual parents for %s", key) c.updateVirtualParentsQueue.Add(key) } + + // Trigger subnet status update when VIP status is updated + // This ensures both CR and IPAM state are synced before status calculation + if oldVip.Status.V4ip != newVip.Status.V4ip || oldVip.Status.V6ip != newVip.Status.V6ip || oldVip.Status.Mac != newVip.Status.Mac { + c.updateSubnetStatusQueue.Add(newVip.Spec.Subnet) + } } func (c *Controller) enqueueDelVirtualIP(obj any) { @@ -346,6 +352,7 @@ func (c *Controller) handleUpdateVirtualParents(key string) error { func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string) error { vipCR, err := c.virtualIpsLister.Get(key) + needUpdateSubnetStatus := false if err != nil { if k8serrors.IsNotFound(err) { if _, err := c.config.KubeOvnClient.KubeovnV1().Vips().Create(context.Background(), &kubeovnv1.Vip{ @@ -369,6 +376,8 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string klog.Error(err) return err } + // New VIP created, trigger subnet status update + needUpdateSubnetStatus = true } else { err := fmt.Errorf("failed to get crd vip '%s', %w", key, err) klog.Error(err) @@ -396,6 +405,8 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string klog.Error(err) return err } + // VIP status updated with IP allocation, trigger subnet status update + needUpdateSubnetStatus = true } var needUpdateLabel bool var op string @@ -424,7 +435,10 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string } } } - c.updateSubnetStatusQueue.Add(subnet) + // Only trigger subnet status update when VIP is actually created or status is updated + if needUpdateSubnetStatus { + c.updateSubnetStatusQueue.Add(subnet) + } return nil } From 24454dd221d42369534aed3681158e3076ca15e8 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Mon, 8 Dec 2025 19:14:21 +0800 Subject: [PATCH 02/30] fix: make sue all kinds of ip 1. make sure ip CR exist and then update subnet status 2. make sure ipam release ip and ip CR deleted and then update subnet status 3. add e2e for all ip creation and deletion Signed-off-by: zbb88888 --- makefiles/e2e.mk | 10 + pkg/controller/controller.go | 8 +- pkg/controller/ip.go | 93 +++-- pkg/controller/ovn_eip.go | 152 ++++---- pkg/controller/pod.go | 3 + pkg/controller/vip.go | 134 ++++--- pkg/controller/vpc_nat_gw_eip.go | 107 +++--- test/e2e/ip/e2e_test.go | 445 +++++++++++++++++++++++ test/e2e/iptables-vpc-nat-gw/e2e_test.go | 377 +++++++++++++++++++ test/e2e/ovn-vpc-nat-gw/e2e_test.go | 262 +++++++++++++ test/e2e/vip/e2e_test.go | 190 +++++++++- 11 files changed, 1566 insertions(+), 215 deletions(-) create mode 100644 test/e2e/ip/e2e_test.go diff --git a/makefiles/e2e.mk b/makefiles/e2e.mk index 92db5979fd4..ec8d053ac17 100644 --- a/makefiles/e2e.mk +++ b/makefiles/e2e.mk @@ -87,6 +87,7 @@ e2e-build: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/multus ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/non-primary-cni ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ip ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vip ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vpc-egress-gateway ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw @@ -188,6 +189,15 @@ kube-ovn-lb-svc-conformance-e2e: ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ --focus=CNI:Kube-OVN ./test/e2e/lb-svc/lb-svc.test -- $(TEST_BIN_ARGS) +.PHONY: ip-conformance-e2e +ip-conformance-e2e: + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ip + E2E_BRANCH=$(E2E_BRANCH) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ + ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ + --focus=CNI:Kube-OVN ./test/e2e/ip/ip.test -- $(TEST_BIN_ARGS) + .PHONY: vip-conformance-e2e vip-conformance-e2e: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vip diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index d13e8a72a9c..984a659ea93 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -159,7 +159,7 @@ type Controller struct { addIptablesEipQueue workqueue.TypedRateLimitingInterface[string] updateIptablesEipQueue workqueue.TypedRateLimitingInterface[string] resetIptablesEipQueue workqueue.TypedRateLimitingInterface[string] - delIptablesEipQueue workqueue.TypedRateLimitingInterface[string] + delIptablesEipQueue workqueue.TypedRateLimitingInterface[*kubeovnv1.IptablesEIP] iptablesFipsLister kubeovnlister.IptablesFIPRuleLister iptablesFipSynced cache.InformerSynced @@ -184,7 +184,7 @@ type Controller struct { addOvnEipQueue workqueue.TypedRateLimitingInterface[string] updateOvnEipQueue workqueue.TypedRateLimitingInterface[string] resetOvnEipQueue workqueue.TypedRateLimitingInterface[string] - delOvnEipQueue workqueue.TypedRateLimitingInterface[string] + delOvnEipQueue workqueue.TypedRateLimitingInterface[*kubeovnv1.OvnEip] ovnFipsLister kubeovnlister.OvnFipLister ovnFipSynced cache.InformerSynced @@ -472,7 +472,7 @@ func Run(ctx context.Context, config *Configuration) { addIptablesEipQueue: newTypedRateLimitingQueue("AddIptablesEip", custCrdRateLimiter), updateIptablesEipQueue: newTypedRateLimitingQueue("UpdateIptablesEip", custCrdRateLimiter), resetIptablesEipQueue: newTypedRateLimitingQueue("ResetIptablesEip", custCrdRateLimiter), - delIptablesEipQueue: newTypedRateLimitingQueue("DeleteIptablesEip", custCrdRateLimiter), + delIptablesEipQueue: newTypedRateLimitingQueue[*kubeovnv1.IptablesEIP]("DeleteIptablesEip", nil), iptablesFipsLister: iptablesFipInformer.Lister(), iptablesFipSynced: iptablesFipInformer.Informer().HasSynced, @@ -563,7 +563,7 @@ func Run(ctx context.Context, config *Configuration) { addOvnEipQueue: newTypedRateLimitingQueue("AddOvnEip", custCrdRateLimiter), updateOvnEipQueue: newTypedRateLimitingQueue("UpdateOvnEip", custCrdRateLimiter), resetOvnEipQueue: newTypedRateLimitingQueue("ResetOvnEip", custCrdRateLimiter), - delOvnEipQueue: newTypedRateLimitingQueue("DeleteOvnEip", custCrdRateLimiter), + delOvnEipQueue: newTypedRateLimitingQueue[*kubeovnv1.OvnEip]("DeleteOvnEip", nil), ovnFipsLister: ovnFipInformer.Lister(), ovnFipSynced: ovnFipInformer.Informer().HasSynced, diff --git a/pkg/controller/ip.go b/pkg/controller/ip.go index f475cac705b..27a791fccb5 100644 --- a/pkg/controller/ip.go +++ b/pkg/controller/ip.go @@ -8,7 +8,6 @@ import ( "maps" "net" "reflect" - "slices" "strings" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -30,12 +29,6 @@ func (c *Controller) enqueueAddIP(obj any) { if strings.HasPrefix(ipObj.Name, util.U2OInterconnName[0:19]) { return } - klog.V(3).Infof("enqueue update status subnet %s", ipObj.Spec.Subnet) - c.updateSubnetStatusQueue.Add(ipObj.Spec.Subnet) - for _, as := range ipObj.Spec.AttachSubnets { - klog.V(3).Infof("enqueue update attach status for subnet %s", as) - c.updateSubnetStatusQueue.Add(as) - } key := cache.MetaObjectToName(ipObj).String() klog.V(3).Infof("enqueue add ip %s", key) @@ -88,13 +81,6 @@ func (c *Controller) enqueueUpdateIP(oldObj, newObj any) { c.updateIPQueue.Add(key) return } - if !slices.Equal(oldIP.Spec.AttachSubnets, newIP.Spec.AttachSubnets) { - klog.V(3).Infof("enqueue update status subnet %s", newIP.Spec.Subnet) - for _, as := range newIP.Spec.AttachSubnets { - klog.V(3).Infof("enqueue update status for attach subnet %s", as) - c.updateSubnetStatusQueue.Add(as) - } - } } func (c *Controller) enqueueDelIP(obj any) { @@ -177,7 +163,12 @@ func (c *Controller) handleAddReservedIP(key string) error { } if lsp != nil { // port already exists means the ip already created - klog.V(3).Infof("ip %s is ready", portName) + // but we still need to ensure finalizer is added + klog.V(3).Infof("ip %s is ready, checking finalizer", portName) + if err = c.handleAddOrUpdateIPFinalizer(ip); err != nil { + klog.Errorf("failed to handle add or update finalizer for ip %s: %v", ip.Name, err) + return err + } return nil } @@ -220,6 +211,13 @@ func (c *Controller) handleAddReservedIP(key string) error { return err } } + + // Handle add or update finalizer after IP is created/updated + if err = c.handleAddOrUpdateIPFinalizer(ip); err != nil { + klog.Errorf("failed to handle add or update finalizer for ip %s: %v", ip.Name, err) + return err + } + return nil } @@ -232,6 +230,13 @@ func (c *Controller) handleUpdateIP(key string) error { klog.Error(err) return err } + + // Handle add or update finalizer + if err = c.handleAddOrUpdateIPFinalizer(cachedIP); err != nil { + klog.Errorf("failed to handle add or update finalizer for ip %s: %v", key, err) + return err + } + if !cachedIP.DeletionTimestamp.IsZero() { klog.Infof("handle deleting ip %s", cachedIP.Name) subnet, err := c.subnetsLister.Get(cachedIP.Spec.Subnet) @@ -272,18 +277,24 @@ func (c *Controller) handleUpdateIP(key string) error { klog.Errorf("failed to handle del ip finalizer %v", err) return err } - c.updateSubnetStatusQueue.Add(cachedIP.Spec.Subnet) } return nil } func (c *Controller) handleDelIP(ip *kubeovnv1.IP) error { - klog.Infof("deleting ip %s enqueue update status subnet %s", ip.Name, ip.Spec.Subnet) - c.updateSubnetStatusQueue.Add(ip.Spec.Subnet) + klog.Infof("deleting ip %s", ip.Name) + + // For IP CRs deleted without finalizer (race condition or direct deletion), + // we need to ensure subnet status is updated. + // Note: IPAM release should have been done before this (either in handleUpdateIP + // or in pod controller), but we trigger subnet status update here as a safety net. + if ip.Spec.Subnet != "" { + c.updateSubnetStatusQueue.Add(ip.Spec.Subnet) + } for _, as := range ip.Spec.AttachSubnets { - klog.V(3).Infof("enqueue update attach status for subnet %s", as) c.updateSubnetStatusQueue.Add(as) } + return nil } @@ -298,6 +309,41 @@ func (c *Controller) syncIPFinalizer(cl client.Client) error { }) } +func (c *Controller) handleAddOrUpdateIPFinalizer(cachedIP *kubeovnv1.IP) error { + if !cachedIP.DeletionTimestamp.IsZero() { + // IP is being deleted, don't handle finalizer add/update + return nil + } + if len(cachedIP.GetFinalizers()) != 0 { + // Finalizer already exists + return nil + } + + newIP := cachedIP.DeepCopy() + controllerutil.AddFinalizer(newIP, util.KubeOVNControllerFinalizer) + patch, err := util.GenerateMergePatchPayload(cachedIP, newIP) + if err != nil { + klog.Errorf("failed to generate patch payload for ip '%s', %v", cachedIP.Name, err) + return err + } + if _, err := c.config.KubeOvnClient.KubeovnV1().IPs().Patch(context.Background(), cachedIP.Name, + types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + klog.Errorf("failed to add finalizer for ip '%s', %v", cachedIP.Name, err) + return err + } + + // Trigger subnet status update after finalizer is added + // This ensures subnet status reflects the new IP allocation + c.updateSubnetStatusQueue.Add(cachedIP.Spec.Subnet) + for _, as := range cachedIP.Spec.AttachSubnets { + c.updateSubnetStatusQueue.Add(as) + } + return nil +} + func (c *Controller) handleDelIPFinalizer(cachedIP *kubeovnv1.IP) error { if len(cachedIP.GetFinalizers()) == 0 { return nil @@ -318,6 +364,13 @@ func (c *Controller) handleDelIPFinalizer(cachedIP *kubeovnv1.IP) error { klog.Errorf("failed to remove finalizer from ip %s, %v", cachedIP.Name, err) return err } + + // Trigger subnet status update after finalizer is removed + // This ensures subnet status reflects the IP release + c.updateSubnetStatusQueue.Add(cachedIP.Spec.Subnet) + for _, as := range cachedIP.Spec.AttachSubnets { + c.updateSubnetStatusQueue.Add(as) + } return nil } @@ -420,7 +473,6 @@ func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, subnetName: "", util.IPReservedLabel: "false", // ip create with pod or node, ip not reserved }, - Finalizers: []string{util.KubeOVNControllerFinalizer}, }, Spec: kubeovnv1.IPSpec{ PodName: key, @@ -469,7 +521,6 @@ func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, if maps.Equal(newIPCR.Labels, ipCR.Labels) && reflect.DeepEqual(newIPCR.Spec, ipCR.Spec) { return nil } - controllerutil.AddFinalizer(newIPCR, util.KubeOVNControllerFinalizer) if _, err = c.config.KubeOvnClient.KubeovnV1().IPs().Update(context.Background(), newIPCR, metav1.UpdateOptions{}); err != nil { err := fmt.Errorf("failed to update ip CR %s: %w", ipCRName, err) diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index c9afe116a67..71532831a5b 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -36,7 +36,7 @@ func (c *Controller) enqueueUpdateOvnEip(oldObj, newObj any) { return } klog.Infof("enqueue del ovn eip %s", key) - c.delOvnEipQueue.Add(key) + c.delOvnEipQueue.Add(newEip) return } oldEip := oldObj.(*kubeovnv1.OvnEip) @@ -52,11 +52,6 @@ func (c *Controller) enqueueUpdateOvnEip(oldObj, newObj any) { c.updateOvnEipQueue.Add(key) } - // Trigger subnet status update when EIP status is updated - // This ensures both CR and IPAM state are synced before status calculation - if oldEip.Status.V4Ip != newEip.Status.V4Ip || oldEip.Status.V6Ip != newEip.Status.V6Ip || oldEip.Status.Ready != newEip.Status.Ready { - c.updateSubnetStatusQueue.Add(newEip.Spec.ExternalSubnet) - } } func (c *Controller) enqueueDelOvnEip(obj any) { @@ -78,7 +73,7 @@ func (c *Controller) enqueueDelOvnEip(obj any) { key := cache.MetaObjectToName(eip).String() klog.Infof("enqueue del ovn eip %s", key) - c.delOvnEipQueue.Add(key) + c.delOvnEipQueue.Add(eip) } func (c *Controller) handleAddOvnEip(key string) error { @@ -149,11 +144,10 @@ func (c *Controller) handleAddOvnEip(key string) error { return err } } - if err = c.handleAddOvnEipFinalizer(cachedEip); err != nil { + if err = c.handleAddOrUpdateOvnEipFinalizer(cachedEip); err != nil { klog.Errorf("failed to add finalizer for ovn eip, %v", err) return err } - c.updateSubnetStatusQueue.Add(subnetName) return nil } @@ -166,6 +160,54 @@ func (c *Controller) handleUpdateOvnEip(key string) error { klog.Error(err) return err } + + // Handle deletion first + if !cachedEip.DeletionTimestamp.IsZero() { + klog.Infof("handle deleting ovn eip %s", key) + + // Check if EIP is still being used by any NAT rules (FIP/DNAT/SNAT) BEFORE cleanup + // Only proceed with cleanup and finalizer removal when no NAT rules are using it + nat, err := c.getOvnEipNat(cachedEip.Spec.V4Ip) + if err != nil { + klog.Errorf("failed to get ovn eip %s nat rules, %v", key, err) + return err + } + if nat != "" { + klog.Infof("ovn eip %s is still being used by NAT rules: %s, waiting for them to be deleted", key, nat) + return nil + } + + // Clean up resources before removing finalizer + if cachedEip.Spec.Type == util.OvnEipTypeLSP { + if err := c.OVNNbClient.DeleteLogicalSwitchPort(cachedEip.Name); err != nil { + klog.Errorf("failed to delete lsp %s, %v", cachedEip.Name, err) + return err + } + } + if cachedEip.Spec.Type == util.OvnEipTypeLRP { + if err := c.OVNNbClient.DeleteLogicalRouterPort(cachedEip.Name); err != nil { + klog.Errorf("failed to delete lrp %s, %v", cachedEip.Name, err) + return err + } + } + + // Release IP from IPAM before removing finalizer + c.ipam.ReleaseAddressByPod(cachedEip.Name, cachedEip.Spec.ExternalSubnet) + + // Now remove finalizer, which will trigger subnet status update + if err = c.handleDelOvnEipFinalizer(cachedEip); err != nil { + klog.Errorf("failed to handle remove ovn eip finalizer , %v", err) + return err + } + return nil + } + + // Always ensure finalizer is added regardless of Status + if err = c.handleAddOrUpdateOvnEipFinalizer(cachedEip); err != nil { + klog.Errorf("failed to handle add or update finalizer for ovn eip %s: %v", key, err) + return err + } + if !cachedEip.Status.Ready { // create eip only in add process, just check to error out here klog.Infof("wait ovn eip %s to be ready only in the handle add process", cachedEip.Name) @@ -222,36 +264,17 @@ func (c *Controller) handleResetOvnEip(key string) error { return nil } -func (c *Controller) handleDelOvnEip(key string) error { - klog.Infof("handle del ovn eip %s", key) - eip, err := c.ovnEipsLister.Get(key) - if err != nil { - if k8serrors.IsNotFound(err) { - return nil - } - klog.Error(err) - return err - } +func (c *Controller) handleDelOvnEip(eip *kubeovnv1.OvnEip) error { + // Cleanup is now handled in handleUpdateOvnEip before finalizer removal + // This function is kept for compatibility with the delete queue + klog.V(3).Infof("ovn eip %s cleanup already done in update handler", eip.Name) - if eip.Spec.Type == util.OvnEipTypeLSP { - if err := c.OVNNbClient.DeleteLogicalSwitchPort(eip.Name); err != nil { - klog.Errorf("failed to delete lsp %s, %v", eip.Name, err) - return err - } - } - if eip.Spec.Type == util.OvnEipTypeLRP { - if err := c.OVNNbClient.DeleteLogicalRouterPort(eip.Name); err != nil { - klog.Errorf("failed to delete lrp %s, %v", eip.Name, err) - return err - } + // For OvnEips deleted without finalizer (race condition or direct deletion), + // we need to ensure subnet status is updated as a safety net. + if eip.Spec.ExternalSubnet != "" { + c.updateSubnetStatusQueue.Add(eip.Spec.ExternalSubnet) } - if err = c.handleDelOvnEipFinalizer(eip); err != nil { - klog.Errorf("failed to handle remove ovn eip finalizer , %v", err) - return err - } - c.ipam.ReleaseAddressByPod(eip.Name, eip.Spec.ExternalSubnet) - c.updateSubnetStatusQueue.Add(eip.Spec.ExternalSubnet) return nil } @@ -290,8 +313,17 @@ func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usageT } } else { ovnEip := cachedEip.DeepCopy() - needUpdate := false + // Ensure labels are set correctly before any update + if ovnEip.Labels == nil { + ovnEip.Labels = make(map[string]string) + } + ovnEip.Labels[util.SubnetNameLabel] = subnet + ovnEip.Labels[util.OvnEipTypeLabel] = usageType + ovnEip.Labels[util.EipV4IpLabel] = v4ip + ovnEip.Labels[util.EipV6IpLabel] = v6ip + + needUpdate := false if mac != "" && ovnEip.Spec.MacAddress != mac { ovnEip.Spec.MacAddress = mac needUpdate = true @@ -309,6 +341,7 @@ func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usageT needUpdate = true } if needUpdate { + // Update with labels and spec in one call if _, err := c.config.KubeOvnClient.KubeovnV1().OvnEips().Update(context.Background(), ovnEip, metav1.UpdateOptions{}); err != nil { errMsg := fmt.Errorf("failed to update ovn eip '%s', %w", key, err) klog.Error(errMsg) @@ -347,41 +380,6 @@ func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usageT return err } } - - var needUpdateLabel bool - var op string - if len(ovnEip.Labels) == 0 { - op = "add" - ovnEip.Labels = map[string]string{ - util.SubnetNameLabel: subnet, - util.OvnEipTypeLabel: usageType, - util.EipV4IpLabel: v4ip, - util.EipV6IpLabel: v6ip, - } - needUpdateLabel = true - } - if ovnEip.Labels[util.SubnetNameLabel] != subnet { - op = "replace" - ovnEip.Labels[util.SubnetNameLabel] = subnet - ovnEip.Labels[util.EipV4IpLabel] = v4ip - ovnEip.Labels[util.EipV6IpLabel] = v6ip - needUpdateLabel = true - } - if ovnEip.Labels[util.OvnEipTypeLabel] != usageType { - op = "replace" - ovnEip.Labels[util.OvnEipTypeLabel] = usageType - needUpdateLabel = true - } - if needUpdateLabel { - patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/labels", "value": %s }]` - raw, _ := json.Marshal(ovnEip.Labels) - patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - if _, err := c.config.KubeOvnClient.KubeovnV1().OvnEips().Patch(context.Background(), ovnEip.Name, types.JSONPatchType, - []byte(patchPayload), metav1.PatchOptions{}); err != nil { - klog.Errorf("failed to patch label for ovn eip '%s', %v", ovnEip.Name, err) - return err - } - } } return nil } @@ -510,7 +508,7 @@ func (c *Controller) syncOvnEipFinalizer(cl client.Client) error { }) } -func (c *Controller) handleAddOvnEipFinalizer(cachedEip *kubeovnv1.OvnEip) error { +func (c *Controller) handleAddOrUpdateOvnEipFinalizer(cachedEip *kubeovnv1.OvnEip) error { if !cachedEip.DeletionTimestamp.IsZero() || len(cachedEip.GetFinalizers()) != 0 { return nil } @@ -529,6 +527,10 @@ func (c *Controller) handleAddOvnEipFinalizer(cachedEip *kubeovnv1.OvnEip) error klog.Errorf("failed to add finalizer for ovn eip '%s', %v", cachedEip.Name, err) return err } + + // Trigger subnet status update after finalizer is added + // This ensures subnet status reflects the new OVN EIP allocation + c.updateSubnetStatusQueue.Add(cachedEip.Spec.ExternalSubnet) return nil } @@ -564,6 +566,10 @@ func (c *Controller) handleDelOvnEipFinalizer(cachedEip *kubeovnv1.OvnEip) error klog.Errorf("failed to remove finalizer from ovn eip '%s', %v", cachedEip.Name, err) return err } + + // Trigger subnet status update after finalizer is removed + // This ensures subnet status reflects the IP release + c.updateSubnetStatusQueue.Add(cachedEip.Spec.ExternalSubnet) return nil } diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 87ed37c1709..4e4452bc1fe 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -1204,6 +1204,9 @@ func (c *Controller) handleDeletePod(key string) (err error) { } // release ipam address after delete ip CR c.ipam.ReleaseAddressByNic(podKey, portName, podNet.Subnet.Name) + // Trigger subnet status update after IPAM release + // This is needed when IP CR is deleted without finalizer (race condition) + c.updateSubnetStatusQueue.Add(podNet.Subnet.Name) } } if pod.Annotations[util.VipAnnotation] != "" { diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index 1f12d8993a3..1950514f1ff 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -43,11 +43,6 @@ func (c *Controller) enqueueUpdateVirtualIP(oldObj, newObj any) { c.updateVirtualParentsQueue.Add(key) } - // Trigger subnet status update when VIP status is updated - // This ensures both CR and IPAM state are synced before status calculation - if oldVip.Status.V4ip != newVip.Status.V4ip || oldVip.Status.V6ip != newVip.Status.V6ip || oldVip.Status.Mac != newVip.Status.Mac { - c.updateSubnetStatusQueue.Add(newVip.Spec.Subnet) - } } func (c *Controller) enqueueDelVirtualIP(obj any) { @@ -166,6 +161,10 @@ func (c *Controller) handleAddVirtualIP(key string) error { if vip.Spec.Type == util.KubeHostVMVip { // vm use the vip as its real ip klog.Infof("created host network pod vm ip %s", key) + if err = c.handleAddOrUpdateVipFinalizer(key); err != nil { + klog.Errorf("failed to handle add or update vip finalizer %v", err) + return err + } return nil } if err := c.handleUpdateVirtualParents(key); err != nil { @@ -173,6 +172,10 @@ func (c *Controller) handleAddVirtualIP(key string) error { klog.Error(err) return err } + if err = c.handleAddOrUpdateVipFinalizer(key); err != nil { + klog.Errorf("failed to handle add or update vip finalizer %v", err) + return err + } return nil } @@ -188,6 +191,31 @@ func (c *Controller) handleUpdateVirtualIP(key string) error { vip := cachedVip.DeepCopy() // should delete if !vip.DeletionTimestamp.IsZero() { + klog.Infof("handle deleting vip %s", vip.Name) + // Clean up resources before removing finalizer + if vip.Spec.Type != "" { + subnet, err := c.subnetsLister.Get(vip.Spec.Subnet) + if err != nil { + klog.Errorf("failed to get subnet %s: %v", vip.Spec.Subnet, err) + return err + } + portName := ovs.PodNameToPortName(vip.Name, vip.Spec.Namespace, subnet.Spec.Provider) + klog.Infof("delete vip lsp %s", portName) + if err := c.OVNNbClient.DeleteLogicalSwitchPort(portName); err != nil { + err = fmt.Errorf("failed to delete lsp %s: %w", vip.Name, err) + klog.Error(err) + return err + } + } + // delete virtual ports + if err := c.OVNNbClient.DeleteLogicalSwitchPort(vip.Name); err != nil { + klog.Errorf("delete virtual logical switch port %s from logical switch %s: %v", vip.Name, vip.Spec.Subnet, err) + return err + } + // Release IP from IPAM before removing finalizer + c.ipam.ReleaseAddressByPod(vip.Name, vip.Spec.Subnet) + + // Now remove finalizer, which will trigger subnet status update if err = c.handleDelVipFinalizer(key); err != nil { klog.Errorf("failed to handle vip finalizer %v", err) return err @@ -223,38 +251,26 @@ func (c *Controller) handleUpdateVirtualIP(key string) error { klog.Error(err) return err } - if err = c.handleAddVipFinalizer(key); err != nil { - klog.Errorf("failed to handle vip finalizer %v", err) - return err - } + } + // Always ensure finalizer is added regardless of Status + if err = c.handleAddOrUpdateVipFinalizer(key); err != nil { + klog.Errorf("failed to handle vip finalizer %v", err) + return err } return nil } func (c *Controller) handleDelVirtualIP(vip *kubeovnv1.Vip) error { - klog.Infof("handle delete vip %s", vip.Name) - // TODO:// clean vip in its parent port aap list - if vip.Spec.Type != "" { - subnet, err := c.subnetsLister.Get(vip.Spec.Subnet) - if err != nil { - klog.Errorf("failed to get subnet %s: %v", vip.Spec.Subnet, err) - return err - } - portName := ovs.PodNameToPortName(vip.Name, vip.Spec.Namespace, subnet.Spec.Provider) - klog.Infof("delete vip lsp %s", portName) - if err := c.OVNNbClient.DeleteLogicalSwitchPort(portName); err != nil { - err = fmt.Errorf("failed to delete lsp %s: %w", vip.Name, err) - klog.Error(err) - return err - } - } - // delete virtual ports - if err := c.OVNNbClient.DeleteLogicalSwitchPort(vip.Name); err != nil { - klog.Errorf("delete virtual logical switch port %s from logical switch %s: %v", vip.Name, vip.Spec.Subnet, err) - return err + // Cleanup is now handled in handleUpdateVirtualIP before finalizer removal + // This function is kept for compatibility with the delete queue + klog.V(3).Infof("vip %s cleanup already done in update handler", vip.Name) + + // For VIPs deleted without finalizer (race condition or direct deletion), + // we need to ensure subnet status is updated as a safety net. + if vip.Spec.Subnet != "" { + c.updateSubnetStatusQueue.Add(vip.Spec.Subnet) } - c.ipam.ReleaseAddressByPod(vip.Name, vip.Spec.Subnet) - c.updateSubnetStatusQueue.Add(vip.Spec.Subnet) + return nil } @@ -352,7 +368,6 @@ func (c *Controller) handleUpdateVirtualParents(key string) error { func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string) error { vipCR, err := c.virtualIpsLister.Get(key) - needUpdateSubnetStatus := false if err != nil { if k8serrors.IsNotFound(err) { if _, err := c.config.KubeOvnClient.KubeovnV1().Vips().Create(context.Background(), &kubeovnv1.Vip{ @@ -376,8 +391,6 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string klog.Error(err) return err } - // New VIP created, trigger subnet status update - needUpdateSubnetStatus = true } else { err := fmt.Errorf("failed to get crd vip '%s', %w", key, err) klog.Error(err) @@ -385,6 +398,14 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string } } else { vip := vipCR.DeepCopy() + + // Ensure labels are set correctly + if vip.Labels == nil { + vip.Labels = make(map[string]string) + } + vip.Labels[util.SubnetNameLabel] = subnet + vip.Labels[util.IPReservedLabel] = "" + if vip.Status.Mac == "" && mac != "" || vip.Status.V4ip == "" && v4ip != "" || vip.Status.V6ip == "" && v6ip != "" { @@ -400,45 +421,14 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string vip.Status.Mac = mac vip.Status.Type = vip.Spec.Type // TODO:// Ready = true as subnet.Status.Ready + // Update with labels, spec and status in one call if _, err := c.config.KubeOvnClient.KubeovnV1().Vips().Update(context.Background(), vip, metav1.UpdateOptions{}); err != nil { err := fmt.Errorf("failed to update vip '%s', %w", key, err) klog.Error(err) return err } - // VIP status updated with IP allocation, trigger subnet status update - needUpdateSubnetStatus = true - } - var needUpdateLabel bool - var op string - if len(vip.Labels) == 0 { - op = "add" - vip.Labels = map[string]string{ - util.SubnetNameLabel: subnet, - util.IPReservedLabel: "", - } - needUpdateLabel = true - } - if _, ok := vip.Labels[util.SubnetNameLabel]; !ok { - op = "add" - vip.Labels[util.SubnetNameLabel] = subnet - vip.Labels[util.IPReservedLabel] = "" - needUpdateLabel = true - } - if needUpdateLabel { - patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/labels", "value": %s }]` - raw, _ := json.Marshal(vip.Labels) - patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - if _, err := c.config.KubeOvnClient.KubeovnV1().Vips().Patch(context.Background(), vip.Name, types.JSONPatchType, - []byte(patchPayload), metav1.PatchOptions{}); err != nil { - klog.Errorf("failed to patch label for vip '%s', %v", vip.Name, err) - return err - } } } - // Only trigger subnet status update when VIP is actually created or status is updated - if needUpdateSubnetStatus { - c.updateSubnetStatusQueue.Add(subnet) - } return nil } @@ -474,7 +464,6 @@ func (c *Controller) podReuseVip(vipName, portName string, keepVIP bool) error { return err } c.ipam.ReleaseAddressByPod(vipName, vip.Spec.Subnet) - c.updateSubnetStatusQueue.Add(vip.Spec.Subnet) return nil } @@ -514,12 +503,11 @@ func (c *Controller) releaseVip(key string) error { if _, _, _, err = c.ipam.GetStaticAddress(key, vip.Name, vip.Status.V4ip, mac, vip.Spec.Subnet, false); err != nil { klog.Errorf("failed to recover IPAM from vip CR %s: %v", vip.Name, err) } - c.updateSubnetStatusQueue.Add(vip.Spec.Subnet) } return nil } -func (c *Controller) handleAddVipFinalizer(key string) error { +func (c *Controller) handleAddOrUpdateVipFinalizer(key string) error { cachedVip, err := c.virtualIpsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { @@ -546,6 +534,10 @@ func (c *Controller) handleAddVipFinalizer(key string) error { klog.Errorf("failed to add finalizer for vip '%s', %v", cachedVip.Name, err) return err } + + // Trigger subnet status update after finalizer is added + // This ensures subnet status reflects the new VIP allocation + c.updateSubnetStatusQueue.Add(cachedVip.Spec.Subnet) return nil } @@ -576,6 +568,10 @@ func (c *Controller) handleDelVipFinalizer(key string) error { klog.Errorf("failed to remove finalizer from vip '%s', %v", cachedVip.Name, err) return err } + + // Trigger subnet status update after finalizer is removed + // This ensures subnet status reflects the IP release + c.updateSubnetStatusQueue.Add(cachedVip.Spec.Subnet) return nil } diff --git a/pkg/controller/vpc_nat_gw_eip.go b/pkg/controller/vpc_nat_gw_eip.go index fb72e4a4a7c..dcf6aebcc34 100644 --- a/pkg/controller/vpc_nat_gw_eip.go +++ b/pkg/controller/vpc_nat_gw_eip.go @@ -39,12 +39,6 @@ func (c *Controller) enqueueUpdateIptablesEip(oldObj, newObj any) { c.updateIptablesEipQueue.Add(key) } - // Trigger subnet status update when EIP CR is updated - // This ensures both CR labels and IPAM state are synced before status calculation - if oldEip.Status.IP != newEip.Status.IP || oldEip.Status.Ready != newEip.Status.Ready { - externalNetwork := util.GetExternalNetwork(newEip.Spec.ExternalSubnet) - c.updateSubnetStatusQueue.Add(externalNetwork) - } } func (c *Controller) enqueueDelIptablesEip(obj any) { @@ -66,12 +60,7 @@ func (c *Controller) enqueueDelIptablesEip(obj any) { key := cache.MetaObjectToName(eip).String() klog.Infof("enqueue del iptables eip %s", key) - c.delIptablesEipQueue.Add(key) - - // Trigger subnet status update when EIP is deleted - // This ensures subnet status reflects the IP release - externalNetwork := util.GetExternalNetwork(eip.Spec.ExternalSubnet) - c.updateSubnetStatusQueue.Add(externalNetwork) + c.delIptablesEipQueue.Add(eip) } func (c *Controller) handleAddIptablesEip(key string) error { @@ -166,6 +155,11 @@ func (c *Controller) handleAddIptablesEip(key string) error { return err } + if err = c.handleAddOrUpdateIptablesEipFinalizer(key); err != nil { + klog.Errorf("failed to handle add or update finalizer for eip %s, %v", key, err) + return err + } + return nil } @@ -225,6 +219,21 @@ func (c *Controller) handleUpdateIptablesEip(key string) error { if !cachedEip.DeletionTimestamp.IsZero() { klog.Infof("clean eip %q in pod", key) + + // Check if EIP is still being used by any NAT rules (FIP/DNAT/SNAT) + // Only remove finalizer when no NAT rules are using it + // Note: We query NAT rules directly instead of relying on cachedEip.Status.Nat + // to avoid cache staleness issues + nat, err := c.getIptablesEipNat(cachedEip.Spec.V4ip) + if err != nil { + klog.Errorf("failed to get eip %s nat rules, %v", key, err) + return err + } + if nat != "" { + klog.Infof("eip %s is still being used by NAT rules: %s, waiting for them to be deleted", key, nat) + return nil + } + if vpcNatEnabled == "true" { v4ipCidr, err := util.GetIPAddrWithMask(cachedEip.Status.IP, v4Cidr) if err != nil { @@ -243,11 +252,14 @@ func (c *Controller) handleUpdateIptablesEip(key string) error { return err } } + // Release IP from IPAM before removing finalizer + c.ipam.ReleaseAddressByPod(key, cachedEip.Spec.ExternalSubnet) + + // Now remove finalizer, which will trigger subnet status update if err = c.handleDelIptablesEipFinalizer(key); err != nil { klog.Errorf("failed to handle del finalizer for eip %s, %v", key, err) return err } - c.ipam.ReleaseAddressByPod(key, cachedEip.Spec.ExternalSubnet) return nil } @@ -337,15 +349,23 @@ func (c *Controller) handleUpdateIptablesEip(key string) error { return err } } - if err = c.handleAddIptablesEipFinalizer(key); err != nil { + if err = c.handleAddOrUpdateIptablesEipFinalizer(key); err != nil { klog.Errorf("failed to handle add finalizer for eip, %v", err) return err } return nil } -func (c *Controller) handleDelIptablesEip(key string) error { - klog.Infof("handle delete iptables eip %s", key) +func (c *Controller) handleDelIptablesEip(eip *kubeovnv1.IptablesEIP) error { + klog.Infof("handle delete iptables eip %s", eip.Name) + + // For IptablesEIPs deleted without finalizer (race condition or direct deletion), + // we need to ensure subnet status is updated as a safety net. + externalNetwork := util.GetExternalNetwork(eip.Spec.ExternalSubnet) + if externalNetwork != "" { + c.updateSubnetStatusQueue.Add(externalNetwork) + } + return nil } @@ -607,12 +627,25 @@ func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, ext } } else { eip := cachedEip.DeepCopy() + + // Ensure labels are set correctly before any update + if eip.Labels == nil { + eip.Labels = make(map[string]string) + } + eip.Labels[util.SubnetNameLabel] = externalNetwork + eip.Labels[util.VpcNatGatewayNameLabel] = natGwDp + eip.Labels[util.EipV4IpLabel] = v4ip + if eip.Spec.QoSPolicy != "" { + eip.Labels[util.QoSLabel] = eip.Spec.QoSPolicy + } + if v4ip != "" { klog.V(3).Infof("update eip cr %s", key) eip.Spec.V4ip = v4ip eip.Spec.V6ip = v6ip eip.Spec.NatGwDp = natGwDp eip.Spec.MacAddress = mac + // Update with labels and spec in one call if _, err := c.config.KubeOvnClient.KubeovnV1().IptablesEIPs().Update(context.Background(), eip, metav1.UpdateOptions{}); err != nil { errMsg := fmt.Errorf("failed to update eip crd %s, %w", key, err) klog.Error(errMsg) @@ -639,35 +672,9 @@ func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, ext return err } } - var needUpdateLabel bool - var op string - if len(eip.Labels) == 0 { - op = "add" - eip.Labels = map[string]string{ - util.SubnetNameLabel: externalNetwork, - util.VpcNatGatewayNameLabel: natGwDp, - util.EipV4IpLabel: v4ip, - } - needUpdateLabel = true - } else if eip.Labels[util.SubnetNameLabel] != externalNetwork { - op = "replace" - eip.Labels[util.SubnetNameLabel] = externalNetwork - eip.Labels[util.VpcNatGatewayNameLabel] = natGwDp - needUpdateLabel = true - } - if eip.Spec.QoSPolicy != "" && eip.Labels[util.QoSLabel] != eip.Spec.QoSPolicy { - eip.Labels[util.QoSLabel] = eip.Spec.QoSPolicy - needUpdateLabel = true - } - if needUpdateLabel { - if err := c.updateIptableLabels(eip.Name, op, "eip", eip.Labels); err != nil { - klog.Errorf("failed to update eip %s labels, %v", eip.Name, err) - return err - } - } - if err = c.handleAddIptablesEipFinalizer(key); err != nil { - klog.Errorf("failed to handle add finalizer for eip, %v", err) + if err = c.handleAddOrUpdateIptablesEipFinalizer(key); err != nil { + klog.Errorf("failed to handle add or update finalizer for eip, %v", err) return err } } @@ -685,7 +692,7 @@ func (c *Controller) syncIptablesEipFinalizer(cl client.Client) error { }) } -func (c *Controller) handleAddIptablesEipFinalizer(key string) error { +func (c *Controller) handleAddOrUpdateIptablesEipFinalizer(key string) error { cachedIptablesEip, err := c.iptablesEipsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { @@ -712,6 +719,11 @@ func (c *Controller) handleAddIptablesEipFinalizer(key string) error { klog.Errorf("failed to add finalizer for iptables eip '%s', %v", cachedIptablesEip.Name, err) return err } + + // Trigger subnet status update after finalizer is added + // This ensures subnet status reflects the new Iptables EIP allocation + externalNetwork := util.GetExternalNetwork(cachedIptablesEip.Spec.ExternalSubnet) + c.updateSubnetStatusQueue.Add(externalNetwork) return nil } @@ -742,6 +754,11 @@ func (c *Controller) handleDelIptablesEipFinalizer(key string) error { klog.Errorf("failed to remove finalizer from iptables eip '%s', %v", cachedIptablesEip.Name, err) return err } + + // Trigger subnet status update after finalizer is removed + // This ensures subnet status reflects the IP release + externalNetwork := util.GetExternalNetwork(cachedIptablesEip.Spec.ExternalSubnet) + c.updateSubnetStatusQueue.Add(externalNetwork) return nil } diff --git a/test/e2e/ip/e2e_test.go b/test/e2e/ip/e2e_test.go new file mode 100644 index 00000000000..b483bf84643 --- /dev/null +++ b/test/e2e/ip/e2e_test.go @@ -0,0 +1,445 @@ +package ip + +import ( + "context" + "flag" + "fmt" + "strings" + "testing" + "time" + + "github.com/onsi/ginkgo/v2" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "k8s.io/kubernetes/test/e2e" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/test/e2e/framework" +) + +var _ = framework.Describe("[group:ip]", func() { + f := framework.NewDefaultFramework("ip") + + var vpcClient *framework.VpcClient + var subnetClient *framework.SubnetClient + var podClient *framework.PodClient + var ipClient *framework.IPClient + var namespaceName, vpcName, subnetName, cidr string + + ginkgo.BeforeEach(func() { + vpcClient = f.VpcClient() + subnetClient = f.SubnetClient() + podClient = f.PodClient() + ipClient = f.IPClient() + namespaceName = f.Namespace.Name + cidr = framework.RandomCIDR(f.ClusterIPFamily) + + randomSuffix := framework.RandomSuffix() + vpcName = "vpc-" + randomSuffix + subnetName = "subnet-" + randomSuffix + + ginkgo.By("Creating vpc " + vpcName) + vpc := framework.MakeVpc(vpcName, "", false, false, []string{namespaceName}) + vpc = vpcClient.CreateSync(vpc) + + ginkgo.By("Creating subnet " + subnetName) + subnet := framework.MakeSubnet(subnetName, "", cidr, "", vpcName, "", nil, nil, []string{namespaceName}) + subnet = subnetClient.CreateSync(subnet) + }) + + ginkgo.AfterEach(func() { + ginkgo.By("Deleting subnet " + subnetName) + subnetClient.DeleteSync(subnetName) + ginkgo.By("Deleting vpc " + vpcName) + vpcClient.DeleteSync(vpcName) + }) + + framework.ConformanceIt("Test IP CR creation and deletion with finalizer and subnet status update", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + ginkgo.By("1. Get initial subnet status") + initialSubnet := subnetClient.Get(subnetName) + initialV4AvailableIPs := initialSubnet.Status.V4AvailableIPs + initialV4UsingIPs := initialSubnet.Status.V4UsingIPs + initialV6AvailableIPs := initialSubnet.Status.V6AvailableIPs + initialV6UsingIPs := initialSubnet.Status.V6UsingIPs + initialV4AvailableIPRange := initialSubnet.Status.V4AvailableIPRange + initialV4UsingIPRange := initialSubnet.Status.V4UsingIPRange + initialV6AvailableIPRange := initialSubnet.Status.V6AvailableIPRange + initialV6UsingIPRange := initialSubnet.Status.V6UsingIPRange + + ginkgo.By("2. Create a pod to trigger IP CR creation") + podName := "test-ip-pod-" + framework.RandomSuffix() + cmd := []string{"sleep", "infinity"} + pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) + pod = podClient.CreateSync(pod) + + ginkgo.By("3. Wait for IP CR to be created and get IP CR name") + var ipCR *apiv1.IP + var ipName string + for i := 0; i < 30; i++ { + // IP CR name format: podName.namespaceName + ipName = fmt.Sprintf("%s.%s", podName, namespaceName) + ipCR = ipClient.Get(ipName) + if ipCR != nil && ipCR.Name != "" { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectNotNil(ipCR, "IP CR should be created for pod %s", podName) + framework.ExpectEqual(ipCR.Spec.Subnet, subnetName, "IP CR should be in the correct subnet") + + ginkgo.By("4. Wait for IP CR finalizer to be added") + for i := 0; i < 60; i++ { + ipCR = ipClient.Get(ipName) + if ipCR != nil && len(ipCR.Finalizers) > 0 { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectNotNil(ipCR, "IP CR should exist") + framework.ExpectContainElement(ipCR.Finalizers, util.KubeOVNControllerFinalizer, + "IP CR should have finalizer after creation") + + ginkgo.By("5. Wait for subnet status to be updated after IP creation") + time.Sleep(5 * time.Second) + + ginkgo.By("6. Verify subnet status after IP CR creation") + afterCreateSubnet := subnetClient.Get(subnetName) + if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + // Verify IP count changed + framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1 after IP creation") + framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1 after IP creation") + + // Verify IP range changed + framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IP creation") + framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IP creation") + + // Verify the IP's address is in the using range + podIP := ipCR.Spec.V4IPAddress + framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V4UsingIPRange, podIP), + "Pod IP %s should be in V4UsingIPRange %s", podIP, afterCreateSubnet.Status.V4UsingIPRange) + } else if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + // Verify IP count changed + framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1 after IP creation") + framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1 after IP creation") + + // Verify IP range changed + framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IP creation") + framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IP creation") + + // Verify the IP's address is in the using range + podIP := ipCR.Spec.V6IPAddress + framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V6UsingIPRange, podIP), + "Pod IP %s should be in V6UsingIPRange %s", podIP, afterCreateSubnet.Status.V6UsingIPRange) + } else { + // Dual stack + framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1 after IP creation") + framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1 after IP creation") + framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1 after IP creation") + framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1 after IP creation") + + framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IP creation") + framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IP creation") + framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IP creation") + framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IP creation") + } + + // Store the status after creation for later comparison + afterCreateV4AvailableIPs := afterCreateSubnet.Status.V4AvailableIPs + afterCreateV4UsingIPs := afterCreateSubnet.Status.V4UsingIPs + afterCreateV6AvailableIPs := afterCreateSubnet.Status.V6AvailableIPs + afterCreateV6UsingIPs := afterCreateSubnet.Status.V6UsingIPs + afterCreateV4AvailableIPRange := afterCreateSubnet.Status.V4AvailableIPRange + afterCreateV4UsingIPRange := afterCreateSubnet.Status.V4UsingIPRange + afterCreateV6AvailableIPRange := afterCreateSubnet.Status.V6AvailableIPRange + afterCreateV6UsingIPRange := afterCreateSubnet.Status.V6UsingIPRange + + ginkgo.By("7. Delete the pod to trigger IP CR deletion") + podClient.DeleteSync(podName) + + ginkgo.By("8. Wait for IP CR to be deleted") + deleted := false + for i := 0; i < 30; i++ { + _, err := f.KubeOVNClientSet.KubeovnV1().IPs().Get(context.Background(), ipName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + deleted = true + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectTrue(deleted, "IP CR should be deleted") + + ginkgo.By("9. Wait for subnet status to be updated after IP deletion") + time.Sleep(5 * time.Second) + + ginkgo.By("10. Verify subnet status after IP CR deletion") + afterDeleteSubnet := subnetClient.Get(subnetName) + if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + // Verify IP count is restored + framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1 after IP deletion") + framework.ExpectEqual(afterCreateV4UsingIPs-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1 after IP deletion") + + // Verify IP range changed + framework.ExpectNotEqual(afterCreateV4AvailableIPRange, afterDeleteSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IP deletion") + framework.ExpectNotEqual(afterCreateV4UsingIPRange, afterDeleteSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IP deletion") + + // Verify counts match initial state + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after IP deletion") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after IP deletion") + } else if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + // Verify IP count is restored + framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1 after IP deletion") + framework.ExpectEqual(afterCreateV6UsingIPs-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1 after IP deletion") + + // Verify IP range changed + framework.ExpectNotEqual(afterCreateV6AvailableIPRange, afterDeleteSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IP deletion") + framework.ExpectNotEqual(afterCreateV6UsingIPRange, afterDeleteSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IP deletion") + + // Verify counts match initial state + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after IP deletion") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after IP deletion") + } else { + // Dual stack + framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1 after IP deletion") + framework.ExpectEqual(afterCreateV4UsingIPs-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1 after IP deletion") + framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1 after IP deletion") + framework.ExpectEqual(afterCreateV6UsingIPs-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1 after IP deletion") + + framework.ExpectNotEqual(afterCreateV4AvailableIPRange, afterDeleteSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IP deletion") + framework.ExpectNotEqual(afterCreateV4UsingIPRange, afterDeleteSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IP deletion") + framework.ExpectNotEqual(afterCreateV6AvailableIPRange, afterDeleteSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IP deletion") + framework.ExpectNotEqual(afterCreateV6UsingIPRange, afterDeleteSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IP deletion") + + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after IP deletion") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after IP deletion") + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after IP deletion") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after IP deletion") + } + + ginkgo.By("11. Test completed: IP CR creation and deletion properly updates subnet status via finalizer handlers") + }) + + framework.ConformanceIt("Test multiple IPs with pod lifecycle", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + ginkgo.By("1. Get initial subnet status") + initialSubnet := subnetClient.Get(subnetName) + initialV4AvailableIPs := initialSubnet.Status.V4AvailableIPs + initialV4UsingIPs := initialSubnet.Status.V4UsingIPs + initialV6AvailableIPs := initialSubnet.Status.V6AvailableIPs + initialV6UsingIPs := initialSubnet.Status.V6UsingIPs + + ginkgo.By("2. Create multiple pods to trigger multiple IP CR creations") + numPods := 3 + podNames := make([]string, numPods) + ipNames := make([]string, numPods) + cmd := []string{"sleep", "infinity"} + + for i := 0; i < numPods; i++ { + podName := fmt.Sprintf("test-multi-ip-pod-%d-%s", i, framework.RandomSuffix()) + podNames[i] = podName + ipNames[i] = fmt.Sprintf("%s.%s", podName, namespaceName) + + pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) + pod = podClient.CreateSync(pod) + } + + ginkgo.By("3. Wait for all IP CRs to be created and get finalizers") + for i := 0; i < numPods; i++ { + var ipCR *apiv1.IP + for j := 0; j < 60; j++ { + ipCR = ipClient.Get(ipNames[i]) + if ipCR != nil && ipCR.Name != "" && len(ipCR.Finalizers) > 0 { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectNotNil(ipCR, "IP CR should be created for pod %s", podNames[i]) + framework.ExpectContainElement(ipCR.Finalizers, util.KubeOVNControllerFinalizer, + "IP CR should have finalizer for pod %s", podNames[i]) + } + + ginkgo.By("4. Wait for subnet status to be updated after all IPs created") + time.Sleep(5 * time.Second) + + ginkgo.By("5. Verify subnet status after multiple IP CR creations") + afterCreateSubnet := subnetClient.Get(subnetName) + if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + framework.ExpectEqual(initialV4AvailableIPs-float64(numPods), afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by %d after creating %d IPs", numPods, numPods) + framework.ExpectEqual(initialV4UsingIPs+float64(numPods), afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by %d after creating %d IPs", numPods, numPods) + } else if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + framework.ExpectEqual(initialV6AvailableIPs-float64(numPods), afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by %d after creating %d IPs", numPods, numPods) + framework.ExpectEqual(initialV6UsingIPs+float64(numPods), afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by %d after creating %d IPs", numPods, numPods) + } else { + // Dual stack + framework.ExpectEqual(initialV4AvailableIPs-float64(numPods), afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by %d after creating %d IPs", numPods, numPods) + framework.ExpectEqual(initialV4UsingIPs+float64(numPods), afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by %d after creating %d IPs", numPods, numPods) + framework.ExpectEqual(initialV6AvailableIPs-float64(numPods), afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by %d after creating %d IPs", numPods, numPods) + framework.ExpectEqual(initialV6UsingIPs+float64(numPods), afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by %d after creating %d IPs", numPods, numPods) + } + + ginkgo.By("6. Delete all pods to trigger IP CR deletions") + for i := 0; i < numPods; i++ { + podClient.DeleteSync(podNames[i]) + } + + ginkgo.By("7. Wait for all IP CRs to be deleted") + for i := 0; i < numPods; i++ { + deleted := false + for j := 0; j < 30; j++ { + _, err := f.KubeOVNClientSet.KubeovnV1().IPs().Get(context.Background(), ipNames[i], metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + deleted = true + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectTrue(deleted, "IP CR %s should be deleted", ipNames[i]) + } + + ginkgo.By("8. Wait for subnet status to be updated after all IPs deleted") + time.Sleep(5 * time.Second) + + ginkgo.By("9. Verify subnet status after multiple IP CR deletions") + afterDeleteSubnet := subnetClient.Get(subnetName) + if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after all IPs deleted") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after all IPs deleted") + } else if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after all IPs deleted") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after all IPs deleted") + } else { + // Dual stack + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after all IPs deleted") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after all IPs deleted") + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after all IPs deleted") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after all IPs deleted") + } + + ginkgo.By("10. Test completed: Multiple IP CRs lifecycle properly updates subnet status") + }) + + framework.ConformanceIt("Test IP finalizer prevents premature deletion", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + ginkgo.By("1. Create a pod to trigger IP CR creation") + podName := "test-ip-finalizer-pod-" + framework.RandomSuffix() + cmd := []string{"sleep", "infinity"} + pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) + pod = podClient.CreateSync(pod) + + ginkgo.By("2. Wait for IP CR to be created with finalizer") + ipName := fmt.Sprintf("%s.%s", podName, namespaceName) + var ipCR *apiv1.IP + for i := 0; i < 60; i++ { + ipCR = ipClient.Get(ipName) + if ipCR != nil && ipCR.Name != "" && len(ipCR.Finalizers) > 0 { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectNotNil(ipCR, "IP CR should be created for pod %s", podName) + framework.ExpectContainElement(ipCR.Finalizers, util.KubeOVNControllerFinalizer, + "IP CR should have finalizer") + + ginkgo.By("3. Verify IP CR details") + framework.ExpectEqual(ipCR.Spec.PodName, podName, + "IP CR should have correct pod name") + framework.ExpectEqual(ipCR.Spec.Namespace, namespaceName, + "IP CR should have correct namespace") + framework.ExpectEqual(ipCR.Spec.Subnet, subnetName, + "IP CR should reference correct subnet") + framework.ExpectNotEmpty(ipCR.Spec.V4IPAddress, "IP CR should have V4 or V6 address assigned") + + ginkgo.By("4. Delete the pod") + podClient.DeleteSync(podName) + + ginkgo.By("5. Verify IP CR is eventually deleted") + deleted := false + for i := 0; i < 30; i++ { + _, err := f.KubeOVNClientSet.KubeovnV1().IPs().Get(context.Background(), ipName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + deleted = true + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectTrue(deleted, "IP CR should be deleted after pod deletion") + + ginkgo.By("6. Test completed: IP CR finalizer works correctly") + }) +}) + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + e2e.RunE2ETests(t) +} diff --git a/test/e2e/iptables-vpc-nat-gw/e2e_test.go b/test/e2e/iptables-vpc-nat-gw/e2e_test.go index 33c2f336613..574b3770b65 100644 --- a/test/e2e/iptables-vpc-nat-gw/e2e_test.go +++ b/test/e2e/iptables-vpc-nat-gw/e2e_test.go @@ -12,6 +12,7 @@ import ( dockernetwork "github.com/moby/moby/api/types/network" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -616,6 +617,382 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { ginkgo.By("Deleting custom vpc " + net2VpcName) vpcClient.DeleteSync(net2VpcName) }) + + // Helper function to wait for EIP to be ready + _ = func(eipClient *framework.IptablesEIPClient, eipName string, timeout time.Duration) *apiv1.IptablesEIP { + ginkgo.GinkgoHelper() + var eip *apiv1.IptablesEIP + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + eip = eipClient.Get(eipName) + if eip != nil && eip.Status.IP != "" && eip.Status.Ready { + framework.Logf("IptablesEIP %s is ready with IP: %s", eipName, eip.Status.IP) + return eip + } + time.Sleep(2 * time.Second) + } + framework.Failf("Timeout waiting for IptablesEIP %s to be ready", eipName) + return nil + } + + // Helper function to verify subnet status after EIP operation + _ = func(subnetClient *framework.SubnetClient, subnetName string, + protocol string, expectedAvailableDelta, expectedUsingDelta float64, + operation string, shouldContainIP string) { + ginkgo.GinkgoHelper() + + subnet := subnetClient.Get(subnetName) + framework.Logf("Verifying subnet %s status after %s: Protocol=%s", subnetName, operation, protocol) + + switch protocol { + case apiv1.ProtocolIPv4: + framework.Logf("V4 Status: Available=%.0f, Using=%.0f", + subnet.Status.V4AvailableIPs, subnet.Status.V4UsingIPs) + if shouldContainIP != "" { + framework.ExpectTrue(strings.Contains(subnet.Status.V4UsingIPRange, shouldContainIP), + "IP %s should be in V4UsingIPRange after %s", shouldContainIP, operation) + } + case apiv1.ProtocolIPv6: + framework.Logf("V6 Status: Available=%.0f, Using=%.0f", + subnet.Status.V6AvailableIPs, subnet.Status.V6UsingIPs) + if shouldContainIP != "" { + framework.ExpectTrue(strings.Contains(subnet.Status.V6UsingIPRange, shouldContainIP), + "IP %s should be in V6UsingIPRange after %s", shouldContainIP, operation) + } + case apiv1.ProtocolDual: + framework.Logf("Dual Stack Status: V4Available=%.0f, V4Using=%.0f, V6Available=%.0f, V6Using=%.0f", + subnet.Status.V4AvailableIPs, subnet.Status.V4UsingIPs, + subnet.Status.V6AvailableIPs, subnet.Status.V6UsingIPs) + } + } + + framework.ConformanceIt("should properly manage IptablesEIP lifecycle with finalizer and update subnet status", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + overlaySubnetV4Cidr := "10.0.3.0/24" + overlaySubnetV4Gw := "10.0.3.1" + lanIP := "10.0.3.254" + natgwQoS := "" + setupVpcNatGwTestEnvironment( + f, dockerExtNet1Network, attachNetClient, + subnetClient, vpcClient, vpcNatGwClient, + vpcName, overlaySubnetName, vpcNatGwName, natgwQoS, + overlaySubnetV4Cidr, overlaySubnetV4Gw, lanIP, + dockerExtNet1Name, networkAttachDefName, net1NicName, + externalSubnetProvider, + false, + ) + + ginkgo.By("1. Get initial external subnet status") + externalSubnetName := util.GetExternalNetwork(networkAttachDefName) + initialSubnet := subnetClient.Get(externalSubnetName) + initialV4AvailableIPs := initialSubnet.Status.V4AvailableIPs + initialV4UsingIPs := initialSubnet.Status.V4UsingIPs + initialV6AvailableIPs := initialSubnet.Status.V6AvailableIPs + initialV6UsingIPs := initialSubnet.Status.V6UsingIPs + initialV4AvailableIPRange := initialSubnet.Status.V4AvailableIPRange + initialV4UsingIPRange := initialSubnet.Status.V4UsingIPRange + initialV6AvailableIPRange := initialSubnet.Status.V6AvailableIPRange + initialV6UsingIPRange := initialSubnet.Status.V6UsingIPRange + + ginkgo.By("2. Create IptablesEIP to trigger IP allocation") + eipName := "test-eip-finalizer-" + framework.RandomSuffix() + eip := framework.MakeIptablesEIP(eipName, "", "", "", vpcNatGwName, "", "") + _ = iptablesEIPClient.CreateSync(eip) + + ginkgo.By("3. Wait for IptablesEIP CR to be created and get IP") + var eipCR *apiv1.IptablesEIP + for i := 0; i < 60; i++ { + eipCR = iptablesEIPClient.Get(eipName) + if eipCR != nil && eipCR.Status.IP != "" && eipCR.Status.Ready { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectNotNil(eipCR, "IptablesEIP CR should be created") + framework.ExpectNotEmpty(eipCR.Status.IP, "IptablesEIP should have IP assigned") + framework.ExpectTrue(eipCR.Status.Ready, "IptablesEIP should be ready") + + ginkgo.By("4. Wait for IptablesEIP CR finalizer to be added") + for i := 0; i < 60; i++ { + eipCR = iptablesEIPClient.Get(eipName) + if eipCR != nil && len(eipCR.Finalizers) > 0 { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectNotNil(eipCR, "IptablesEIP CR should exist") + framework.ExpectContainElement(eipCR.Finalizers, util.KubeOVNControllerFinalizer, + "IptablesEIP CR should have finalizer after creation") + + ginkgo.By("5. Wait for external subnet status to be updated after IptablesEIP creation") + time.Sleep(5 * time.Second) + + ginkgo.By("6. Verify external subnet status after IptablesEIP CR creation") + afterCreateSubnet := subnetClient.Get(externalSubnetName) + if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + // Verify IP count changed + framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1 after IptablesEIP creation") + framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1 after IptablesEIP creation") + + // Verify IP range changed + framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IptablesEIP creation") + framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IptablesEIP creation") + + // Verify the EIP's address is in the using range + eipIP := eipCR.Status.IP + framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V4UsingIPRange, eipIP), + "EIP IP %s should be in V4UsingIPRange %s", eipIP, afterCreateSubnet.Status.V4UsingIPRange) + } else if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + // Verify IP count changed + framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1 after IptablesEIP creation") + framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1 after IptablesEIP creation") + + // Verify IP range changed + framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IptablesEIP creation") + framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IptablesEIP creation") + + // Verify the EIP's address is in the using range + eipIP := eipCR.Status.IP + framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V6UsingIPRange, eipIP), + "EIP IP %s should be in V6UsingIPRange %s", eipIP, afterCreateSubnet.Status.V6UsingIPRange) + } else { + // Dual stack + framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1 after IptablesEIP creation") + framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1 after IptablesEIP creation") + framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1 after IptablesEIP creation") + framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1 after IptablesEIP creation") + + framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IptablesEIP creation") + framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IptablesEIP creation") + framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IptablesEIP creation") + framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IptablesEIP creation") + } + + // Store the status after creation for later comparison + afterCreateV4AvailableIPs := afterCreateSubnet.Status.V4AvailableIPs + afterCreateV4UsingIPs := afterCreateSubnet.Status.V4UsingIPs + afterCreateV6AvailableIPs := afterCreateSubnet.Status.V6AvailableIPs + afterCreateV6UsingIPs := afterCreateSubnet.Status.V6UsingIPs + afterCreateV4AvailableIPRange := afterCreateSubnet.Status.V4AvailableIPRange + afterCreateV4UsingIPRange := afterCreateSubnet.Status.V4UsingIPRange + afterCreateV6AvailableIPRange := afterCreateSubnet.Status.V6AvailableIPRange + afterCreateV6UsingIPRange := afterCreateSubnet.Status.V6UsingIPRange + + ginkgo.By("7. Delete the IptablesEIP to trigger IP release") + iptablesEIPClient.DeleteSync(eipName) + + ginkgo.By("8. Wait for IptablesEIP CR to be deleted") + deleted := false + for i := 0; i < 30; i++ { + _, err := f.KubeOVNClientSet.KubeovnV1().IptablesEIPs().Get(context.Background(), eipName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + deleted = true + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectTrue(deleted, "IptablesEIP CR should be deleted") + + ginkgo.By("9. Wait for external subnet status to be updated after IptablesEIP deletion") + time.Sleep(5 * time.Second) + + ginkgo.By("10. Verify external subnet status after IptablesEIP CR deletion") + afterDeleteSubnet := subnetClient.Get(externalSubnetName) + if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + // Verify IP count is restored + framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1 after IptablesEIP deletion") + framework.ExpectEqual(afterCreateV4UsingIPs-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1 after IptablesEIP deletion") + + // Verify IP range changed + framework.ExpectNotEqual(afterCreateV4AvailableIPRange, afterDeleteSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IptablesEIP deletion") + framework.ExpectNotEqual(afterCreateV4UsingIPRange, afterDeleteSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IptablesEIP deletion") + + // Verify counts match initial state + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after IptablesEIP deletion") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after IptablesEIP deletion") + } else if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + // Verify IP count is restored + framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1 after IptablesEIP deletion") + framework.ExpectEqual(afterCreateV6UsingIPs-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1 after IptablesEIP deletion") + + // Verify IP range changed + framework.ExpectNotEqual(afterCreateV6AvailableIPRange, afterDeleteSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IptablesEIP deletion") + framework.ExpectNotEqual(afterCreateV6UsingIPRange, afterDeleteSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IptablesEIP deletion") + + // Verify counts match initial state + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after IptablesEIP deletion") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after IptablesEIP deletion") + } else { + // Dual stack + framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1 after IptablesEIP deletion") + framework.ExpectEqual(afterCreateV4UsingIPs-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1 after IptablesEIP deletion") + framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1 after IptablesEIP deletion") + framework.ExpectEqual(afterCreateV6UsingIPs-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1 after IptablesEIP deletion") + + framework.ExpectNotEqual(afterCreateV4AvailableIPRange, afterDeleteSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IptablesEIP deletion") + framework.ExpectNotEqual(afterCreateV4UsingIPRange, afterDeleteSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IptablesEIP deletion") + framework.ExpectNotEqual(afterCreateV6AvailableIPRange, afterDeleteSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IptablesEIP deletion") + framework.ExpectNotEqual(afterCreateV6UsingIPRange, afterDeleteSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IptablesEIP deletion") + + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after IptablesEIP deletion") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after IptablesEIP deletion") + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after IptablesEIP deletion") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after IptablesEIP deletion") + } + + ginkgo.By("11. Test completed: IptablesEIP CR creation and deletion properly updates external subnet status via finalizer handlers") + }) + + framework.ConformanceIt("Test IptablesEIP finalizer cannot be removed when used by NAT rules", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + overlaySubnetV4Cidr := "10.0.4.0/24" + overlaySubnetV4Gw := "10.0.4.1" + lanIP := "10.0.4.254" + natgwQoS := "" + setupVpcNatGwTestEnvironment( + f, dockerExtNet1Network, attachNetClient, + subnetClient, vpcClient, vpcNatGwClient, + vpcName, overlaySubnetName, vpcNatGwName, natgwQoS, + overlaySubnetV4Cidr, overlaySubnetV4Gw, lanIP, + dockerExtNet1Name, networkAttachDefName, net1NicName, + externalSubnetProvider, + false, + ) + + ginkgo.By("1. Create a VIP for FIP") + vipName := "test-vip-" + framework.RandomSuffix() + vip := framework.MakeVip(f.Namespace.Name, vipName, overlaySubnetName, "", "", "") + _ = vipClient.CreateSync(vip) + vip = vipClient.Get(vipName) + + ginkgo.By("2. Create IptablesEIP") + eipName := "test-eip-with-fip-" + framework.RandomSuffix() + eip := framework.MakeIptablesEIP(eipName, "", "", "", vpcNatGwName, "", "") + _ = iptablesEIPClient.CreateSync(eip) + + ginkgo.By("3. Wait for IptablesEIP to be ready") + var eipCR *apiv1.IptablesEIP + for i := 0; i < 60; i++ { + eipCR = iptablesEIPClient.Get(eipName) + if eipCR != nil && eipCR.Status.IP != "" && eipCR.Status.Ready { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectNotNil(eipCR, "IptablesEIP CR should be created") + framework.ExpectTrue(eipCR.Status.Ready, "IptablesEIP should be ready") + + ginkgo.By("4. Create IptablesFIP using the EIP") + fipName := "test-fip-" + framework.RandomSuffix() + fip := framework.MakeIptablesFIPRule(fipName, eipName, vip.Status.V4ip) + _ = iptablesFIPClient.CreateSync(fip) + + ginkgo.By("5. Wait for EIP status to show it's being used by FIP") + for i := 0; i < 60; i++ { + eipCR = iptablesEIPClient.Get(eipName) + if eipCR != nil && strings.Contains(eipCR.Status.Nat, util.FipUsingEip) { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectTrue(strings.Contains(eipCR.Status.Nat, util.FipUsingEip), + "EIP status.Nat should contain 'fip' when used by FIP rule") + + ginkgo.By("6. Delete the IptablesEIP (should not remove finalizer while FIP exists)") + err := f.KubeOVNClientSet.KubeovnV1().IptablesEIPs().Delete(context.Background(), eipName, metav1.DeleteOptions{}) + framework.ExpectNoError(err, "Deleting IptablesEIP should succeed") + + ginkgo.By("7. Wait and verify EIP still exists with finalizer (blocked by FIP)") + time.Sleep(5 * time.Second) + eipCR = iptablesEIPClient.Get(eipName) + framework.ExpectNotNil(eipCR, "IptablesEIP should still exist") + framework.ExpectNotNil(eipCR.DeletionTimestamp, "IptablesEIP should have DeletionTimestamp") + framework.ExpectContainElement(eipCR.Finalizers, util.KubeOVNControllerFinalizer, + "IptablesEIP should still have finalizer because it's being used by FIP") + + ginkgo.By("8. Delete the FIP to unblock EIP deletion") + iptablesFIPClient.DeleteSync(fipName) + + ginkgo.By("9. Wait for FIP to be deleted") + fipDeleted := false + for i := 0; i < 30; i++ { + _, err := f.KubeOVNClientSet.KubeovnV1().IptablesFIPRules().Get(context.Background(), fipName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + fipDeleted = true + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectTrue(fipDeleted, "FIP should be deleted") + + ginkgo.By("10. Wait for EIP status.Nat to be cleared") + for i := 0; i < 30; i++ { + eipCR = iptablesEIPClient.Get(eipName) + if eipCR == nil || eipCR.Status.Nat == "" { + break + } + time.Sleep(1 * time.Second) + } + + ginkgo.By("11. Verify EIP is now deleted after FIP is removed") + eipDeleted := false + for i := 0; i < 30; i++ { + _, err := f.KubeOVNClientSet.KubeovnV1().IptablesEIPs().Get(context.Background(), eipName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + eipDeleted = true + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectTrue(eipDeleted, "IptablesEIP should be deleted after FIP is removed") + + ginkgo.By("12. Clean up VIP") + vipClient.DeleteSync(vipName) + + ginkgo.By("13. Test completed: IptablesEIP finalizer correctly blocks deletion when used by NAT rules") + }) }) func iperf(f *framework.Framework, iperfClientPod *corev1.Pod, iperfServerEIP *apiv1.IptablesEIP) string { diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index ccc4f39bd32..1ed4cfbd515 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -1027,6 +1027,268 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectHaveKeyWithValue(node.Labels, util.NodeExtGwLabel, "false") } }) + + // Helper function to wait for OvnEip to be ready + waitForOvnEipReady := func(eipClient *framework.OvnEipClient, eipName string, timeout time.Duration) *kubeovnv1.OvnEip { + ginkgo.GinkgoHelper() + var eip *kubeovnv1.OvnEip + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + eip = eipClient.Get(eipName) + if eip != nil && eip.Status.V4Ip != "" && eip.Status.Ready { + framework.Logf("OvnEip %s is ready with V4IP: %s", eipName, eip.Status.V4Ip) + return eip + } + time.Sleep(2 * time.Second) + } + framework.Failf("Timeout waiting for OvnEip %s to be ready", eipName) + return nil + } + + framework.ConformanceIt("should properly manage OvnEip lifecycle with finalizer and update subnet status", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + ginkgo.By("Setting up provider network and underlay subnet") + network, err := docker.NetworkInspect(dockerNetworkName) + framework.ExpectNoError(err, "getting docker network "+dockerNetworkName) + + exchangeLinkName := false + itFn(exchangeLinkName, providerNetworkName, linkMap, &providerBridgeIps) + + ginkgo.By("Creating underlay vlan " + vlanName) + vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) + _ = vlanClient.Create(vlan) + + ginkgo.By("Creating underlay subnet " + underlaySubnetName) + var cidrV4, cidrV6, gatewayV4, gatewayV6 string + for _, config := range dockerNetwork.IPAM.Config { + switch util.CheckProtocol(config.Subnet.String()) { + case kubeovnv1.ProtocolIPv4: + if f.HasIPv4() { + cidrV4 = config.Subnet.String() + gatewayV4 = config.Gateway.String() + } + case kubeovnv1.ProtocolIPv6: + if f.HasIPv6() { + cidrV6 = config.Subnet.String() + gatewayV6 = config.Gateway.String() + } + } + } + cidr := make([]string, 0, 2) + gateway := make([]string, 0, 2) + if f.HasIPv4() { + cidr = append(cidr, cidrV4) + gateway = append(gateway, gatewayV4) + } + if f.HasIPv6() { + cidr = append(cidr, cidrV6) + gateway = append(gateway, gatewayV6) + } + excludeIPs := make([]string, 0, len(network.Containers)*2) + for _, container := range network.Containers { + if container.IPv4Address.IsValid() && f.HasIPv4() { + excludeIPs = append(excludeIPs, container.IPv4Address.Addr().String()) + } + if container.IPv6Address.IsValid() && f.HasIPv6() { + excludeIPs = append(excludeIPs, container.IPv6Address.Addr().String()) + } + } + vlanSubnetCidr := strings.Join(cidr, ",") + vlanSubnetGw := strings.Join(gateway, ",") + underlaySubnet := framework.MakeSubnet(underlaySubnetName, vlanName, vlanSubnetCidr, vlanSubnetGw, "", "", excludeIPs, nil, nil) + _ = subnetClient.CreateSync(underlaySubnet) + + ginkgo.By("Step 1: Recording initial underlay subnet status") + initialSubnet := subnetClient.Get(underlaySubnetName) + framework.Logf("Initial subnet status - V4: Available=%.0f Using=%.0f, V6: Available=%.0f Using=%.0f", + initialSubnet.Status.V4AvailableIPs, initialSubnet.Status.V4UsingIPs, + initialSubnet.Status.V6AvailableIPs, initialSubnet.Status.V6UsingIPs) + + // Store initial status + initialV4Available := initialSubnet.Status.V4AvailableIPs + initialV4Using := initialSubnet.Status.V4UsingIPs + initialV6Available := initialSubnet.Status.V6AvailableIPs + initialV6Using := initialSubnet.Status.V6UsingIPs + + ginkgo.By("Step 2: Creating OvnEip and waiting for it to be ready") + eipName := "test-ovn-eip-lifecycle-" + framework.RandomSuffix() + eip := makeOvnEip(eipName, underlaySubnetName, "", "", "", util.OvnEipTypeNAT) + _ = ovnEipClient.CreateSync(eip) + + eipCR := waitForOvnEipReady(ovnEipClient, eipName, 2*time.Minute) + framework.ExpectNotEmpty(eipCR.Status.V4Ip, "OvnEip should have V4 IP assigned") + + ginkgo.By("Step 3: Verifying finalizer is added to OvnEip") + framework.WaitUntil(time.Minute, func(_ context.Context) (bool, error) { + eipCR = ovnEipClient.Get(eipName) + return eipCR != nil && len(eipCR.Finalizers) > 0, nil + }, "OvnEip should have finalizer added") + framework.ExpectContainElement(eipCR.Finalizers, util.KubeOVNControllerFinalizer, + "OvnEip must have controller finalizer") + + ginkgo.By("Step 4: Verifying subnet status updated after OvnEip creation") + time.Sleep(5 * time.Second) + afterCreateSubnet := subnetClient.Get(underlaySubnetName) + + // Verify based on protocol + protocol := afterCreateSubnet.Spec.Protocol + framework.Logf("Verifying subnet status for protocol: %s", protocol) + + if protocol == kubeovnv1.ProtocolIPv4 || protocol == kubeovnv1.ProtocolDual { + framework.ExpectEqual(initialV4Available-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1") + framework.ExpectEqual(initialV4Using+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1") + framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V4UsingIPRange, eipCR.Status.V4Ip), + "EIP V4 IP should be in using range") + } + if protocol == kubeovnv1.ProtocolIPv6 || protocol == kubeovnv1.ProtocolDual { + framework.ExpectEqual(initialV6Available-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1") + framework.ExpectEqual(initialV6Using+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1") + } + + // Store after-create status + afterCreateV4Available := afterCreateSubnet.Status.V4AvailableIPs + afterCreateV4Using := afterCreateSubnet.Status.V4UsingIPs + afterCreateV6Available := afterCreateSubnet.Status.V6AvailableIPs + afterCreateV6Using := afterCreateSubnet.Status.V6UsingIPs + + ginkgo.By("Step 5: Deleting OvnEip and verifying cleanup") + ovnEipClient.DeleteSync(eipName) + + framework.WaitUntil(time.Minute, func(_ context.Context) (bool, error) { + _, err := f.KubeOVNClientSet.KubeovnV1().OvnEips().Get(context.Background(), eipName, metav1.GetOptions{}) + return k8serrors.IsNotFound(err), nil + }, "OvnEip should be deleted") + + ginkgo.By("Step 6: Verifying subnet status restored after OvnEip deletion") + time.Sleep(5 * time.Second) + afterDeleteSubnet := subnetClient.Get(underlaySubnetName) + + if protocol == kubeovnv1.ProtocolIPv4 || protocol == kubeovnv1.ProtocolDual { + framework.ExpectEqual(afterCreateV4Available+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1") + framework.ExpectEqual(afterCreateV4Using-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1") + framework.ExpectEqual(initialV4Available, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value") + framework.ExpectEqual(initialV4Using, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value") + } + if protocol == kubeovnv1.ProtocolIPv6 || protocol == kubeovnv1.ProtocolDual { + framework.ExpectEqual(afterCreateV6Available+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1") + framework.ExpectEqual(afterCreateV6Using-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1") + framework.ExpectEqual(initialV6Available, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value") + framework.ExpectEqual(initialV6Using, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value") + } + + framework.Logf("OvnEip lifecycle test completed successfully") + }) + + framework.ConformanceIt("should block OvnEip deletion when used by NAT rules", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + ginkgo.By("Setting up test environment") + network, err := docker.NetworkInspect(dockerNetworkName) + framework.ExpectNoError(err) + + exchangeLinkName := false + itFn(exchangeLinkName, providerNetworkName, linkMap, &providerBridgeIps) + + ginkgo.By("Creating underlay vlan and subnet") + vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) + _ = vlanClient.Create(vlan) + + var cidrV4, gatewayV4 string + for _, config := range dockerNetwork.IPAM.Config { + if util.CheckProtocol(config.Subnet.String()) == kubeovnv1.ProtocolIPv4 && f.HasIPv4() { + cidrV4 = config.Subnet.String() + gatewayV4 = config.Gateway.String() + break + } + } + excludeIPs := make([]string, 0, len(network.Containers)) + for _, container := range network.Containers { + if container.IPv4Address.IsValid() && f.HasIPv4() { + excludeIPs = append(excludeIPs, container.IPv4Address.Addr().String()) + } + } + underlaySubnet := framework.MakeSubnet(underlaySubnetName, vlanName, cidrV4, gatewayV4, "", "", excludeIPs, nil, nil) + _ = subnetClient.CreateSync(underlaySubnet) + + ginkgo.By("Step 1: Creating custom VPC and subnet for testing") + testVpcName := "test-vpc-dep-" + framework.RandomSuffix() + testSubnetName := "test-subnet-dep-" + framework.RandomSuffix() + testVpc := framework.MakeVpc(testVpcName, "", false, false, nil) + _ = vpcClient.CreateSync(testVpc) + + testSubnet := framework.MakeSubnet(testSubnetName, "", "192.168.100.0/24", "192.168.100.1", testVpcName, util.OvnProvider, nil, nil, nil) + _ = subnetClient.CreateSync(testSubnet) + + ginkgo.By("Step 2: Creating VIP for FIP") + vipName := "test-vip-dep-" + framework.RandomSuffix() + vip := makeOvnVip(namespaceName, vipName, testSubnetName, "", "", "") + vip = vipClient.CreateSync(vip) + framework.ExpectNotEmpty(vip.Status.V4ip) + + ginkgo.By("Step 3: Creating OvnEip") + eipName := "test-eip-with-dep-" + framework.RandomSuffix() + eip := makeOvnEip(eipName, underlaySubnetName, "", "", "", util.OvnEipTypeNAT) + _ = ovnEipClient.CreateSync(eip) + + eipCR := waitForOvnEipReady(ovnEipClient, eipName, 2*time.Minute) + + ginkgo.By("Step 4: Creating OvnFip using the EIP") + fipName := "test-fip-dep-" + framework.RandomSuffix() + fip := makeOvnFip(fipName, eipName, "", "", testVpcName, vip.Status.V4ip) + _ = ovnFipClient.CreateSync(fip) + + ginkgo.By("Step 5: Verifying EIP Status.Nat shows FIP usage") + framework.WaitUntil(time.Minute, func(_ context.Context) (bool, error) { + eipCR = ovnEipClient.Get(eipName) + return eipCR != nil && strings.Contains(eipCR.Status.Nat, util.FipUsingEip), nil + }, "EIP Status.Nat should contain 'fip'") + + ginkgo.By("Step 6: Attempting to delete EIP (should be blocked by FIP)") + err = f.KubeOVNClientSet.KubeovnV1().OvnEips().Delete(context.Background(), eipName, metav1.DeleteOptions{}) + framework.ExpectNoError(err, "Delete operation should succeed") + + ginkgo.By("Step 7: Verifying EIP still exists with finalizer (blocked)") + time.Sleep(5 * time.Second) + eipCR = ovnEipClient.Get(eipName) + framework.ExpectNotNil(eipCR, "EIP should still exist") + framework.ExpectNotNil(eipCR.DeletionTimestamp, "EIP should have DeletionTimestamp") + framework.ExpectContainElement(eipCR.Finalizers, util.KubeOVNControllerFinalizer, + "EIP should still have finalizer because FIP is using it") + + ginkgo.By("Step 8: Deleting FIP to unblock EIP deletion") + ovnFipClient.DeleteSync(fipName) + + framework.WaitUntil(time.Minute, func(_ context.Context) (bool, error) { + _, err := f.KubeOVNClientSet.KubeovnV1().OvnFips().Get(context.Background(), fipName, metav1.GetOptions{}) + return k8serrors.IsNotFound(err), nil + }, "FIP should be deleted") + + ginkgo.By("Step 9: Verifying EIP is now deleted after FIP removal") + framework.WaitUntil(time.Minute, func(_ context.Context) (bool, error) { + _, err := f.KubeOVNClientSet.KubeovnV1().OvnEips().Get(context.Background(), eipName, metav1.GetOptions{}) + return k8serrors.IsNotFound(err), nil + }, "EIP should be deleted after FIP is removed") + + ginkgo.By("Step 10: Cleaning up test resources") + vipClient.DeleteSync(vipName) + subnetClient.DeleteSync(testSubnetName) + vpcClient.DeleteSync(testVpcName) + + framework.Logf("OvnEip dependency blocking test completed successfully") + }) }) func init() { diff --git a/test/e2e/vip/e2e_test.go b/test/e2e/vip/e2e_test.go index 580cd78e765..1879d3148fd 100644 --- a/test/e2e/vip/e2e_test.go +++ b/test/e2e/vip/e2e_test.go @@ -199,13 +199,196 @@ var _ = framework.Describe("[group:vip]", func() { securityGroupClient.DeleteSync(securityGroupName) }) + framework.ConformanceIt("Test vip subnet status update with finalizer", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + ginkgo.By("1. Get initial subnet status") + initialSubnet := subnetClient.Get(subnetName) + initialV4AvailableIPs := initialSubnet.Status.V4AvailableIPs + initialV4UsingIPs := initialSubnet.Status.V4UsingIPs + initialV6AvailableIPs := initialSubnet.Status.V6AvailableIPs + initialV6UsingIPs := initialSubnet.Status.V6UsingIPs + initialV4AvailableIPRange := initialSubnet.Status.V4AvailableIPRange + initialV4UsingIPRange := initialSubnet.Status.V4UsingIPRange + initialV6AvailableIPRange := initialSubnet.Status.V6AvailableIPRange + initialV6UsingIPRange := initialSubnet.Status.V6UsingIPRange + + ginkgo.By("2. Create a VIP and verify finalizer is added") + testVipName := "test-vip-finalizer-" + framework.RandomSuffix() + testVip := makeOvnVip(namespaceName, testVipName, subnetName, "", "", "") + testVip = vipClient.CreateSync(testVip) + + // Verify VIP has finalizer + framework.ExpectContainElement(testVip.Finalizers, util.KubeOVNControllerFinalizer) + + ginkgo.By("3. Wait for subnet status to be updated after VIP creation") + time.Sleep(5 * time.Second) + + ginkgo.By("4. Verify subnet status after VIP creation") + afterCreateSubnet := subnetClient.Get(subnetName) + if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + // Verify IP count changed + framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1 after VIP creation") + framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1 after VIP creation") + + // Verify IP range changed + framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after VIP creation") + framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after VIP creation") + + // Verify the VIP's IP is in the using range + vipIP := testVip.Status.V4ip + framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V4UsingIPRange, vipIP), + "VIP IP %s should be in V4UsingIPRange %s", vipIP, afterCreateSubnet.Status.V4UsingIPRange) + } else if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + // Verify IP count changed + framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1 after VIP creation") + framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1 after VIP creation") + + // Verify IP range changed + framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after VIP creation") + framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after VIP creation") + + // Verify the VIP's IP is in the using range + vipIP := testVip.Status.V6ip + framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V6UsingIPRange, vipIP), + "VIP IP %s should be in V6UsingIPRange %s", vipIP, afterCreateSubnet.Status.V6UsingIPRange) + } else { + // Dual stack + framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1 after VIP creation") + framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1 after VIP creation") + framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1 after VIP creation") + framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1 after VIP creation") + + framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after VIP creation") + framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after VIP creation") + framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after VIP creation") + framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after VIP creation") + } + + // Store the status after creation for later comparison + afterCreateV4AvailableIPs := afterCreateSubnet.Status.V4AvailableIPs + afterCreateV4UsingIPs := afterCreateSubnet.Status.V4UsingIPs + afterCreateV6AvailableIPs := afterCreateSubnet.Status.V6AvailableIPs + afterCreateV6UsingIPs := afterCreateSubnet.Status.V6UsingIPs + afterCreateV4AvailableIPRange := afterCreateSubnet.Status.V4AvailableIPRange + afterCreateV4UsingIPRange := afterCreateSubnet.Status.V4UsingIPRange + afterCreateV6AvailableIPRange := afterCreateSubnet.Status.V6AvailableIPRange + afterCreateV6UsingIPRange := afterCreateSubnet.Status.V6UsingIPRange + + ginkgo.By("5. Delete the VIP") + vipClient.DeleteSync(testVipName) + + ginkgo.By("6. Wait for subnet status to be updated after VIP deletion") + time.Sleep(5 * time.Second) + + ginkgo.By("7. Verify subnet status after VIP deletion") + afterDeleteSubnet := subnetClient.Get(subnetName) + if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + // Verify IP count is restored + framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1 after VIP deletion") + framework.ExpectEqual(afterCreateV4UsingIPs-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1 after VIP deletion") + + // Verify IP range changed + framework.ExpectNotEqual(afterCreateV4AvailableIPRange, afterDeleteSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after VIP deletion") + framework.ExpectNotEqual(afterCreateV4UsingIPRange, afterDeleteSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after VIP deletion") + + // Verify counts match initial state + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after VIP deletion") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after VIP deletion") + } else if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + // Verify IP count is restored + framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1 after VIP deletion") + framework.ExpectEqual(afterCreateV6UsingIPs-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1 after VIP deletion") + + // Verify IP range changed + framework.ExpectNotEqual(afterCreateV6AvailableIPRange, afterDeleteSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after VIP deletion") + framework.ExpectNotEqual(afterCreateV6UsingIPRange, afterDeleteSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after VIP deletion") + + // Verify counts match initial state + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after VIP deletion") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after VIP deletion") + } else { + // Dual stack + framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1 after VIP deletion") + framework.ExpectEqual(afterCreateV4UsingIPs-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1 after VIP deletion") + framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1 after VIP deletion") + framework.ExpectEqual(afterCreateV6UsingIPs-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1 after VIP deletion") + + framework.ExpectNotEqual(afterCreateV4AvailableIPRange, afterDeleteSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after VIP deletion") + framework.ExpectNotEqual(afterCreateV4UsingIPRange, afterDeleteSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after VIP deletion") + framework.ExpectNotEqual(afterCreateV6AvailableIPRange, afterDeleteSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after VIP deletion") + framework.ExpectNotEqual(afterCreateV6UsingIPRange, afterDeleteSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after VIP deletion") + + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after VIP deletion") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after VIP deletion") + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after VIP deletion") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after VIP deletion") + } + + ginkgo.By("8. Test completed: VIP creation and deletion properly updates subnet status via finalizer handlers") + }) + framework.ConformanceIt("Test vip", func() { f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") ginkgo.By("0. Test subnet status counting vip") oldSubnet := subnetClient.Get(subnetName) countingVip := makeOvnVip(namespaceName, countingVipName, subnetName, "", "", "") - _ = vipClient.CreateSync(countingVip) - time.Sleep(3 * time.Second) + countingVip = vipClient.CreateSync(countingVip) + + // Wait for finalizer to be added + ginkgo.By("Waiting for VIP finalizer to be added") + for i := 0; i < 10; i++ { + countingVip = vipClient.Get(countingVipName) + if len(countingVip.Finalizers) > 0 { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectContainElement(countingVip.Finalizers, util.KubeOVNControllerFinalizer) + + // Wait for subnet status to be updated + ginkgo.By("Waiting for subnet status to be updated after VIP creation") + time.Sleep(5 * time.Second) newSubnet := subnetClient.Get(subnetName) if newSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { framework.ExpectEqual(oldSubnet.Status.V4AvailableIPs-1, newSubnet.Status.V4AvailableIPs) @@ -220,8 +403,9 @@ var _ = framework.Describe("[group:vip]", func() { } oldSubnet = newSubnet // delete counting vip + ginkgo.By("Deleting counting VIP and waiting for subnet status update") vipClient.DeleteSync(countingVipName) - time.Sleep(3 * time.Second) + time.Sleep(5 * time.Second) newSubnet = subnetClient.Get(subnetName) if newSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { framework.ExpectEqual(oldSubnet.Status.V4AvailableIPs+1, newSubnet.Status.V4AvailableIPs) From 06dcf7d0c1997ddb345393e93389b586f048ae48 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Fri, 12 Dec 2025 20:22:09 +0800 Subject: [PATCH 03/30] Enhance e2e tests for OVN VPC NAT Gateway and VIP - Adjusted the wait duration in the OVN VPC NAT Gateway e2e tests to 2 seconds for checking finalizer addition and resource deletion, improving test responsiveness. - Refactored conditional checks for subnet protocols in VIP e2e tests to use switch-case statements for better readability and maintainability. - Added default cases in protocol checks to handle dual stack scenarios more clearly. - Simplified the loop for waiting on VIP finalizer addition to enhance code clarity. --- makefiles/e2e.mk | 9 + pkg/controller/ovn_eip.go | 1 - pkg/controller/vip.go | 1 - pkg/controller/vpc_nat_gw_eip.go | 1 - test/e2e/ip/e2e_test.go | 60 +- test/e2e/iptables-eip-qos/e2e_test.go | 987 ++++++++++++++++++++ test/e2e/iptables-vpc-nat-gw/e2e_test.go | 1087 +++------------------- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 11 +- test/e2e/vip/e2e_test.go | 16 +- 9 files changed, 1182 insertions(+), 991 deletions(-) create mode 100644 test/e2e/iptables-eip-qos/e2e_test.go diff --git a/makefiles/e2e.mk b/makefiles/e2e.mk index ec8d053ac17..8e910be8714 100644 --- a/makefiles/e2e.mk +++ b/makefiles/e2e.mk @@ -224,6 +224,15 @@ iptables-vpc-nat-gw-conformance-e2e: ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ --focus=CNI:Kube-OVN ./test/e2e/iptables-vpc-nat-gw/iptables-vpc-nat-gw.test -- $(TEST_BIN_ARGS) +.PHONY: iptables-eip-qos-conformance-e2e +iptables-eip-qos-conformance-e2e: + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-eip-qos + E2E_BRANCH=$(E2E_BRANCH) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ + ginkgo $(GINKGO_OUTPUT_OPT) --randomize-all -v \ + --focus=CNI:Kube-OVN ./test/e2e/iptables-eip-qos/iptables-eip-qos.test -- $(TEST_BIN_ARGS) + .PHONY: ovn-vpc-nat-gw-conformance-e2e ovn-vpc-nat-gw-conformance-e2e: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ovn-vpc-nat-gw diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index 71532831a5b..46ddd40a51e 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -51,7 +51,6 @@ func (c *Controller) enqueueUpdateOvnEip(oldObj, newObj any) { klog.Infof("enqueue update ovn eip %s", key) c.updateOvnEipQueue.Add(key) } - } func (c *Controller) enqueueDelOvnEip(obj any) { diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index 1950514f1ff..ad9a9f5e270 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -42,7 +42,6 @@ func (c *Controller) enqueueUpdateVirtualIP(oldObj, newObj any) { klog.Infof("enqueue update virtual parents for %s", key) c.updateVirtualParentsQueue.Add(key) } - } func (c *Controller) enqueueDelVirtualIP(obj any) { diff --git a/pkg/controller/vpc_nat_gw_eip.go b/pkg/controller/vpc_nat_gw_eip.go index dcf6aebcc34..476e7109cb6 100644 --- a/pkg/controller/vpc_nat_gw_eip.go +++ b/pkg/controller/vpc_nat_gw_eip.go @@ -38,7 +38,6 @@ func (c *Controller) enqueueUpdateIptablesEip(oldObj, newObj any) { klog.Infof("enqueue update iptables eip %s", key) c.updateIptablesEipQueue.Add(key) } - } func (c *Controller) enqueueDelIptablesEip(obj any) { diff --git a/test/e2e/ip/e2e_test.go b/test/e2e/ip/e2e_test.go index b483bf84643..9178565d438 100644 --- a/test/e2e/ip/e2e_test.go +++ b/test/e2e/ip/e2e_test.go @@ -44,11 +44,11 @@ var _ = framework.Describe("[group:ip]", func() { ginkgo.By("Creating vpc " + vpcName) vpc := framework.MakeVpc(vpcName, "", false, false, []string{namespaceName}) - vpc = vpcClient.CreateSync(vpc) + _ = vpcClient.CreateSync(vpc) ginkgo.By("Creating subnet " + subnetName) subnet := framework.MakeSubnet(subnetName, "", cidr, "", vpcName, "", nil, nil, []string{namespaceName}) - subnet = subnetClient.CreateSync(subnet) + _ = subnetClient.CreateSync(subnet) }) ginkgo.AfterEach(func() { @@ -76,12 +76,12 @@ var _ = framework.Describe("[group:ip]", func() { podName := "test-ip-pod-" + framework.RandomSuffix() cmd := []string{"sleep", "infinity"} pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) - pod = podClient.CreateSync(pod) + _ = podClient.CreateSync(pod) ginkgo.By("3. Wait for IP CR to be created and get IP CR name") var ipCR *apiv1.IP var ipName string - for i := 0; i < 30; i++ { + for range 30 { // IP CR name format: podName.namespaceName ipName = fmt.Sprintf("%s.%s", podName, namespaceName) ipCR = ipClient.Get(ipName) @@ -94,7 +94,7 @@ var _ = framework.Describe("[group:ip]", func() { framework.ExpectEqual(ipCR.Spec.Subnet, subnetName, "IP CR should be in the correct subnet") ginkgo.By("4. Wait for IP CR finalizer to be added") - for i := 0; i < 60; i++ { + for range 60 { ipCR = ipClient.Get(ipName) if ipCR != nil && len(ipCR.Finalizers) > 0 { break @@ -110,7 +110,8 @@ var _ = framework.Describe("[group:ip]", func() { ginkgo.By("6. Verify subnet status after IP CR creation") afterCreateSubnet := subnetClient.Get(subnetName) - if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + switch afterCreateSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: // Verify IP count changed framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, "V4AvailableIPs should decrease by 1 after IP creation") @@ -127,7 +128,7 @@ var _ = framework.Describe("[group:ip]", func() { podIP := ipCR.Spec.V4IPAddress framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V4UsingIPRange, podIP), "Pod IP %s should be in V4UsingIPRange %s", podIP, afterCreateSubnet.Status.V4UsingIPRange) - } else if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + case apiv1.ProtocolIPv6: // Verify IP count changed framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, "V6AvailableIPs should decrease by 1 after IP creation") @@ -144,7 +145,7 @@ var _ = framework.Describe("[group:ip]", func() { podIP := ipCR.Spec.V6IPAddress framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V6UsingIPRange, podIP), "Pod IP %s should be in V6UsingIPRange %s", podIP, afterCreateSubnet.Status.V6UsingIPRange) - } else { + default: // Dual stack framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, "V4AvailableIPs should decrease by 1 after IP creation") @@ -180,7 +181,7 @@ var _ = framework.Describe("[group:ip]", func() { ginkgo.By("8. Wait for IP CR to be deleted") deleted := false - for i := 0; i < 30; i++ { + for range 30 { _, err := f.KubeOVNClientSet.KubeovnV1().IPs().Get(context.Background(), ipName, metav1.GetOptions{}) if err != nil && k8serrors.IsNotFound(err) { deleted = true @@ -195,7 +196,8 @@ var _ = framework.Describe("[group:ip]", func() { ginkgo.By("10. Verify subnet status after IP CR deletion") afterDeleteSubnet := subnetClient.Get(subnetName) - if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + switch afterDeleteSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: // Verify IP count is restored framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, "V4AvailableIPs should increase by 1 after IP deletion") @@ -213,7 +215,7 @@ var _ = framework.Describe("[group:ip]", func() { "V4AvailableIPs should return to initial value after IP deletion") framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, "V4UsingIPs should return to initial value after IP deletion") - } else if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + case apiv1.ProtocolIPv6: // Verify IP count is restored framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, "V6AvailableIPs should increase by 1 after IP deletion") @@ -231,7 +233,7 @@ var _ = framework.Describe("[group:ip]", func() { "V6AvailableIPs should return to initial value after IP deletion") framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, "V6UsingIPs should return to initial value after IP deletion") - } else { + default: // Dual stack framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, "V4AvailableIPs should increase by 1 after IP deletion") @@ -280,19 +282,19 @@ var _ = framework.Describe("[group:ip]", func() { ipNames := make([]string, numPods) cmd := []string{"sleep", "infinity"} - for i := 0; i < numPods; i++ { + for i := range numPods { podName := fmt.Sprintf("test-multi-ip-pod-%d-%s", i, framework.RandomSuffix()) podNames[i] = podName ipNames[i] = fmt.Sprintf("%s.%s", podName, namespaceName) pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) - pod = podClient.CreateSync(pod) + _ = podClient.CreateSync(pod) } ginkgo.By("3. Wait for all IP CRs to be created and get finalizers") - for i := 0; i < numPods; i++ { + for i := range numPods { var ipCR *apiv1.IP - for j := 0; j < 60; j++ { + for range 60 { ipCR = ipClient.Get(ipNames[i]) if ipCR != nil && ipCR.Name != "" && len(ipCR.Finalizers) > 0 { break @@ -309,17 +311,18 @@ var _ = framework.Describe("[group:ip]", func() { ginkgo.By("5. Verify subnet status after multiple IP CR creations") afterCreateSubnet := subnetClient.Get(subnetName) - if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + switch afterCreateSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: framework.ExpectEqual(initialV4AvailableIPs-float64(numPods), afterCreateSubnet.Status.V4AvailableIPs, "V4AvailableIPs should decrease by %d after creating %d IPs", numPods, numPods) framework.ExpectEqual(initialV4UsingIPs+float64(numPods), afterCreateSubnet.Status.V4UsingIPs, "V4UsingIPs should increase by %d after creating %d IPs", numPods, numPods) - } else if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + case apiv1.ProtocolIPv6: framework.ExpectEqual(initialV6AvailableIPs-float64(numPods), afterCreateSubnet.Status.V6AvailableIPs, "V6AvailableIPs should decrease by %d after creating %d IPs", numPods, numPods) framework.ExpectEqual(initialV6UsingIPs+float64(numPods), afterCreateSubnet.Status.V6UsingIPs, "V6UsingIPs should increase by %d after creating %d IPs", numPods, numPods) - } else { + default: // Dual stack framework.ExpectEqual(initialV4AvailableIPs-float64(numPods), afterCreateSubnet.Status.V4AvailableIPs, "V4AvailableIPs should decrease by %d after creating %d IPs", numPods, numPods) @@ -332,14 +335,14 @@ var _ = framework.Describe("[group:ip]", func() { } ginkgo.By("6. Delete all pods to trigger IP CR deletions") - for i := 0; i < numPods; i++ { + for i := range numPods { podClient.DeleteSync(podNames[i]) } ginkgo.By("7. Wait for all IP CRs to be deleted") - for i := 0; i < numPods; i++ { + for i := range numPods { deleted := false - for j := 0; j < 30; j++ { + for range 30 { _, err := f.KubeOVNClientSet.KubeovnV1().IPs().Get(context.Background(), ipNames[i], metav1.GetOptions{}) if err != nil && k8serrors.IsNotFound(err) { deleted = true @@ -355,17 +358,18 @@ var _ = framework.Describe("[group:ip]", func() { ginkgo.By("9. Verify subnet status after multiple IP CR deletions") afterDeleteSubnet := subnetClient.Get(subnetName) - if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + switch afterDeleteSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, "V4AvailableIPs should return to initial value after all IPs deleted") framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, "V4UsingIPs should return to initial value after all IPs deleted") - } else if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + case apiv1.ProtocolIPv6: framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, "V6AvailableIPs should return to initial value after all IPs deleted") framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, "V6UsingIPs should return to initial value after all IPs deleted") - } else { + default: // Dual stack framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, "V4AvailableIPs should return to initial value after all IPs deleted") @@ -387,12 +391,12 @@ var _ = framework.Describe("[group:ip]", func() { podName := "test-ip-finalizer-pod-" + framework.RandomSuffix() cmd := []string{"sleep", "infinity"} pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) - pod = podClient.CreateSync(pod) + _ = podClient.CreateSync(pod) ginkgo.By("2. Wait for IP CR to be created with finalizer") ipName := fmt.Sprintf("%s.%s", podName, namespaceName) var ipCR *apiv1.IP - for i := 0; i < 60; i++ { + for range 60 { ipCR = ipClient.Get(ipName) if ipCR != nil && ipCR.Name != "" && len(ipCR.Finalizers) > 0 { break @@ -417,7 +421,7 @@ var _ = framework.Describe("[group:ip]", func() { ginkgo.By("5. Verify IP CR is eventually deleted") deleted := false - for i := 0; i < 30; i++ { + for range 30 { _, err := f.KubeOVNClientSet.KubeovnV1().IPs().Get(context.Background(), ipName, metav1.GetOptions{}) if err != nil && k8serrors.IsNotFound(err) { deleted = true diff --git a/test/e2e/iptables-eip-qos/e2e_test.go b/test/e2e/iptables-eip-qos/e2e_test.go new file mode 100644 index 00000000000..0e82db9bc6a --- /dev/null +++ b/test/e2e/iptables-eip-qos/e2e_test.go @@ -0,0 +1,987 @@ +package ovn_eip + +import ( + "context" + "errors" + "flag" + "fmt" + "strconv" + "strings" + "testing" + "time" + + dockernetwork "github.com/moby/moby/api/types/network" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + "k8s.io/kubernetes/test/e2e" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + + "github.com/onsi/ginkgo/v2" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/test/e2e/framework" + "github.com/kubeovn/kube-ovn/test/e2e/framework/docker" + "github.com/kubeovn/kube-ovn/test/e2e/framework/kind" +) + +const ( + dockerExtNet1Name = "kube-ovn-ext-net1" + dockerExtNet2Name = "kube-ovn-ext-net2" + vpcNatGWConfigMapName = "ovn-vpc-nat-gw-config" + vpcNatConfigName = "ovn-vpc-nat-config" + networkAttachDefName = "ovn-vpc-external-network" + externalSubnetProvider = "ovn-vpc-external-network.kube-system" +) + +const ( + iperf2Port = "20288" + skipIperf = false +) + +const ( + eipLimit = iota*5 + 10 + updatedEIPLimit + newEIPLimit + specificIPLimit + defaultNicLimit +) + +func setupNetworkAttachmentDefinition( + f *framework.Framework, + dockerExtNetNetwork *dockernetwork.Inspect, + attachNetClient *framework.NetworkAttachmentDefinitionClient, + subnetClient *framework.SubnetClient, + externalNetworkName string, + nicName string, + provider string, + dockerExtNetName string, +) { + ginkgo.GinkgoHelper() + + ginkgo.By("Getting docker network " + dockerExtNetName) + network, err := docker.NetworkInspect(dockerExtNetName) + framework.ExpectNoError(err, "getting docker network "+dockerExtNetName) + ginkgo.By("Getting or creating network attachment definition " + externalNetworkName) + attachConf := fmt.Sprintf(`{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "%s", + "mode": "bridge", + "ipam": { + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": "%s" + } + }`, nicName, provider) + + // Try to get existing NAD first using raw Kubernetes API to avoid ExpectNoError + nad, err := attachNetClient.NetworkAttachmentDefinitionInterface.Get(context.TODO(), externalNetworkName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // NAD doesn't exist, create it + attachNet := framework.MakeNetworkAttachmentDefinition(externalNetworkName, framework.KubeOvnNamespace, attachConf) + nad = attachNetClient.Create(attachNet) + } else { + framework.ExpectNoError(err, "getting network attachment definition "+externalNetworkName) + } + + ginkgo.By("Got network attachment definition " + nad.Name) + + ginkgo.By("Creating underlay macvlan subnet " + externalNetworkName) + var cidrV4, cidrV6, gatewayV4, gatewayV6 string + for _, config := range dockerExtNetNetwork.IPAM.Config { + switch util.CheckProtocol(config.Subnet.Addr().String()) { + case apiv1.ProtocolIPv4: + if f.HasIPv4() { + cidrV4 = config.Subnet.String() + gatewayV4 = config.Gateway.String() + } + case apiv1.ProtocolIPv6: + if f.HasIPv6() { + cidrV6 = config.Subnet.String() + gatewayV6 = config.Gateway.String() + } + } + } + cidr := make([]string, 0, 2) + gateway := make([]string, 0, 2) + if f.HasIPv4() { + cidr = append(cidr, cidrV4) + gateway = append(gateway, gatewayV4) + } + if f.HasIPv6() { + cidr = append(cidr, cidrV6) + gateway = append(gateway, gatewayV6) + } + excludeIPs := make([]string, 0, len(network.Containers)*2) + for _, container := range network.Containers { + if container.IPv4Address.IsValid() && f.HasIPv4() { + excludeIPs = append(excludeIPs, container.IPv4Address.Addr().String()) + } + if container.IPv6Address.IsValid() && f.HasIPv6() { + excludeIPs = append(excludeIPs, container.IPv6Address.Addr().String()) + } + } + + // Check if subnet already exists + existingSubnet := subnetClient.Get(externalNetworkName) + if existingSubnet == nil { + // Subnet doesn't exist, create it + macvlanSubnet := framework.MakeSubnet(externalNetworkName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", provider, excludeIPs, nil, nil) + _ = subnetClient.CreateSync(macvlanSubnet) + } +} + +func setupVpcNatGwTestEnvironment( + f *framework.Framework, + dockerExtNetNetwork *dockernetwork.Inspect, + attachNetClient *framework.NetworkAttachmentDefinitionClient, + subnetClient *framework.SubnetClient, + vpcClient *framework.VpcClient, + vpcNatGwClient *framework.VpcNatGatewayClient, + vpcName string, + overlaySubnetName string, + vpcNatGwName string, + natGwQosPolicy string, + overlaySubnetV4Cidr string, + overlaySubnetV4Gw string, + lanIP string, + dockerExtNetName string, + externalNetworkName string, + nicName string, + provider string, + skipNADSetup bool, +) { + ginkgo.GinkgoHelper() + + if !skipNADSetup { + setupNetworkAttachmentDefinition( + f, dockerExtNetNetwork, attachNetClient, + subnetClient, externalNetworkName, nicName, provider, dockerExtNetName) + } + + ginkgo.By("Getting config map " + vpcNatGWConfigMapName) + _, err := f.ClientSet.CoreV1().ConfigMaps(framework.KubeOvnNamespace).Get(context.Background(), vpcNatGWConfigMapName, metav1.GetOptions{}) + framework.ExpectNoError(err, "failed to get ConfigMap") + + ginkgo.By("Creating custom vpc " + vpcName) + vpc := framework.MakeVpc(vpcName, lanIP, false, false, nil) + _ = vpcClient.CreateSync(vpc) + + ginkgo.By("Creating custom overlay subnet " + overlaySubnetName) + overlaySubnet := framework.MakeSubnet(overlaySubnetName, "", overlaySubnetV4Cidr, overlaySubnetV4Gw, vpcName, "", nil, nil, nil) + _ = subnetClient.CreateSync(overlaySubnet) + + ginkgo.By("Creating custom vpc nat gw " + vpcNatGwName) + vpcNatGw := framework.MakeVpcNatGateway(vpcNatGwName, vpcName, overlaySubnetName, lanIP, externalNetworkName, natGwQosPolicy) + _ = vpcNatGwClient.CreateSync(vpcNatGw, f.ClientSet) +} + +type qosParams struct { + vpc1Name string + vpc2Name string + vpc1SubnetName string + vpc2SubnetName string + vpcNat1GwName string + vpcNat2GwName string + vpc1EIPName string + vpc2EIPName string + vpc1FIPName string + vpc2FIPName string + vpc1PodName string + vpc2PodName string + attachDefName string + subnetProvider string +} + +// waitForIptablesEIPReady waits for an IptablesEIP to be ready +func waitForIptablesEIPReady(eipClient *framework.IptablesEIPClient, eipName string, timeout time.Duration) *apiv1.IptablesEIP { + ginkgo.GinkgoHelper() + var eip *apiv1.IptablesEIP + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + eip = eipClient.Get(eipName) + if eip != nil && eip.Status.IP != "" && eip.Status.Ready { + framework.Logf("IptablesEIP %s is ready with IP: %s", eipName, eip.Status.IP) + return eip + } + time.Sleep(2 * time.Second) + } + framework.Failf("Timeout waiting for IptablesEIP %s to be ready", eipName) + return nil +} + +func iperf(f *framework.Framework, iperfClientPod *corev1.Pod, iperfServerEIP *apiv1.IptablesEIP) string { + ginkgo.GinkgoHelper() + + for i := range 20 { + command := fmt.Sprintf("iperf -e -p %s --reportstyle C -i 1 -c %s -t 10", iperf2Port, iperfServerEIP.Status.IP) + stdOutput, errOutput, err := framework.ExecShellInPod(context.Background(), f, iperfClientPod.Namespace, iperfClientPod.Name, command) + framework.Logf("output from exec on client pod %s (eip %s)\n", iperfClientPod.Name, iperfServerEIP.Name) + if stdOutput != "" && err == nil { + framework.Logf("output:\n%s", stdOutput) + return stdOutput + } + framework.Logf("exec %s failed err: %v, errOutput: %s, stdOutput: %s, retried %d times.", command, err, errOutput, stdOutput, i) + time.Sleep(6 * time.Second) + } + framework.ExpectNoError(errors.New("iperf failed")) + return "" +} + +func checkQos(f *framework.Framework, + vpc1Pod, vpc2Pod *corev1.Pod, vpc1EIP, vpc2EIP *apiv1.IptablesEIP, + limit int, expect bool, +) { + ginkgo.GinkgoHelper() + + if !skipIperf { + if expect { + output := iperf(f, vpc1Pod, vpc2EIP) + framework.ExpectTrue(validRateLimit(output, limit)) + output = iperf(f, vpc2Pod, vpc1EIP) + framework.ExpectTrue(validRateLimit(output, limit)) + } else { + output := iperf(f, vpc1Pod, vpc2EIP) + framework.ExpectFalse(validRateLimit(output, limit)) + output = iperf(f, vpc2Pod, vpc1EIP) + framework.ExpectFalse(validRateLimit(output, limit)) + } + } +} + +func getNicDefaultQoSPolicy(limit int) apiv1.QoSPolicyBandwidthLimitRules { + return apiv1.QoSPolicyBandwidthLimitRules{ + apiv1.QoSPolicyBandwidthLimitRule{ + Name: "net1-ingress", + Interface: "net1", + RateMax: strconv.Itoa(limit), + BurstMax: strconv.Itoa(limit), + Priority: 3, + Direction: apiv1.QoSDirectionIngress, + }, + apiv1.QoSPolicyBandwidthLimitRule{ + Name: "net1-egress", + Interface: "net1", + RateMax: strconv.Itoa(limit), + BurstMax: strconv.Itoa(limit), + Priority: 3, + Direction: apiv1.QoSDirectionEgress, + }, + } +} + +func getEIPQoSRule(limit int) apiv1.QoSPolicyBandwidthLimitRules { + return apiv1.QoSPolicyBandwidthLimitRules{ + apiv1.QoSPolicyBandwidthLimitRule{ + Name: "eip-ingress", + RateMax: strconv.Itoa(limit), + BurstMax: strconv.Itoa(limit), + Priority: 1, + Direction: apiv1.QoSDirectionIngress, + }, + apiv1.QoSPolicyBandwidthLimitRule{ + Name: "eip-egress", + RateMax: strconv.Itoa(limit), + BurstMax: strconv.Itoa(limit), + Priority: 1, + Direction: apiv1.QoSDirectionEgress, + }, + } +} + +func getSpecialQoSRule(limit int, ip string) apiv1.QoSPolicyBandwidthLimitRules { + return apiv1.QoSPolicyBandwidthLimitRules{ + apiv1.QoSPolicyBandwidthLimitRule{ + Name: "net1-extip-ingress", + Interface: "net1", + RateMax: strconv.Itoa(limit), + BurstMax: strconv.Itoa(limit), + Priority: 2, + Direction: apiv1.QoSDirectionIngress, + MatchType: apiv1.QoSMatchTypeIP, + MatchValue: "src " + ip + "/32", + }, + apiv1.QoSPolicyBandwidthLimitRule{ + Name: "net1-extip-egress", + Interface: "net1", + RateMax: strconv.Itoa(limit), + BurstMax: strconv.Itoa(limit), + Priority: 2, + Direction: apiv1.QoSDirectionEgress, + MatchType: apiv1.QoSMatchTypeIP, + MatchValue: "dst " + ip + "/32", + }, + } +} + +// defaultQoSCases test default qos policy= +func defaultQoSCases(f *framework.Framework, + vpcNatGwClient *framework.VpcNatGatewayClient, + podClient *framework.PodClient, + qosPolicyClient *framework.QoSPolicyClient, + vpc1Pod *corev1.Pod, + vpc2Pod *corev1.Pod, + vpc1EIP *apiv1.IptablesEIP, + vpc2EIP *apiv1.IptablesEIP, + natgwName string, +) { + ginkgo.GinkgoHelper() + + // create nic qos policy + qosPolicyName := "default-nic-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + qosPolicyName) + rules := getNicDefaultQoSPolicy(defaultNicLimit) + + qosPolicy := framework.MakeQoSPolicy(qosPolicyName, true, apiv1.QoSBindingTypeNatGw, rules) + _ = qosPolicyClient.CreateSync(qosPolicy) + + ginkgo.By("Patch natgw " + natgwName + " with qos policy " + qosPolicyName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, qosPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) + + ginkgo.By("Delete natgw pod " + natgwName + "-0") + natGwPodName := util.GenNatGwPodName(natgwName) + podClient.DeleteSync(natGwPodName) + + ginkgo.By("Wait for natgw " + natgwName + "qos rebuild") + time.Sleep(5 * time.Second) + + ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) + + ginkgo.By("Remove qos policy " + qosPolicyName + " from natgw " + natgwName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") + + ginkgo.By("Deleting qos policy " + qosPolicyName) + qosPolicyClient.DeleteSync(qosPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is not limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, false) +} + +// eipQoSCases test default qos policy +func eipQoSCases(f *framework.Framework, + eipClient *framework.IptablesEIPClient, + podClient *framework.PodClient, + qosPolicyClient *framework.QoSPolicyClient, + vpc1Pod *corev1.Pod, + vpc2Pod *corev1.Pod, + vpc1EIP *apiv1.IptablesEIP, + vpc2EIP *apiv1.IptablesEIP, + eipName string, + natgwName string, +) { + ginkgo.GinkgoHelper() + + // create eip qos policy + qosPolicyName := "eip-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + qosPolicyName) + rules := getEIPQoSRule(eipLimit) + + qosPolicy := framework.MakeQoSPolicy(qosPolicyName, false, apiv1.QoSBindingTypeEIP, rules) + qosPolicy = qosPolicyClient.CreateSync(qosPolicy) + + ginkgo.By("Patch eip " + eipName + " with qos policy " + qosPolicyName) + _ = eipClient.PatchQoSPolicySync(eipName, qosPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(eipLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, eipLimit, true) + + ginkgo.By("Update qos policy " + qosPolicyName + " with new rate limit") + + rules = getEIPQoSRule(updatedEIPLimit) + modifiedqosPolicy := qosPolicy.DeepCopy() + modifiedqosPolicy.Spec.BandwidthLimitRules = rules + qosPolicyClient.Patch(qosPolicy, modifiedqosPolicy) + qosPolicyClient.WaitToQoSReady(qosPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is changed to " + strconv.Itoa(updatedEIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, updatedEIPLimit, true) + + ginkgo.By("Delete natgw pod " + natgwName + "-0") + natGwPodName := util.GenNatGwPodName(natgwName) + podClient.DeleteSync(natGwPodName) + + ginkgo.By("Wait for natgw " + natgwName + "qos rebuid") + time.Sleep(5 * time.Second) + + ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(updatedEIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, updatedEIPLimit, true) + + newQoSPolicyName := "new-eip-qos-policy-" + framework.RandomSuffix() + newRules := getEIPQoSRule(newEIPLimit) + newQoSPolicy := framework.MakeQoSPolicy(newQoSPolicyName, false, apiv1.QoSBindingTypeEIP, newRules) + _ = qosPolicyClient.CreateSync(newQoSPolicy) + + ginkgo.By("Change qos policy of eip " + eipName + " to " + newQoSPolicyName) + _ = eipClient.PatchQoSPolicySync(eipName, newQoSPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(newEIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, newEIPLimit, true) + + ginkgo.By("Remove qos policy " + qosPolicyName + " from natgw " + eipName) + _ = eipClient.PatchQoSPolicySync(eipName, "") + + ginkgo.By("Deleting qos policy " + qosPolicyName) + qosPolicyClient.DeleteSync(qosPolicyName) + + ginkgo.By("Deleting qos policy " + newQoSPolicyName) + qosPolicyClient.DeleteSync(newQoSPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is not limited to " + strconv.Itoa(newEIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, newEIPLimit, false) +} + +// specifyingIPQoSCases test default qos policy +func specifyingIPQoSCases(f *framework.Framework, + vpcNatGwClient *framework.VpcNatGatewayClient, + qosPolicyClient *framework.QoSPolicyClient, + vpc1Pod *corev1.Pod, + vpc2Pod *corev1.Pod, + vpc1EIP *apiv1.IptablesEIP, + vpc2EIP *apiv1.IptablesEIP, + natgwName string, +) { + ginkgo.GinkgoHelper() + + // create nic qos policy + qosPolicyName := "specifying-ip-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + qosPolicyName) + + rules := getSpecialQoSRule(specificIPLimit, vpc2EIP.Status.IP) + + qosPolicy := framework.MakeQoSPolicy(qosPolicyName, true, apiv1.QoSBindingTypeNatGw, rules) + _ = qosPolicyClient.CreateSync(qosPolicy) + + ginkgo.By("Patch natgw " + natgwName + " with qos policy " + qosPolicyName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, qosPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(specificIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, specificIPLimit, true) + + ginkgo.By("Remove qos policy " + qosPolicyName + " from natgw " + natgwName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") + + ginkgo.By("Deleting qos policy " + qosPolicyName) + qosPolicyClient.DeleteSync(qosPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is not limited to " + strconv.Itoa(specificIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, specificIPLimit, false) +} + +// priorityQoSCases test qos match priority +func priorityQoSCases(f *framework.Framework, + vpcNatGwClient *framework.VpcNatGatewayClient, + eipClient *framework.IptablesEIPClient, + qosPolicyClient *framework.QoSPolicyClient, + vpc1Pod *corev1.Pod, + vpc2Pod *corev1.Pod, + vpc1EIP *apiv1.IptablesEIP, + vpc2EIP *apiv1.IptablesEIP, + natgwName string, + eipName string, +) { + ginkgo.GinkgoHelper() + + // create nic qos policy + natGwQoSPolicyName := "priority-nic-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + natGwQoSPolicyName) + // default qos policy + special qos policy + natgwRules := getNicDefaultQoSPolicy(defaultNicLimit) + natgwRules = append(natgwRules, getSpecialQoSRule(specificIPLimit, vpc2EIP.Status.IP)...) + + natgwQoSPolicy := framework.MakeQoSPolicy(natGwQoSPolicyName, true, apiv1.QoSBindingTypeNatGw, natgwRules) + _ = qosPolicyClient.CreateSync(natgwQoSPolicy) + + ginkgo.By("Patch natgw " + natgwName + " with qos policy " + natGwQoSPolicyName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, natGwQoSPolicyName) + + eipQoSPolicyName := "eip-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + eipQoSPolicyName) + eipRules := getEIPQoSRule(eipLimit) + + eipQoSPolicy := framework.MakeQoSPolicy(eipQoSPolicyName, false, apiv1.QoSBindingTypeEIP, eipRules) + _ = qosPolicyClient.CreateSync(eipQoSPolicy) + + ginkgo.By("Patch eip " + eipName + " with qos policy " + eipQoSPolicyName) + _ = eipClient.PatchQoSPolicySync(eipName, eipQoSPolicyName) + + // match qos of priority 1 + ginkgo.By("Check qos to match priority 1 is limited to " + strconv.Itoa(eipLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, eipLimit, true) + + ginkgo.By("Remove qos policy " + eipQoSPolicyName + " from natgw " + eipName) + _ = eipClient.PatchQoSPolicySync(eipName, "") + + ginkgo.By("Deleting qos policy " + eipQoSPolicyName) + qosPolicyClient.DeleteSync(eipQoSPolicyName) + + // match qos of priority 2 + ginkgo.By("Check qos to match priority 2 is limited to " + strconv.Itoa(specificIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, specificIPLimit, true) + + // change qos policy of natgw + newNatGwQoSPolicyName := "new-priority-nic-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + newNatGwQoSPolicyName) + newNatgwRules := getNicDefaultQoSPolicy(defaultNicLimit) + + newNatgwQoSPolicy := framework.MakeQoSPolicy(newNatGwQoSPolicyName, true, apiv1.QoSBindingTypeNatGw, newNatgwRules) + _ = qosPolicyClient.CreateSync(newNatgwQoSPolicy) + + ginkgo.By("Change qos policy of natgw " + natgwName + " to " + newNatGwQoSPolicyName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, newNatGwQoSPolicyName) + + // match qos of priority 3 + ginkgo.By("Check qos to match priority 3 is limited to " + strconv.Itoa(specificIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) + + ginkgo.By("Remove qos policy " + natGwQoSPolicyName + " from natgw " + natgwName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") + + ginkgo.By("Deleting qos policy " + natGwQoSPolicyName) + qosPolicyClient.DeleteSync(natGwQoSPolicyName) + + ginkgo.By("Deleting qos policy " + newNatGwQoSPolicyName) + qosPolicyClient.DeleteSync(newNatGwQoSPolicyName) + + ginkgo.By("Check qos " + natGwQoSPolicyName + " is not limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, false) +} + +func createNatGwAndSetQosCases(f *framework.Framework, + vpcNatGwClient *framework.VpcNatGatewayClient, + ipClient *framework.IPClient, + eipClient *framework.IptablesEIPClient, + fipClient *framework.IptablesFIPClient, + subnetClient *framework.SubnetClient, + qosPolicyClient *framework.QoSPolicyClient, + vpc1Pod *corev1.Pod, + vpc2Pod *corev1.Pod, + vpc2EIP *apiv1.IptablesEIP, + natgwName string, + eipName string, + fipName string, + vpcName string, + overlaySubnetName string, + lanIP string, + attachDefName string, +) { + ginkgo.GinkgoHelper() + + // delete fip + ginkgo.By("Deleting fip " + fipName) + fipClient.DeleteSync(fipName) + + ginkgo.By("Deleting eip " + eipName) + eipClient.DeleteSync(eipName) + + // the only pod for vpc nat gateway + vpcNatGw1PodName := util.GenNatGwPodName(natgwName) + + // delete vpc nat gw statefulset remaining ip for eth0 and net2 + ginkgo.By("Deleting custom vpc nat gw " + natgwName) + vpcNatGwClient.DeleteSync(natgwName) + + overlaySubnet1 := subnetClient.Get(overlaySubnetName) + macvlanSubnet := subnetClient.Get(attachDefName) + eth0IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, overlaySubnet1.Spec.Provider) + net1IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, macvlanSubnet.Spec.Provider) + ginkgo.By("Deleting vpc nat gw eth0 ip " + eth0IpName) + ipClient.DeleteSync(eth0IpName) + ginkgo.By("Deleting vpc nat gw net1 ip " + net1IpName) + ipClient.DeleteSync(net1IpName) + + natgwQoSPolicyName := "default-nic-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + natgwQoSPolicyName) + rules := getNicDefaultQoSPolicy(defaultNicLimit) + + qosPolicy := framework.MakeQoSPolicy(natgwQoSPolicyName, true, apiv1.QoSBindingTypeNatGw, rules) + _ = qosPolicyClient.CreateSync(qosPolicy) + + ginkgo.By("Creating custom vpc nat gw") + vpcNatGw := framework.MakeVpcNatGateway(natgwName, vpcName, overlaySubnetName, lanIP, attachDefName, natgwQoSPolicyName) + _ = vpcNatGwClient.CreateSync(vpcNatGw, f.ClientSet) + + eipQoSPolicyName := "eip-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + eipQoSPolicyName) + rules = getEIPQoSRule(eipLimit) + + eipQoSPolicy := framework.MakeQoSPolicy(eipQoSPolicyName, false, apiv1.QoSBindingTypeEIP, rules) + _ = qosPolicyClient.CreateSync(eipQoSPolicy) + + ginkgo.By("Creating eip " + eipName) + vpc1EIP := framework.MakeIptablesEIP(eipName, "", "", "", natgwName, attachDefName, eipQoSPolicyName) + _ = eipClient.CreateSync(vpc1EIP) + vpc1EIP = waitForIptablesEIPReady(eipClient, eipName, 60*time.Second) + + ginkgo.By("Creating fip " + fipName) + fip := framework.MakeIptablesFIPRule(fipName, eipName, vpc1Pod.Status.PodIP) + _ = fipClient.CreateSync(fip) + + ginkgo.By("Check qos " + eipQoSPolicyName + " is limited to " + strconv.Itoa(eipLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, eipLimit, true) + + ginkgo.By("Remove qos policy " + eipQoSPolicyName + " from natgw " + natgwName) + _ = eipClient.PatchQoSPolicySync(eipName, "") + + ginkgo.By("Check qos " + natgwQoSPolicyName + " is limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) + + ginkgo.By("Remove qos policy " + natgwQoSPolicyName + " from natgw " + natgwName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") + + ginkgo.By("Check qos " + natgwQoSPolicyName + " is not limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, false) + + ginkgo.By("Deleting qos policy " + natgwQoSPolicyName) + qosPolicyClient.DeleteSync(natgwQoSPolicyName) + + ginkgo.By("Deleting qos policy " + eipQoSPolicyName) + qosPolicyClient.DeleteSync(eipQoSPolicyName) +} + +func validRateLimit(text string, limit int) bool { + maxValue := float64(limit) * 1024 * 1024 * 1.2 + minValue := float64(limit) * 1024 * 1024 * 0.8 + lines := strings.SplitSeq(text, "\n") + for line := range lines { + if line == "" { + continue + } + fields := strings.Split(line, ",") + number, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + continue + } + if v := float64(number); v >= minValue && v <= maxValue { + return true + } + } + return false +} + +var _ = framework.Describe("[group:qos-policy]", func() { + f := framework.NewDefaultFramework("qos-policy") + + var skip bool + var cs clientset.Interface + var attachNetClient *framework.NetworkAttachmentDefinitionClient + var clusterName string + var vpcClient *framework.VpcClient + var vpcNatGwClient *framework.VpcNatGatewayClient + var subnetClient *framework.SubnetClient + var podClient *framework.PodClient + var ipClient *framework.IPClient + var iptablesEIPClient *framework.IptablesEIPClient + var iptablesFIPClient *framework.IptablesFIPClient + var qosPolicyClient *framework.QoSPolicyClient + + var net1NicName string + var dockerExtNetName string + + // docker network + var dockerExtNetNetwork *dockernetwork.Inspect + + var vpcQosParams *qosParams + var vpc1Pod *corev1.Pod + var vpc2Pod *corev1.Pod + var vpc1EIP *apiv1.IptablesEIP + var vpc2EIP *apiv1.IptablesEIP + var vpc1FIP *apiv1.IptablesFIPRule + var vpc2FIP *apiv1.IptablesFIPRule + + var lanIP string + var overlaySubnetV4Cidr string + var overlaySubnetV4Gw string + var eth0Exist, net1Exist bool + var annotations1 map[string]string + var annotations2 map[string]string + var iperfServerCmd []string + + ginkgo.BeforeEach(func() { + randomSuffix := framework.RandomSuffix() + vpcQosParams = &qosParams{ + vpc1Name: "qos-vpc1-" + randomSuffix, + vpc2Name: "qos-vpc2-" + randomSuffix, + vpc1SubnetName: "qos-vpc1-subnet-" + randomSuffix, + vpc2SubnetName: "qos-vpc2-subnet-" + randomSuffix, + vpcNat1GwName: "qos-gw1-" + randomSuffix, + vpcNat2GwName: "qos-gw2-" + randomSuffix, + vpc1EIPName: "qos-vpc1-eip-" + randomSuffix, + vpc2EIPName: "qos-vpc2-eip-" + randomSuffix, + vpc1FIPName: "qos-vpc1-fip-" + randomSuffix, + vpc2FIPName: "qos-vpc2-fip-" + randomSuffix, + vpc1PodName: "qos-vpc1-pod-" + randomSuffix, + vpc2PodName: "qos-vpc2-pod-" + randomSuffix, + attachDefName: "qos-ovn-vpc-external-network-" + randomSuffix, + } + vpcQosParams.subnetProvider = fmt.Sprintf("%s.%s", vpcQosParams.attachDefName, framework.KubeOvnNamespace) + + dockerExtNetName = "kube-ovn-qos-" + randomSuffix + + cs = f.ClientSet + podClient = f.PodClient() + attachNetClient = f.NetworkAttachmentDefinitionClientNS(framework.KubeOvnNamespace) + subnetClient = f.SubnetClient() + vpcClient = f.VpcClient() + vpcNatGwClient = f.VpcNatGatewayClient() + iptablesEIPClient = f.IptablesEIPClient() + ipClient = f.IPClient() + iptablesFIPClient = f.IptablesFIPClient() + qosPolicyClient = f.QoSPolicyClient() + + if skip { + ginkgo.Skip("underlay spec only runs on kind clusters") + } + + if clusterName == "" { + ginkgo.By("Getting k8s nodes") + k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) + framework.ExpectNoError(err) + + cluster, ok := kind.IsKindProvided(k8sNodes.Items[0].Spec.ProviderID) + if !ok { + skip = true + ginkgo.Skip("underlay spec only runs on kind clusters") + } + clusterName = cluster + } + + ginkgo.By("Ensuring docker network " + dockerExtNetName + " exists") + network, err := docker.NetworkCreate(dockerExtNetName, true, true) + framework.ExpectNoError(err, "creating docker network "+dockerExtNetName) + dockerExtNetNetwork = network + + ginkgo.By("Getting kind nodes") + nodes, err := kind.ListNodes(clusterName, "") + framework.ExpectNoError(err, "getting nodes in kind cluster") + framework.ExpectNotEmpty(nodes) + + ginkgo.By("Connecting nodes to the docker network") + err = kind.NetworkConnect(dockerExtNetNetwork.ID, nodes) + framework.ExpectNoError(err, "connecting nodes to network "+dockerExtNetName) + + ginkgo.By("Getting node links that belong to the docker network") + nodes, err = kind.ListNodes(clusterName, "") + framework.ExpectNoError(err, "getting nodes in kind cluster") + + ginkgo.By("Validating node links") + network1, err := docker.NetworkInspect(dockerExtNetName) + framework.ExpectNoError(err) + for _, node := range nodes { + links, err := node.ListLinks() + framework.ExpectNoError(err, "failed to list links on node %s: %v", node.Name(), err) + net1Mac := network1.Containers[node.ID].MacAddress + for _, link := range links { + ginkgo.By("exist node nic " + link.IfName) + if link.IfName == "eth0" { + eth0Exist = true + } + if link.Address == net1Mac.String() { + net1NicName = link.IfName + net1Exist = true + } + } + framework.ExpectTrue(eth0Exist) + framework.ExpectTrue(net1Exist) + } + setupNetworkAttachmentDefinition( + f, dockerExtNetNetwork, attachNetClient, + subnetClient, vpcQosParams.attachDefName, net1NicName, vpcQosParams.subnetProvider, dockerExtNetName) + }) + + ginkgo.AfterEach(func() { + ginkgo.By("Deleting macvlan underlay subnet " + vpcQosParams.attachDefName) + subnetClient.DeleteSync(vpcQosParams.attachDefName) + + // delete net1 attachment definition + ginkgo.By("Deleting nad " + vpcQosParams.attachDefName) + attachNetClient.Delete(vpcQosParams.attachDefName) + + ginkgo.By("Getting nodes") + nodes, err := kind.ListNodes(clusterName, "") + framework.ExpectNoError(err, "getting nodes in cluster") + + if dockerExtNetNetwork != nil { + ginkgo.By("Disconnecting nodes from the docker network") + err = kind.NetworkDisconnect(dockerExtNetNetwork.ID, nodes) + framework.ExpectNoError(err, "disconnecting nodes from network "+dockerExtNetName) + ginkgo.By("Deleting docker network " + dockerExtNetName + " exists") + err := docker.NetworkRemove(dockerExtNetNetwork.ID) + framework.ExpectNoError(err, "deleting docker network "+dockerExtNetName) + } + }) + + _ = framework.Describe("vpc qos", func() { + ginkgo.BeforeEach(func() { + iperfServerCmd = []string{"iperf", "-s", "-i", "1", "-p", iperf2Port} + overlaySubnetV4Cidr = "10.0.0.0/24" + overlaySubnetV4Gw = "10.0.0.1" + lanIP = "10.0.0.254" + natgwQoS := "" + setupVpcNatGwTestEnvironment( + f, dockerExtNetNetwork, attachNetClient, + subnetClient, vpcClient, vpcNatGwClient, + vpcQosParams.vpc1Name, vpcQosParams.vpc1SubnetName, vpcQosParams.vpcNat1GwName, + natgwQoS, overlaySubnetV4Cidr, overlaySubnetV4Gw, lanIP, + dockerExtNetName, vpcQosParams.attachDefName, net1NicName, + vpcQosParams.subnetProvider, + true, + ) + annotations1 = map[string]string{ + util.LogicalSwitchAnnotation: vpcQosParams.vpc1SubnetName, + } + ginkgo.By("Creating pod " + vpcQosParams.vpc1PodName) + vpc1Pod = framework.MakePod(f.Namespace.Name, vpcQosParams.vpc1PodName, nil, annotations1, framework.AgnhostImage, iperfServerCmd, nil) + vpc1Pod = podClient.CreateSync(vpc1Pod) + + ginkgo.By("Creating eip " + vpcQosParams.vpc1EIPName) + vpc1EIP = framework.MakeIptablesEIP(vpcQosParams.vpc1EIPName, "", "", "", vpcQosParams.vpcNat1GwName, vpcQosParams.attachDefName, "") + _ = iptablesEIPClient.CreateSync(vpc1EIP) + vpc1EIP = waitForIptablesEIPReady(iptablesEIPClient, vpcQosParams.vpc1EIPName, 60*time.Second) + + ginkgo.By("Creating fip " + vpcQosParams.vpc1FIPName) + vpc1FIP = framework.MakeIptablesFIPRule(vpcQosParams.vpc1FIPName, vpcQosParams.vpc1EIPName, vpc1Pod.Status.PodIP) + _ = iptablesFIPClient.CreateSync(vpc1FIP) + + setupVpcNatGwTestEnvironment( + f, dockerExtNetNetwork, attachNetClient, + subnetClient, vpcClient, vpcNatGwClient, + vpcQosParams.vpc2Name, vpcQosParams.vpc2SubnetName, vpcQosParams.vpcNat2GwName, + natgwQoS, overlaySubnetV4Cidr, overlaySubnetV4Gw, lanIP, + dockerExtNetName, vpcQosParams.attachDefName, net1NicName, + vpcQosParams.subnetProvider, + true, + ) + + annotations2 = map[string]string{ + util.LogicalSwitchAnnotation: vpcQosParams.vpc2SubnetName, + } + + ginkgo.By("Creating pod " + vpcQosParams.vpc2PodName) + vpc2Pod = framework.MakePod(f.Namespace.Name, vpcQosParams.vpc2PodName, nil, annotations2, framework.AgnhostImage, iperfServerCmd, nil) + vpc2Pod = podClient.CreateSync(vpc2Pod) + + ginkgo.By("Creating eip " + vpcQosParams.vpc2EIPName) + vpc2EIP = framework.MakeIptablesEIP(vpcQosParams.vpc2EIPName, "", "", "", vpcQosParams.vpcNat2GwName, vpcQosParams.attachDefName, "") + _ = iptablesEIPClient.CreateSync(vpc2EIP) + vpc2EIP = waitForIptablesEIPReady(iptablesEIPClient, vpcQosParams.vpc2EIPName, 60*time.Second) + + ginkgo.By("Creating fip " + vpcQosParams.vpc2FIPName) + vpc2FIP = framework.MakeIptablesFIPRule(vpcQosParams.vpc2FIPName, vpcQosParams.vpc2EIPName, vpc2Pod.Status.PodIP) + _ = iptablesFIPClient.CreateSync(vpc2FIP) + }) + ginkgo.AfterEach(func() { + ginkgo.By("Deleting fip " + vpcQosParams.vpc1FIPName) + iptablesFIPClient.DeleteSync(vpcQosParams.vpc1FIPName) + + ginkgo.By("Deleting fip " + vpcQosParams.vpc2FIPName) + iptablesFIPClient.DeleteSync(vpcQosParams.vpc2FIPName) + + ginkgo.By("Deleting eip " + vpcQosParams.vpc1EIPName) + iptablesEIPClient.DeleteSync(vpcQosParams.vpc1EIPName) + + ginkgo.By("Deleting eip " + vpcQosParams.vpc2EIPName) + iptablesEIPClient.DeleteSync(vpcQosParams.vpc2EIPName) + + ginkgo.By("Deleting pod " + vpcQosParams.vpc1PodName) + podClient.DeleteSync(vpcQosParams.vpc1PodName) + + ginkgo.By("Deleting pod " + vpcQosParams.vpc2PodName) + podClient.DeleteSync(vpcQosParams.vpc2PodName) + + ginkgo.By("Deleting custom vpc nat gw " + vpcQosParams.vpcNat1GwName) + vpcNatGwClient.DeleteSync(vpcQosParams.vpcNat1GwName) + + ginkgo.By("Deleting custom vpc nat gw " + vpcQosParams.vpcNat2GwName) + vpcNatGwClient.DeleteSync(vpcQosParams.vpcNat2GwName) + + // the only pod for vpc nat gateway + vpcNatGw1PodName := util.GenNatGwPodName(vpcQosParams.vpcNat1GwName) + + // delete vpc nat gw statefulset remaining ip for eth0 and net2 + overlaySubnet1 := subnetClient.Get(vpcQosParams.vpc1SubnetName) + macvlanSubnet := subnetClient.Get(vpcQosParams.attachDefName) + eth0IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, overlaySubnet1.Spec.Provider) + net1IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, macvlanSubnet.Spec.Provider) + ginkgo.By("Deleting vpc nat gw eth0 ip " + eth0IpName) + ipClient.DeleteSync(eth0IpName) + ginkgo.By("Deleting vpc nat gw net1 ip " + net1IpName) + ipClient.DeleteSync(net1IpName) + ginkgo.By("Deleting overlay subnet " + vpcQosParams.vpc1SubnetName) + subnetClient.DeleteSync(vpcQosParams.vpc1SubnetName) + + ginkgo.By("Getting overlay subnet " + vpcQosParams.vpc2SubnetName) + overlaySubnet2 := subnetClient.Get(vpcQosParams.vpc2SubnetName) + + vpcNatGw2PodName := util.GenNatGwPodName(vpcQosParams.vpcNat2GwName) + eth0IpName = ovs.PodNameToPortName(vpcNatGw2PodName, framework.KubeOvnNamespace, overlaySubnet2.Spec.Provider) + net1IpName = ovs.PodNameToPortName(vpcNatGw2PodName, framework.KubeOvnNamespace, macvlanSubnet.Spec.Provider) + ginkgo.By("Deleting vpc nat gw eth0 ip " + eth0IpName) + ipClient.DeleteSync(eth0IpName) + ginkgo.By("Deleting vpc nat gw net1 ip " + net1IpName) + ipClient.DeleteSync(net1IpName) + ginkgo.By("Deleting overlay subnet " + vpcQosParams.vpc2SubnetName) + subnetClient.DeleteSync(vpcQosParams.vpc2SubnetName) + + ginkgo.By("Deleting custom vpc " + vpcQosParams.vpc1Name) + vpcClient.DeleteSync(vpcQosParams.vpc1Name) + + ginkgo.By("Deleting custom vpc " + vpcQosParams.vpc2Name) + vpcClient.DeleteSync(vpcQosParams.vpc2Name) + }) + framework.ConformanceIt("default nic qos", func() { + // case 1: set qos policy for natgw + // case 2: rebuild qos when natgw pod restart + defaultQoSCases(f, vpcNatGwClient, podClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpcNat1GwName) + }) + framework.ConformanceIt("eip qos", func() { + // case 1: set qos policy for eip + // case 2: update qos policy for eip + // case 3: change qos policy of eip + // case 4: rebuild qos when natgw pod restart + eipQoSCases(f, iptablesEIPClient, podClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpc1EIPName, vpcQosParams.vpcNat1GwName) + }) + framework.ConformanceIt("specifying ip qos", func() { + // case 1: set specific ip qos policy for natgw + specifyingIPQoSCases(f, vpcNatGwClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpcNat1GwName) + }) + framework.ConformanceIt("qos priority matching", func() { + // case 1: test qos match priority + // case 2: change qos policy of natgw + priorityQoSCases(f, vpcNatGwClient, iptablesEIPClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpcNat1GwName, vpcQosParams.vpc1EIPName) + }) + framework.ConformanceIt("create resource with qos policy", func() { + // case 1: test qos when create natgw with qos policy + // case 2: test qos when create eip with qos policy + createNatGwAndSetQosCases(f, + vpcNatGwClient, ipClient, iptablesEIPClient, iptablesFIPClient, + subnetClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc2EIP, vpcQosParams.vpcNat1GwName, + vpcQosParams.vpc1EIPName, vpcQosParams.vpc1FIPName, vpcQosParams.vpc1Name, + vpcQosParams.vpc1SubnetName, lanIP, vpcQosParams.attachDefName) + }) + }) +}) + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + e2e.RunE2ETests(t) +} diff --git a/test/e2e/iptables-vpc-nat-gw/e2e_test.go b/test/e2e/iptables-vpc-nat-gw/e2e_test.go index 574b3770b65..2e1df53e624 100644 --- a/test/e2e/iptables-vpc-nat-gw/e2e_test.go +++ b/test/e2e/iptables-vpc-nat-gw/e2e_test.go @@ -2,16 +2,13 @@ package ovn_eip import ( "context" - "errors" "flag" "fmt" - "strconv" "strings" "testing" "time" dockernetwork "github.com/moby/moby/api/types/network" - corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" @@ -40,36 +37,6 @@ const ( externalSubnetProvider = "ovn-vpc-external-network.kube-system" ) -const ( - iperf2Port = "20288" - skipIperf = false -) - -const ( - eipLimit = iota*5 + 10 - updatedEIPLimit - newEIPLimit - specificIPLimit - defaultNicLimit -) - -type qosParams struct { - vpc1Name string - vpc2Name string - vpc1SubnetName string - vpc2SubnetName string - vpcNat1GwName string - vpcNat2GwName string - vpc1EIPName string - vpc2EIPName string - vpc1FIPName string - vpc2FIPName string - vpc1PodName string - vpc2PodName string - attachDefName string - subnetProvider string -} - func setupNetworkAttachmentDefinition( f *framework.Framework, dockerExtNetNetwork *dockernetwork.Inspect, @@ -85,7 +52,7 @@ func setupNetworkAttachmentDefinition( ginkgo.By("Getting docker network " + dockerExtNetName) network, err := docker.NetworkInspect(dockerExtNetName) framework.ExpectNoError(err, "getting docker network "+dockerExtNetName) - ginkgo.By("Getting network attachment definition " + externalNetworkName) + ginkgo.By("Getting or creating network attachment definition " + externalNetworkName) attachConf := fmt.Sprintf(`{ "cniVersion": "0.3.0", "type": "macvlan", @@ -97,9 +64,16 @@ func setupNetworkAttachmentDefinition( "provider": "%s" } }`, nicName, provider) - attachNet := framework.MakeNetworkAttachmentDefinition(externalNetworkName, framework.KubeOvnNamespace, attachConf) - attachNetClient.Create(attachNet) - nad := attachNetClient.Get(externalNetworkName) + + // Try to get existing NAD first + nad, err := attachNetClient.NetworkAttachmentDefinitionInterface.Get(context.TODO(), externalNetworkName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // NAD doesn't exist, create it + attachNet := framework.MakeNetworkAttachmentDefinition(externalNetworkName, framework.KubeOvnNamespace, attachConf) + nad = attachNetClient.Create(attachNet) + } else { + framework.ExpectNoError(err, "getting network attachment definition "+externalNetworkName) + } ginkgo.By("Got network attachment definition " + nad.Name) @@ -138,8 +112,16 @@ func setupNetworkAttachmentDefinition( excludeIPs = append(excludeIPs, container.IPv6Address.Addr().String()) } } - macvlanSubnet := framework.MakeSubnet(externalNetworkName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", provider, excludeIPs, nil, nil) - _ = subnetClient.CreateSync(macvlanSubnet) + + // Check if subnet already exists + _, err = subnetClient.SubnetInterface.Get(context.TODO(), externalNetworkName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // Subnet doesn't exist, create it + macvlanSubnet := framework.MakeSubnet(externalNetworkName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", provider, excludeIPs, nil, nil) + _ = subnetClient.CreateSync(macvlanSubnet) + } else { + framework.ExpectNoError(err, "getting subnet "+externalNetworkName) + } } func setupVpcNatGwTestEnvironment( @@ -208,6 +190,54 @@ func cleanVpcNatGwTestEnvironment( vpcClient.DeleteSync(vpcName) } +// waitForIptablesEIPReady waits for an IptablesEIP to be ready +func waitForIptablesEIPReady(eipClient *framework.IptablesEIPClient, eipName string, timeout time.Duration) *apiv1.IptablesEIP { + ginkgo.GinkgoHelper() + var eip *apiv1.IptablesEIP + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + eip = eipClient.Get(eipName) + if eip != nil && eip.Status.IP != "" && eip.Status.Ready { + framework.Logf("IptablesEIP %s is ready with IP: %s", eipName, eip.Status.IP) + return eip + } + time.Sleep(2 * time.Second) + } + framework.Failf("Timeout waiting for IptablesEIP %s to be ready", eipName) + return nil +} + +// verifySubnetStatusAfterEIPOperation verifies subnet status after EIP operation +func verifySubnetStatusAfterEIPOperation(subnetClient *framework.SubnetClient, subnetName string, + protocol, operation, shouldContainIP string, +) { + ginkgo.GinkgoHelper() + + subnet := subnetClient.Get(subnetName) + framework.Logf("Verifying subnet %s status after %s: Protocol=%s", subnetName, operation, protocol) + + switch protocol { + case apiv1.ProtocolIPv4: + framework.Logf("V4 Status: Available=%.0f, Using=%.0f", + subnet.Status.V4AvailableIPs, subnet.Status.V4UsingIPs) + if shouldContainIP != "" { + framework.ExpectTrue(strings.Contains(subnet.Status.V4UsingIPRange, shouldContainIP), + "IP %s should be in V4UsingIPRange after %s", shouldContainIP, operation) + } + case apiv1.ProtocolIPv6: + framework.Logf("V6 Status: Available=%.0f, Using=%.0f", + subnet.Status.V6AvailableIPs, subnet.Status.V6UsingIPs) + if shouldContainIP != "" { + framework.ExpectTrue(strings.Contains(subnet.Status.V6UsingIPRange, shouldContainIP), + "IP %s should be in V6UsingIPRange after %s", shouldContainIP, operation) + } + case apiv1.ProtocolDual: + framework.Logf("Dual Stack Status: V4Available=%.0f, V4Using=%.0f, V6Available=%.0f, V6Using=%.0f", + subnet.Status.V4AvailableIPs, subnet.Status.V4UsingIPs, + subnet.Status.V6AvailableIPs, subnet.Status.V6UsingIPs) + } +} + var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { f := framework.NewDefaultFramework("iptables-vpc-nat-gw") @@ -241,37 +271,38 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { var net2VpcName string var net2EipName string - vpcName = "vpc-" + framework.RandomSuffix() - vpcNatGwName = "gw-" + framework.RandomSuffix() - - fipVipName = "fip-vip-" + framework.RandomSuffix() - fipEipName = "fip-eip-" + framework.RandomSuffix() - fipName = "fip-" + framework.RandomSuffix() - - dnatVipName = "dnat-vip-" + framework.RandomSuffix() - dnatEipName = "dnat-eip-" + framework.RandomSuffix() - dnatName = "dnat-" + framework.RandomSuffix() - - // sharing case - sharedVipName = "shared-vip-" + framework.RandomSuffix() - sharedEipName = "shared-eip-" + framework.RandomSuffix() - sharedEipDnatName = "shared-eip-dnat-" + framework.RandomSuffix() - sharedEipSnatName = "shared-eip-snat-" + framework.RandomSuffix() - sharedEipFipShouldOkName = "shared-eip-fip-should-ok-" + framework.RandomSuffix() - sharedEipFipShouldFailName = "shared-eip-fip-should-fail-" + framework.RandomSuffix() - - snatEipName = "snat-eip-" + framework.RandomSuffix() - snatName = "snat-" + framework.RandomSuffix() - overlaySubnetName = "overlay-subnet-" + framework.RandomSuffix() - - net2AttachDefName = "net2-ovn-vpc-external-network-" + framework.RandomSuffix() - net2SubnetProvider = fmt.Sprintf("%s.%s", net2AttachDefName, framework.KubeOvnNamespace) - net2OverlaySubnetName = "net2-overlay-subnet-" + framework.RandomSuffix() - net2VpcNatGwName = "net2-gw-" + framework.RandomSuffix() - net2VpcName = "net2-vpc-" + framework.RandomSuffix() - net2EipName = "net2-eip-" + framework.RandomSuffix() - ginkgo.BeforeEach(func() { + randomSuffix := framework.RandomSuffix() + vpcName = "vpc-" + randomSuffix + vpcNatGwName = "gw-" + randomSuffix + + fipVipName = "fip-vip-" + randomSuffix + fipEipName = "fip-eip-" + randomSuffix + fipName = "fip-" + randomSuffix + + dnatVipName = "dnat-vip-" + randomSuffix + dnatEipName = "dnat-eip-" + randomSuffix + dnatName = "dnat-" + randomSuffix + + // sharing case + sharedVipName = "shared-vip-" + randomSuffix + sharedEipName = "shared-eip-" + randomSuffix + sharedEipDnatName = "shared-eip-dnat-" + randomSuffix + sharedEipSnatName = "shared-eip-snat-" + randomSuffix + sharedEipFipShouldOkName = "shared-eip-fip-should-ok-" + randomSuffix + sharedEipFipShouldFailName = "shared-eip-fip-should-fail-" + randomSuffix + + snatEipName = "snat-eip-" + randomSuffix + snatName = "snat-" + randomSuffix + overlaySubnetName = "overlay-subnet-" + randomSuffix + + net2AttachDefName = "net2-ovn-vpc-external-network-" + randomSuffix + net2SubnetProvider = fmt.Sprintf("%s.%s", net2AttachDefName, framework.KubeOvnNamespace) + net2OverlaySubnetName = "net2-overlay-subnet-" + randomSuffix + net2VpcNatGwName = "net2-gw-" + randomSuffix + net2VpcName = "net2-vpc-" + randomSuffix + net2EipName = "net2-eip-" + randomSuffix + cs = f.ClientSet attachNetClient = f.NetworkAttachmentDefinitionClientNS(framework.KubeOvnNamespace) subnetClient = f.SubnetClient() @@ -521,10 +552,10 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { iptablesFIPClient.DeleteSync(sharedEipFipShouldOkName) ginkgo.By("Deleting share iptables fip " + sharedEipFipShouldFailName) iptablesFIPClient.DeleteSync(sharedEipFipShouldFailName) - ginkgo.By("Deleting share iptables dnat " + dnatName) - iptablesDnatRuleClient.DeleteSync(dnatName) - ginkgo.By("Deleting share iptables snat " + snatName) - iptablesSnatRuleClient.DeleteSync(snatName) + ginkgo.By("Deleting share iptables dnat " + sharedEipDnatName) + iptablesDnatRuleClient.DeleteSync(sharedEipDnatName) + ginkgo.By("Deleting share iptables snat " + sharedEipSnatName) + iptablesSnatRuleClient.DeleteSync(sharedEipSnatName) ginkgo.By("Deleting iptables fip " + fipName) iptablesFIPClient.DeleteSync(fipName) @@ -618,54 +649,6 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { vpcClient.DeleteSync(net2VpcName) }) - // Helper function to wait for EIP to be ready - _ = func(eipClient *framework.IptablesEIPClient, eipName string, timeout time.Duration) *apiv1.IptablesEIP { - ginkgo.GinkgoHelper() - var eip *apiv1.IptablesEIP - deadline := time.Now().Add(timeout) - for time.Now().Before(deadline) { - eip = eipClient.Get(eipName) - if eip != nil && eip.Status.IP != "" && eip.Status.Ready { - framework.Logf("IptablesEIP %s is ready with IP: %s", eipName, eip.Status.IP) - return eip - } - time.Sleep(2 * time.Second) - } - framework.Failf("Timeout waiting for IptablesEIP %s to be ready", eipName) - return nil - } - - // Helper function to verify subnet status after EIP operation - _ = func(subnetClient *framework.SubnetClient, subnetName string, - protocol string, expectedAvailableDelta, expectedUsingDelta float64, - operation string, shouldContainIP string) { - ginkgo.GinkgoHelper() - - subnet := subnetClient.Get(subnetName) - framework.Logf("Verifying subnet %s status after %s: Protocol=%s", subnetName, operation, protocol) - - switch protocol { - case apiv1.ProtocolIPv4: - framework.Logf("V4 Status: Available=%.0f, Using=%.0f", - subnet.Status.V4AvailableIPs, subnet.Status.V4UsingIPs) - if shouldContainIP != "" { - framework.ExpectTrue(strings.Contains(subnet.Status.V4UsingIPRange, shouldContainIP), - "IP %s should be in V4UsingIPRange after %s", shouldContainIP, operation) - } - case apiv1.ProtocolIPv6: - framework.Logf("V6 Status: Available=%.0f, Using=%.0f", - subnet.Status.V6AvailableIPs, subnet.Status.V6UsingIPs) - if shouldContainIP != "" { - framework.ExpectTrue(strings.Contains(subnet.Status.V6UsingIPRange, shouldContainIP), - "IP %s should be in V6UsingIPRange after %s", shouldContainIP, operation) - } - case apiv1.ProtocolDual: - framework.Logf("Dual Stack Status: V4Available=%.0f, V4Using=%.0f, V6Available=%.0f, V6Using=%.0f", - subnet.Status.V4AvailableIPs, subnet.Status.V4UsingIPs, - subnet.Status.V6AvailableIPs, subnet.Status.V6UsingIPs) - } - } - framework.ConformanceIt("should properly manage IptablesEIP lifecycle with finalizer and update subnet status", func() { f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") @@ -700,21 +683,13 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { eip := framework.MakeIptablesEIP(eipName, "", "", "", vpcNatGwName, "", "") _ = iptablesEIPClient.CreateSync(eip) - ginkgo.By("3. Wait for IptablesEIP CR to be created and get IP") - var eipCR *apiv1.IptablesEIP - for i := 0; i < 60; i++ { - eipCR = iptablesEIPClient.Get(eipName) - if eipCR != nil && eipCR.Status.IP != "" && eipCR.Status.Ready { - break - } - time.Sleep(1 * time.Second) - } - framework.ExpectNotNil(eipCR, "IptablesEIP CR should be created") + ginkgo.By("3. Wait for IptablesEIP CR to be ready") + eipCR := waitForIptablesEIPReady(iptablesEIPClient, eipName, 60*time.Second) + framework.ExpectNotNil(eipCR, "IptablesEIP CR should be created and ready") framework.ExpectNotEmpty(eipCR.Status.IP, "IptablesEIP should have IP assigned") - framework.ExpectTrue(eipCR.Status.Ready, "IptablesEIP should be ready") ginkgo.By("4. Wait for IptablesEIP CR finalizer to be added") - for i := 0; i < 60; i++ { + for range 60 { eipCR = iptablesEIPClient.Get(eipName) if eipCR != nil && len(eipCR.Finalizers) > 0 { break @@ -730,41 +705,30 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { ginkgo.By("6. Verify external subnet status after IptablesEIP CR creation") afterCreateSubnet := subnetClient.Get(externalSubnetName) - if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { - // Verify IP count changed + verifySubnetStatusAfterEIPOperation(subnetClient, externalSubnetName, + afterCreateSubnet.Spec.Protocol, "IptablesEIP creation", eipCR.Status.IP) + + // Verify IP count and range changes + switch afterCreateSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, "V4AvailableIPs should decrease by 1 after IptablesEIP creation") framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, "V4UsingIPs should increase by 1 after IptablesEIP creation") - - // Verify IP range changed framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, "V4AvailableIPRange should change after IptablesEIP creation") framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, "V4UsingIPRange should change after IptablesEIP creation") - - // Verify the EIP's address is in the using range - eipIP := eipCR.Status.IP - framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V4UsingIPRange, eipIP), - "EIP IP %s should be in V4UsingIPRange %s", eipIP, afterCreateSubnet.Status.V4UsingIPRange) - } else if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { - // Verify IP count changed + case apiv1.ProtocolIPv6: framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, "V6AvailableIPs should decrease by 1 after IptablesEIP creation") framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, "V6UsingIPs should increase by 1 after IptablesEIP creation") - - // Verify IP range changed framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, "V6AvailableIPRange should change after IptablesEIP creation") framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, "V6UsingIPRange should change after IptablesEIP creation") - - // Verify the EIP's address is in the using range - eipIP := eipCR.Status.IP - framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V6UsingIPRange, eipIP), - "EIP IP %s should be in V6UsingIPRange %s", eipIP, afterCreateSubnet.Status.V6UsingIPRange) - } else { + default: // Dual stack framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, "V4AvailableIPs should decrease by 1 after IptablesEIP creation") @@ -774,7 +738,6 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { "V6AvailableIPs should decrease by 1 after IptablesEIP creation") framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, "V6UsingIPs should increase by 1 after IptablesEIP creation") - framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, "V4AvailableIPRange should change after IptablesEIP creation") framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, @@ -800,7 +763,7 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { ginkgo.By("8. Wait for IptablesEIP CR to be deleted") deleted := false - for i := 0; i < 30; i++ { + for range 30 { _, err := f.KubeOVNClientSet.KubeovnV1().IptablesEIPs().Get(context.Background(), eipName, metav1.GetOptions{}) if err != nil && k8serrors.IsNotFound(err) { deleted = true @@ -815,7 +778,12 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { ginkgo.By("10. Verify external subnet status after IptablesEIP CR deletion") afterDeleteSubnet := subnetClient.Get(externalSubnetName) - if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + verifySubnetStatusAfterEIPOperation(subnetClient, externalSubnetName, + afterDeleteSubnet.Spec.Protocol, "IptablesEIP deletion", "") + + // Verify IP count and range restoration + switch afterDeleteSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: // Verify IP count is restored framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, "V4AvailableIPs should increase by 1 after IptablesEIP deletion") @@ -833,7 +801,7 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { "V4AvailableIPs should return to initial value after IptablesEIP deletion") framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, "V4UsingIPs should return to initial value after IptablesEIP deletion") - } else if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + case apiv1.ProtocolIPv6: // Verify IP count is restored framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, "V6AvailableIPs should increase by 1 after IptablesEIP deletion") @@ -851,7 +819,7 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { "V6AvailableIPs should return to initial value after IptablesEIP deletion") framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, "V6UsingIPs should return to initial value after IptablesEIP deletion") - } else { + default: // Dual stack framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, "V4AvailableIPs should increase by 1 after IptablesEIP deletion") @@ -913,16 +881,8 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { _ = iptablesEIPClient.CreateSync(eip) ginkgo.By("3. Wait for IptablesEIP to be ready") - var eipCR *apiv1.IptablesEIP - for i := 0; i < 60; i++ { - eipCR = iptablesEIPClient.Get(eipName) - if eipCR != nil && eipCR.Status.IP != "" && eipCR.Status.Ready { - break - } - time.Sleep(1 * time.Second) - } - framework.ExpectNotNil(eipCR, "IptablesEIP CR should be created") - framework.ExpectTrue(eipCR.Status.Ready, "IptablesEIP should be ready") + eipCR := waitForIptablesEIPReady(iptablesEIPClient, eipName, 60*time.Second) + framework.ExpectNotNil(eipCR, "IptablesEIP CR should be created and ready") ginkgo.By("4. Create IptablesFIP using the EIP") fipName := "test-fip-" + framework.RandomSuffix() @@ -930,7 +890,7 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { _ = iptablesFIPClient.CreateSync(fip) ginkgo.By("5. Wait for EIP status to show it's being used by FIP") - for i := 0; i < 60; i++ { + for range 60 { eipCR = iptablesEIPClient.Get(eipName) if eipCR != nil && strings.Contains(eipCR.Status.Nat, util.FipUsingEip) { break @@ -957,7 +917,7 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { ginkgo.By("9. Wait for FIP to be deleted") fipDeleted := false - for i := 0; i < 30; i++ { + for range 30 { _, err := f.KubeOVNClientSet.KubeovnV1().IptablesFIPRules().Get(context.Background(), fipName, metav1.GetOptions{}) if err != nil && k8serrors.IsNotFound(err) { fipDeleted = true @@ -967,10 +927,15 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { } framework.ExpectTrue(fipDeleted, "FIP should be deleted") - ginkgo.By("10. Wait for EIP status.Nat to be cleared") - for i := 0; i < 30; i++ { - eipCR = iptablesEIPClient.Get(eipName) - if eipCR == nil || eipCR.Status.Nat == "" { + ginkgo.By("10. Wait for EIP status.Nat to be cleared or EIP to be deleted") + for range 30 { + eipCR, err := f.KubeOVNClientSet.KubeovnV1().IptablesEIPs().Get(context.Background(), eipName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // EIP already deleted, which is expected + break + } + framework.ExpectNoError(err, "Failed to get IptablesEIP") + if eipCR.Status.Nat == "" { break } time.Sleep(1 * time.Second) @@ -978,7 +943,7 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { ginkgo.By("11. Verify EIP is now deleted after FIP is removed") eipDeleted := false - for i := 0; i < 30; i++ { + for range 30 { _, err := f.KubeOVNClientSet.KubeovnV1().IptablesEIPs().Get(context.Background(), eipName, metav1.GetOptions{}) if err != nil && k8serrors.IsNotFound(err) { eipDeleted = true @@ -995,780 +960,6 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { }) }) -func iperf(f *framework.Framework, iperfClientPod *corev1.Pod, iperfServerEIP *apiv1.IptablesEIP) string { - ginkgo.GinkgoHelper() - - for i := range 20 { - command := fmt.Sprintf("iperf -e -p %s --reportstyle C -i 1 -c %s -t 10", iperf2Port, iperfServerEIP.Status.IP) - stdOutput, errOutput, err := framework.ExecShellInPod(context.Background(), f, iperfClientPod.Namespace, iperfClientPod.Name, command) - framework.Logf("output from exec on client pod %s (eip %s)\n", iperfClientPod.Name, iperfServerEIP.Name) - if stdOutput != "" && err == nil { - framework.Logf("output:\n%s", stdOutput) - return stdOutput - } - framework.Logf("exec %s failed err: %v, errOutput: %s, stdOutput: %s, retried %d times.", command, err, errOutput, stdOutput, i) - time.Sleep(6 * time.Second) - } - framework.ExpectNoError(errors.New("iperf failed")) - return "" -} - -func checkQos(f *framework.Framework, - vpc1Pod, vpc2Pod *corev1.Pod, vpc1EIP, vpc2EIP *apiv1.IptablesEIP, - limit int, expect bool, -) { - ginkgo.GinkgoHelper() - - if !skipIperf { - if expect { - output := iperf(f, vpc1Pod, vpc2EIP) - framework.ExpectTrue(validRateLimit(output, limit)) - output = iperf(f, vpc2Pod, vpc1EIP) - framework.ExpectTrue(validRateLimit(output, limit)) - } else { - output := iperf(f, vpc1Pod, vpc2EIP) - framework.ExpectFalse(validRateLimit(output, limit)) - output = iperf(f, vpc2Pod, vpc1EIP) - framework.ExpectFalse(validRateLimit(output, limit)) - } - } -} - -func newVPCQoSParamsInit() *qosParams { - qosParams := &qosParams{ - vpc1Name: "qos-vpc1-" + framework.RandomSuffix(), - vpc2Name: "qos-vpc2-" + framework.RandomSuffix(), - vpc1SubnetName: "qos-vpc1-subnet-" + framework.RandomSuffix(), - vpc2SubnetName: "qos-vpc2-subnet-" + framework.RandomSuffix(), - vpcNat1GwName: "qos-vpc1-gw-" + framework.RandomSuffix(), - vpcNat2GwName: "qos-vpc2-gw-" + framework.RandomSuffix(), - vpc1EIPName: "qos-vpc1-eip-" + framework.RandomSuffix(), - vpc2EIPName: "qos-vpc2-eip-" + framework.RandomSuffix(), - vpc1FIPName: "qos-vpc1-fip-" + framework.RandomSuffix(), - vpc2FIPName: "qos-vpc2-fip-" + framework.RandomSuffix(), - vpc1PodName: "qos-vpc1-pod-" + framework.RandomSuffix(), - vpc2PodName: "qos-vpc2-pod-" + framework.RandomSuffix(), - attachDefName: "qos-ovn-vpc-external-network-" + framework.RandomSuffix(), - } - qosParams.subnetProvider = fmt.Sprintf("%s.%s", qosParams.attachDefName, framework.KubeOvnNamespace) - return qosParams -} - -func getNicDefaultQoSPolicy(limit int) apiv1.QoSPolicyBandwidthLimitRules { - return apiv1.QoSPolicyBandwidthLimitRules{ - apiv1.QoSPolicyBandwidthLimitRule{ - Name: "net1-ingress", - Interface: "net1", - RateMax: strconv.Itoa(limit), - BurstMax: strconv.Itoa(limit), - Priority: 3, - Direction: apiv1.QoSDirectionIngress, - }, - apiv1.QoSPolicyBandwidthLimitRule{ - Name: "net1-egress", - Interface: "net1", - RateMax: strconv.Itoa(limit), - BurstMax: strconv.Itoa(limit), - Priority: 3, - Direction: apiv1.QoSDirectionEgress, - }, - } -} - -func getEIPQoSRule(limit int) apiv1.QoSPolicyBandwidthLimitRules { - return apiv1.QoSPolicyBandwidthLimitRules{ - apiv1.QoSPolicyBandwidthLimitRule{ - Name: "eip-ingress", - RateMax: strconv.Itoa(limit), - BurstMax: strconv.Itoa(limit), - Priority: 1, - Direction: apiv1.QoSDirectionIngress, - }, - apiv1.QoSPolicyBandwidthLimitRule{ - Name: "eip-egress", - RateMax: strconv.Itoa(limit), - BurstMax: strconv.Itoa(limit), - Priority: 1, - Direction: apiv1.QoSDirectionEgress, - }, - } -} - -func getSpecialQoSRule(limit int, ip string) apiv1.QoSPolicyBandwidthLimitRules { - return apiv1.QoSPolicyBandwidthLimitRules{ - apiv1.QoSPolicyBandwidthLimitRule{ - Name: "net1-extip-ingress", - Interface: "net1", - RateMax: strconv.Itoa(limit), - BurstMax: strconv.Itoa(limit), - Priority: 2, - Direction: apiv1.QoSDirectionIngress, - MatchType: apiv1.QoSMatchTypeIP, - MatchValue: "src " + ip + "/32", - }, - apiv1.QoSPolicyBandwidthLimitRule{ - Name: "net1-extip-egress", - Interface: "net1", - RateMax: strconv.Itoa(limit), - BurstMax: strconv.Itoa(limit), - Priority: 2, - Direction: apiv1.QoSDirectionEgress, - MatchType: apiv1.QoSMatchTypeIP, - MatchValue: "dst " + ip + "/32", - }, - } -} - -// defaultQoSCases test default qos policy= -func defaultQoSCases(f *framework.Framework, - vpcNatGwClient *framework.VpcNatGatewayClient, - podClient *framework.PodClient, - qosPolicyClient *framework.QoSPolicyClient, - vpc1Pod *corev1.Pod, - vpc2Pod *corev1.Pod, - vpc1EIP *apiv1.IptablesEIP, - vpc2EIP *apiv1.IptablesEIP, - natgwName string, -) { - ginkgo.GinkgoHelper() - - // create nic qos policy - qosPolicyName := "default-nic-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + qosPolicyName) - rules := getNicDefaultQoSPolicy(defaultNicLimit) - - qosPolicy := framework.MakeQoSPolicy(qosPolicyName, true, apiv1.QoSBindingTypeNatGw, rules) - _ = qosPolicyClient.CreateSync(qosPolicy) - - ginkgo.By("Patch natgw " + natgwName + " with qos policy " + qosPolicyName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, qosPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) - - ginkgo.By("Delete natgw pod " + natgwName + "-0") - natGwPodName := util.GenNatGwPodName(natgwName) - podClient.DeleteSync(natGwPodName) - - ginkgo.By("Wait for natgw " + natgwName + "qos rebuild") - time.Sleep(5 * time.Second) - - ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) - - ginkgo.By("Remove qos policy " + qosPolicyName + " from natgw " + natgwName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") - - ginkgo.By("Deleting qos policy " + qosPolicyName) - qosPolicyClient.DeleteSync(qosPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is not limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, false) -} - -// eipQoSCases test default qos policy -func eipQoSCases(f *framework.Framework, - eipClient *framework.IptablesEIPClient, - podClient *framework.PodClient, - qosPolicyClient *framework.QoSPolicyClient, - vpc1Pod *corev1.Pod, - vpc2Pod *corev1.Pod, - vpc1EIP *apiv1.IptablesEIP, - vpc2EIP *apiv1.IptablesEIP, - eipName string, - natgwName string, -) { - ginkgo.GinkgoHelper() - - // create eip qos policy - qosPolicyName := "eip-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + qosPolicyName) - rules := getEIPQoSRule(eipLimit) - - qosPolicy := framework.MakeQoSPolicy(qosPolicyName, false, apiv1.QoSBindingTypeEIP, rules) - qosPolicy = qosPolicyClient.CreateSync(qosPolicy) - - ginkgo.By("Patch eip " + eipName + " with qos policy " + qosPolicyName) - _ = eipClient.PatchQoSPolicySync(eipName, qosPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(eipLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, eipLimit, true) - - ginkgo.By("Update qos policy " + qosPolicyName + " with new rate limit") - - rules = getEIPQoSRule(updatedEIPLimit) - modifiedqosPolicy := qosPolicy.DeepCopy() - modifiedqosPolicy.Spec.BandwidthLimitRules = rules - qosPolicyClient.Patch(qosPolicy, modifiedqosPolicy) - qosPolicyClient.WaitToQoSReady(qosPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is changed to " + strconv.Itoa(updatedEIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, updatedEIPLimit, true) - - ginkgo.By("Delete natgw pod " + natgwName + "-0") - natGwPodName := util.GenNatGwPodName(natgwName) - podClient.DeleteSync(natGwPodName) - - ginkgo.By("Wait for natgw " + natgwName + "qos rebuid") - time.Sleep(5 * time.Second) - - ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(updatedEIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, updatedEIPLimit, true) - - newQoSPolicyName := "new-eip-qos-policy-" + framework.RandomSuffix() - newRules := getEIPQoSRule(newEIPLimit) - newQoSPolicy := framework.MakeQoSPolicy(newQoSPolicyName, false, apiv1.QoSBindingTypeEIP, newRules) - _ = qosPolicyClient.CreateSync(newQoSPolicy) - - ginkgo.By("Change qos policy of eip " + eipName + " to " + newQoSPolicyName) - _ = eipClient.PatchQoSPolicySync(eipName, newQoSPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(newEIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, newEIPLimit, true) - - ginkgo.By("Remove qos policy " + qosPolicyName + " from natgw " + eipName) - _ = eipClient.PatchQoSPolicySync(eipName, "") - - ginkgo.By("Deleting qos policy " + qosPolicyName) - qosPolicyClient.DeleteSync(qosPolicyName) - - ginkgo.By("Deleting qos policy " + newQoSPolicyName) - qosPolicyClient.DeleteSync(newQoSPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is not limited to " + strconv.Itoa(newEIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, newEIPLimit, false) -} - -// specifyingIPQoSCases test default qos policy -func specifyingIPQoSCases(f *framework.Framework, - vpcNatGwClient *framework.VpcNatGatewayClient, - qosPolicyClient *framework.QoSPolicyClient, - vpc1Pod *corev1.Pod, - vpc2Pod *corev1.Pod, - vpc1EIP *apiv1.IptablesEIP, - vpc2EIP *apiv1.IptablesEIP, - natgwName string, -) { - ginkgo.GinkgoHelper() - - // create nic qos policy - qosPolicyName := "specifying-ip-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + qosPolicyName) - - rules := getSpecialQoSRule(specificIPLimit, vpc2EIP.Status.IP) - - qosPolicy := framework.MakeQoSPolicy(qosPolicyName, true, apiv1.QoSBindingTypeNatGw, rules) - _ = qosPolicyClient.CreateSync(qosPolicy) - - ginkgo.By("Patch natgw " + natgwName + " with qos policy " + qosPolicyName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, qosPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(specificIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, specificIPLimit, true) - - ginkgo.By("Remove qos policy " + qosPolicyName + " from natgw " + natgwName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") - - ginkgo.By("Deleting qos policy " + qosPolicyName) - qosPolicyClient.DeleteSync(qosPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is not limited to " + strconv.Itoa(specificIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, specificIPLimit, false) -} - -// priorityQoSCases test qos match priority -func priorityQoSCases(f *framework.Framework, - vpcNatGwClient *framework.VpcNatGatewayClient, - eipClient *framework.IptablesEIPClient, - qosPolicyClient *framework.QoSPolicyClient, - vpc1Pod *corev1.Pod, - vpc2Pod *corev1.Pod, - vpc1EIP *apiv1.IptablesEIP, - vpc2EIP *apiv1.IptablesEIP, - natgwName string, - eipName string, -) { - ginkgo.GinkgoHelper() - - // create nic qos policy - natGwQoSPolicyName := "priority-nic-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + natGwQoSPolicyName) - // default qos policy + special qos policy - natgwRules := getNicDefaultQoSPolicy(defaultNicLimit) - natgwRules = append(natgwRules, getSpecialQoSRule(specificIPLimit, vpc2EIP.Status.IP)...) - - natgwQoSPolicy := framework.MakeQoSPolicy(natGwQoSPolicyName, true, apiv1.QoSBindingTypeNatGw, natgwRules) - _ = qosPolicyClient.CreateSync(natgwQoSPolicy) - - ginkgo.By("Patch natgw " + natgwName + " with qos policy " + natGwQoSPolicyName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, natGwQoSPolicyName) - - eipQoSPolicyName := "eip-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + eipQoSPolicyName) - eipRules := getEIPQoSRule(eipLimit) - - eipQoSPolicy := framework.MakeQoSPolicy(eipQoSPolicyName, false, apiv1.QoSBindingTypeEIP, eipRules) - _ = qosPolicyClient.CreateSync(eipQoSPolicy) - - ginkgo.By("Patch eip " + eipName + " with qos policy " + eipQoSPolicyName) - _ = eipClient.PatchQoSPolicySync(eipName, eipQoSPolicyName) - - // match qos of priority 1 - ginkgo.By("Check qos to match priority 1 is limited to " + strconv.Itoa(eipLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, eipLimit, true) - - ginkgo.By("Remove qos policy " + eipQoSPolicyName + " from natgw " + eipName) - _ = eipClient.PatchQoSPolicySync(eipName, "") - - ginkgo.By("Deleting qos policy " + eipQoSPolicyName) - qosPolicyClient.DeleteSync(eipQoSPolicyName) - - // match qos of priority 2 - ginkgo.By("Check qos to match priority 2 is limited to " + strconv.Itoa(specificIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, specificIPLimit, true) - - // change qos policy of natgw - newNatGwQoSPolicyName := "new-priority-nic-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + newNatGwQoSPolicyName) - newNatgwRules := getNicDefaultQoSPolicy(defaultNicLimit) - - newNatgwQoSPolicy := framework.MakeQoSPolicy(newNatGwQoSPolicyName, true, apiv1.QoSBindingTypeNatGw, newNatgwRules) - _ = qosPolicyClient.CreateSync(newNatgwQoSPolicy) - - ginkgo.By("Change qos policy of natgw " + natgwName + " to " + newNatGwQoSPolicyName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, newNatGwQoSPolicyName) - - // match qos of priority 3 - ginkgo.By("Check qos to match priority 3 is limited to " + strconv.Itoa(specificIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) - - ginkgo.By("Remove qos policy " + natGwQoSPolicyName + " from natgw " + natgwName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") - - ginkgo.By("Deleting qos policy " + natGwQoSPolicyName) - qosPolicyClient.DeleteSync(natGwQoSPolicyName) - - ginkgo.By("Deleting qos policy " + newNatGwQoSPolicyName) - qosPolicyClient.DeleteSync(newNatGwQoSPolicyName) - - ginkgo.By("Check qos " + natGwQoSPolicyName + " is not limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, false) -} - -func createNatGwAndSetQosCases(f *framework.Framework, - vpcNatGwClient *framework.VpcNatGatewayClient, - ipClient *framework.IPClient, - eipClient *framework.IptablesEIPClient, - fipClient *framework.IptablesFIPClient, - subnetClient *framework.SubnetClient, - qosPolicyClient *framework.QoSPolicyClient, - vpc1Pod *corev1.Pod, - vpc2Pod *corev1.Pod, - vpc2EIP *apiv1.IptablesEIP, - natgwName string, - eipName string, - fipName string, - vpcName string, - overlaySubnetName string, - lanIP string, - attachDefName string, -) { - ginkgo.GinkgoHelper() - - // delete fip - ginkgo.By("Deleting fip " + fipName) - fipClient.DeleteSync(fipName) - - ginkgo.By("Deleting eip " + eipName) - eipClient.DeleteSync(eipName) - - // the only pod for vpc nat gateway - vpcNatGw1PodName := util.GenNatGwPodName(natgwName) - - // delete vpc nat gw statefulset remaining ip for eth0 and net2 - ginkgo.By("Deleting custom vpc nat gw " + natgwName) - vpcNatGwClient.DeleteSync(natgwName) - - overlaySubnet1 := subnetClient.Get(overlaySubnetName) - macvlanSubnet := subnetClient.Get(attachDefName) - eth0IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, overlaySubnet1.Spec.Provider) - net1IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, macvlanSubnet.Spec.Provider) - ginkgo.By("Deleting vpc nat gw eth0 ip " + eth0IpName) - ipClient.DeleteSync(eth0IpName) - ginkgo.By("Deleting vpc nat gw net1 ip " + net1IpName) - ipClient.DeleteSync(net1IpName) - - natgwQoSPolicyName := "default-nic-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + natgwQoSPolicyName) - rules := getNicDefaultQoSPolicy(defaultNicLimit) - - qosPolicy := framework.MakeQoSPolicy(natgwQoSPolicyName, true, apiv1.QoSBindingTypeNatGw, rules) - _ = qosPolicyClient.CreateSync(qosPolicy) - - ginkgo.By("Creating custom vpc nat gw") - vpcNatGw := framework.MakeVpcNatGateway(natgwName, vpcName, overlaySubnetName, lanIP, attachDefName, natgwQoSPolicyName) - _ = vpcNatGwClient.CreateSync(vpcNatGw, f.ClientSet) - - eipQoSPolicyName := "eip-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + eipQoSPolicyName) - rules = getEIPQoSRule(eipLimit) - - eipQoSPolicy := framework.MakeQoSPolicy(eipQoSPolicyName, false, apiv1.QoSBindingTypeEIP, rules) - _ = qosPolicyClient.CreateSync(eipQoSPolicy) - - ginkgo.By("Creating eip " + eipName) - vpc1EIP := framework.MakeIptablesEIP(eipName, "", "", "", natgwName, attachDefName, eipQoSPolicyName) - vpc1EIP = eipClient.CreateSync(vpc1EIP) - - ginkgo.By("Creating fip " + fipName) - fip := framework.MakeIptablesFIPRule(fipName, eipName, vpc1Pod.Status.PodIP) - _ = fipClient.CreateSync(fip) - - ginkgo.By("Check qos " + eipQoSPolicyName + " is limited to " + strconv.Itoa(eipLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, eipLimit, true) - - ginkgo.By("Remove qos policy " + eipQoSPolicyName + " from natgw " + natgwName) - _ = eipClient.PatchQoSPolicySync(eipName, "") - - ginkgo.By("Check qos " + natgwQoSPolicyName + " is limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) - - ginkgo.By("Remove qos policy " + natgwQoSPolicyName + " from natgw " + natgwName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") - - ginkgo.By("Check qos " + natgwQoSPolicyName + " is not limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, false) - - ginkgo.By("Deleting qos policy " + natgwQoSPolicyName) - qosPolicyClient.DeleteSync(natgwQoSPolicyName) - - ginkgo.By("Deleting qos policy " + eipQoSPolicyName) - qosPolicyClient.DeleteSync(eipQoSPolicyName) -} - -func validRateLimit(text string, limit int) bool { - maxValue := float64(limit) * 1024 * 1024 * 1.2 - minValue := float64(limit) * 1024 * 1024 * 0.8 - lines := strings.SplitSeq(text, "\n") - for line := range lines { - if line == "" { - continue - } - fields := strings.Split(line, ",") - number, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - continue - } - if v := float64(number); v >= minValue && v <= maxValue { - return true - } - } - return false -} - -var _ = framework.Describe("[group:qos-policy]", func() { - f := framework.NewDefaultFramework("qos-policy") - - var skip bool - var cs clientset.Interface - var attachNetClient *framework.NetworkAttachmentDefinitionClient - var clusterName string - var vpcClient *framework.VpcClient - var vpcNatGwClient *framework.VpcNatGatewayClient - var subnetClient *framework.SubnetClient - var podClient *framework.PodClient - var ipClient *framework.IPClient - var iptablesEIPClient *framework.IptablesEIPClient - var iptablesFIPClient *framework.IptablesFIPClient - var qosPolicyClient *framework.QoSPolicyClient - - var net1NicName string - var dockerExtNetName string - - // docker network - var dockerExtNetNetwork *dockernetwork.Inspect - - var vpcQosParams *qosParams - var vpc1Pod *corev1.Pod - var vpc2Pod *corev1.Pod - var vpc1EIP *apiv1.IptablesEIP - var vpc2EIP *apiv1.IptablesEIP - var vpc1FIP *apiv1.IptablesFIPRule - var vpc2FIP *apiv1.IptablesFIPRule - - var lanIP string - var overlaySubnetV4Cidr string - var overlaySubnetV4Gw string - var eth0Exist, net1Exist bool - var annotations1 map[string]string - var annotations2 map[string]string - var iperfServerCmd []string - - ginkgo.BeforeEach(func() { - vpcQosParams = newVPCQoSParamsInit() - - dockerExtNetName = "kube-ovn-qos-" + framework.RandomSuffix() - - vpcQosParams.vpc1SubnetName = "qos-vpc1-subnet-" + framework.RandomSuffix() - vpcQosParams.vpc2SubnetName = "qos-vpc2-subnet-" + framework.RandomSuffix() - - vpcQosParams.vpcNat1GwName = "qos-gw1-" + framework.RandomSuffix() - vpcQosParams.vpcNat2GwName = "qos-gw2-" + framework.RandomSuffix() - - vpcQosParams.vpc1EIPName = "qos-vpc1-eip-" + framework.RandomSuffix() - vpcQosParams.vpc2EIPName = "qos-vpc2-eip-" + framework.RandomSuffix() - - vpcQosParams.vpc1FIPName = "qos-vpc1-fip-" + framework.RandomSuffix() - vpcQosParams.vpc2FIPName = "qos-vpc2-fip-" + framework.RandomSuffix() - - vpcQosParams.vpc1PodName = "qos-vpc1-pod-" + framework.RandomSuffix() - vpcQosParams.vpc2PodName = "qos-vpc2-pod-" + framework.RandomSuffix() - - vpcQosParams.attachDefName = "qos-ovn-vpc-external-network-" + framework.RandomSuffix() - vpcQosParams.subnetProvider = fmt.Sprintf("%s.%s", vpcQosParams.attachDefName, framework.KubeOvnNamespace) - - cs = f.ClientSet - podClient = f.PodClient() - attachNetClient = f.NetworkAttachmentDefinitionClientNS(framework.KubeOvnNamespace) - subnetClient = f.SubnetClient() - vpcClient = f.VpcClient() - vpcNatGwClient = f.VpcNatGatewayClient() - iptablesEIPClient = f.IptablesEIPClient() - ipClient = f.IPClient() - iptablesFIPClient = f.IptablesFIPClient() - qosPolicyClient = f.QoSPolicyClient() - - if skip { - ginkgo.Skip("underlay spec only runs on kind clusters") - } - - if clusterName == "" { - ginkgo.By("Getting k8s nodes") - k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) - framework.ExpectNoError(err) - - cluster, ok := kind.IsKindProvided(k8sNodes.Items[0].Spec.ProviderID) - if !ok { - skip = true - ginkgo.Skip("underlay spec only runs on kind clusters") - } - clusterName = cluster - } - - ginkgo.By("Ensuring docker network " + dockerExtNetName + " exists") - network, err := docker.NetworkCreate(dockerExtNetName, true, true) - framework.ExpectNoError(err, "creating docker network "+dockerExtNetName) - dockerExtNetNetwork = network - - ginkgo.By("Getting kind nodes") - nodes, err := kind.ListNodes(clusterName, "") - framework.ExpectNoError(err, "getting nodes in kind cluster") - framework.ExpectNotEmpty(nodes) - - ginkgo.By("Connecting nodes to the docker network") - err = kind.NetworkConnect(dockerExtNetNetwork.ID, nodes) - framework.ExpectNoError(err, "connecting nodes to network "+dockerExtNetName) - - ginkgo.By("Getting node links that belong to the docker network") - nodes, err = kind.ListNodes(clusterName, "") - framework.ExpectNoError(err, "getting nodes in kind cluster") - - ginkgo.By("Validating node links") - network1, err := docker.NetworkInspect(dockerExtNetName) - framework.ExpectNoError(err) - for _, node := range nodes { - links, err := node.ListLinks() - framework.ExpectNoError(err, "failed to list links on node %s: %v", node.Name(), err) - net1Mac := network1.Containers[node.ID].MacAddress - for _, link := range links { - ginkgo.By("exist node nic " + link.IfName) - if link.IfName == "eth0" { - eth0Exist = true - } - if link.Address == net1Mac.String() { - net1NicName = link.IfName - net1Exist = true - } - } - framework.ExpectTrue(eth0Exist) - framework.ExpectTrue(net1Exist) - } - setupNetworkAttachmentDefinition( - f, dockerExtNetNetwork, attachNetClient, - subnetClient, vpcQosParams.attachDefName, net1NicName, vpcQosParams.subnetProvider, dockerExtNetName) - }) - - ginkgo.AfterEach(func() { - ginkgo.By("Deleting macvlan underlay subnet " + vpcQosParams.attachDefName) - subnetClient.DeleteSync(vpcQosParams.attachDefName) - - // delete net1 attachment definition - ginkgo.By("Deleting nad " + vpcQosParams.attachDefName) - attachNetClient.Delete(vpcQosParams.attachDefName) - - ginkgo.By("Getting nodes") - nodes, err := kind.ListNodes(clusterName, "") - framework.ExpectNoError(err, "getting nodes in cluster") - - if dockerExtNetNetwork != nil { - ginkgo.By("Disconnecting nodes from the docker network") - err = kind.NetworkDisconnect(dockerExtNetNetwork.ID, nodes) - framework.ExpectNoError(err, "disconnecting nodes from network "+dockerExtNetName) - ginkgo.By("Deleting docker network " + dockerExtNetName + " exists") - err := docker.NetworkRemove(dockerExtNetNetwork.ID) - framework.ExpectNoError(err, "deleting docker network "+dockerExtNetName) - } - }) - - _ = framework.Describe("vpc qos", func() { - ginkgo.BeforeEach(func() { - iperfServerCmd = []string{"iperf", "-s", "-i", "1", "-p", iperf2Port} - overlaySubnetV4Cidr = "10.0.0.0/24" - overlaySubnetV4Gw = "10.0.0.1" - lanIP = "10.0.0.254" - natgwQoS := "" - setupVpcNatGwTestEnvironment( - f, dockerExtNetNetwork, attachNetClient, - subnetClient, vpcClient, vpcNatGwClient, - vpcQosParams.vpc1Name, vpcQosParams.vpc1SubnetName, vpcQosParams.vpcNat1GwName, - natgwQoS, overlaySubnetV4Cidr, overlaySubnetV4Gw, lanIP, - dockerExtNetName, vpcQosParams.attachDefName, net1NicName, - vpcQosParams.subnetProvider, - true, - ) - annotations1 = map[string]string{ - util.LogicalSwitchAnnotation: vpcQosParams.vpc1SubnetName, - } - ginkgo.By("Creating pod " + vpcQosParams.vpc1PodName) - vpc1Pod = framework.MakePod(f.Namespace.Name, vpcQosParams.vpc1PodName, nil, annotations1, framework.AgnhostImage, iperfServerCmd, nil) - vpc1Pod = podClient.CreateSync(vpc1Pod) - - ginkgo.By("Creating eip " + vpcQosParams.vpc1EIPName) - vpc1EIP = framework.MakeIptablesEIP(vpcQosParams.vpc1EIPName, "", "", "", vpcQosParams.vpcNat1GwName, vpcQosParams.attachDefName, "") - vpc1EIP = iptablesEIPClient.CreateSync(vpc1EIP) - - ginkgo.By("Creating fip " + vpcQosParams.vpc1FIPName) - vpc1FIP = framework.MakeIptablesFIPRule(vpcQosParams.vpc1FIPName, vpcQosParams.vpc1EIPName, vpc1Pod.Status.PodIP) - _ = iptablesFIPClient.CreateSync(vpc1FIP) - - setupVpcNatGwTestEnvironment( - f, dockerExtNetNetwork, attachNetClient, - subnetClient, vpcClient, vpcNatGwClient, - vpcQosParams.vpc2Name, vpcQosParams.vpc2SubnetName, vpcQosParams.vpcNat2GwName, - natgwQoS, overlaySubnetV4Cidr, overlaySubnetV4Gw, lanIP, - dockerExtNetName, vpcQosParams.attachDefName, net1NicName, - vpcQosParams.subnetProvider, - true, - ) - - annotations2 = map[string]string{ - util.LogicalSwitchAnnotation: vpcQosParams.vpc2SubnetName, - } - - ginkgo.By("Creating pod " + vpcQosParams.vpc2PodName) - vpc2Pod = framework.MakePod(f.Namespace.Name, vpcQosParams.vpc2PodName, nil, annotations2, framework.AgnhostImage, iperfServerCmd, nil) - vpc2Pod = podClient.CreateSync(vpc2Pod) - - ginkgo.By("Creating eip " + vpcQosParams.vpc2EIPName) - vpc2EIP = framework.MakeIptablesEIP(vpcQosParams.vpc2EIPName, "", "", "", vpcQosParams.vpcNat2GwName, vpcQosParams.attachDefName, "") - vpc2EIP = iptablesEIPClient.CreateSync(vpc2EIP) - - ginkgo.By("Creating fip " + vpcQosParams.vpc2FIPName) - vpc2FIP = framework.MakeIptablesFIPRule(vpcQosParams.vpc2FIPName, vpcQosParams.vpc2EIPName, vpc2Pod.Status.PodIP) - _ = iptablesFIPClient.CreateSync(vpc2FIP) - }) - ginkgo.AfterEach(func() { - ginkgo.By("Deleting fip " + vpcQosParams.vpc1FIPName) - iptablesFIPClient.DeleteSync(vpcQosParams.vpc1FIPName) - - ginkgo.By("Deleting fip " + vpcQosParams.vpc2FIPName) - iptablesFIPClient.DeleteSync(vpcQosParams.vpc2FIPName) - - ginkgo.By("Deleting eip " + vpcQosParams.vpc1EIPName) - iptablesEIPClient.DeleteSync(vpcQosParams.vpc1EIPName) - - ginkgo.By("Deleting eip " + vpcQosParams.vpc2EIPName) - iptablesEIPClient.DeleteSync(vpcQosParams.vpc2EIPName) - - ginkgo.By("Deleting pod " + vpcQosParams.vpc1PodName) - podClient.DeleteSync(vpcQosParams.vpc1PodName) - - ginkgo.By("Deleting pod " + vpcQosParams.vpc2PodName) - podClient.DeleteSync(vpcQosParams.vpc2PodName) - - ginkgo.By("Deleting custom vpc nat gw " + vpcQosParams.vpcNat1GwName) - vpcNatGwClient.DeleteSync(vpcQosParams.vpcNat1GwName) - - ginkgo.By("Deleting custom vpc nat gw " + vpcQosParams.vpcNat2GwName) - vpcNatGwClient.DeleteSync(vpcQosParams.vpcNat2GwName) - - // the only pod for vpc nat gateway - vpcNatGw1PodName := util.GenNatGwPodName(vpcQosParams.vpcNat1GwName) - - // delete vpc nat gw statefulset remaining ip for eth0 and net2 - overlaySubnet1 := subnetClient.Get(vpcQosParams.vpc1SubnetName) - macvlanSubnet := subnetClient.Get(vpcQosParams.attachDefName) - eth0IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, overlaySubnet1.Spec.Provider) - net1IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, macvlanSubnet.Spec.Provider) - ginkgo.By("Deleting vpc nat gw eth0 ip " + eth0IpName) - ipClient.DeleteSync(eth0IpName) - ginkgo.By("Deleting vpc nat gw net1 ip " + net1IpName) - ipClient.DeleteSync(net1IpName) - ginkgo.By("Deleting overlay subnet " + vpcQosParams.vpc1SubnetName) - subnetClient.DeleteSync(vpcQosParams.vpc1SubnetName) - - ginkgo.By("Getting overlay subnet " + vpcQosParams.vpc2SubnetName) - overlaySubnet2 := subnetClient.Get(vpcQosParams.vpc2SubnetName) - - vpcNatGw2PodName := util.GenNatGwPodName(vpcQosParams.vpcNat2GwName) - eth0IpName = ovs.PodNameToPortName(vpcNatGw2PodName, framework.KubeOvnNamespace, overlaySubnet2.Spec.Provider) - net1IpName = ovs.PodNameToPortName(vpcNatGw2PodName, framework.KubeOvnNamespace, macvlanSubnet.Spec.Provider) - ginkgo.By("Deleting vpc nat gw eth0 ip " + eth0IpName) - ipClient.DeleteSync(eth0IpName) - ginkgo.By("Deleting vpc nat gw net1 ip " + net1IpName) - ipClient.DeleteSync(net1IpName) - ginkgo.By("Deleting overlay subnet " + vpcQosParams.vpc2SubnetName) - subnetClient.DeleteSync(vpcQosParams.vpc2SubnetName) - - ginkgo.By("Deleting custom vpc " + vpcQosParams.vpc1Name) - vpcClient.DeleteSync(vpcQosParams.vpc1Name) - - ginkgo.By("Deleting custom vpc " + vpcQosParams.vpc2Name) - vpcClient.DeleteSync(vpcQosParams.vpc2Name) - }) - framework.ConformanceIt("default nic qos", func() { - // case 1: set qos policy for natgw - // case 2: rebuild qos when natgw pod restart - defaultQoSCases(f, vpcNatGwClient, podClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpcNat1GwName) - }) - framework.ConformanceIt("eip qos", func() { - // case 1: set qos policy for eip - // case 2: update qos policy for eip - // case 3: change qos policy of eip - // case 4: rebuild qos when natgw pod restart - eipQoSCases(f, iptablesEIPClient, podClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpc1EIPName, vpcQosParams.vpcNat1GwName) - }) - framework.ConformanceIt("specifying ip qos", func() { - // case 1: set specific ip qos policy for natgw - specifyingIPQoSCases(f, vpcNatGwClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpcNat1GwName) - }) - framework.ConformanceIt("qos priority matching", func() { - // case 1: test qos match priority - // case 2: change qos policy of natgw - priorityQoSCases(f, vpcNatGwClient, iptablesEIPClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpcNat1GwName, vpcQosParams.vpc1EIPName) - }) - framework.ConformanceIt("create resource with qos policy", func() { - // case 1: test qos when create natgw with qos policy - // case 2: test qos when create eip with qos policy - createNatGwAndSetQosCases(f, - vpcNatGwClient, ipClient, iptablesEIPClient, iptablesFIPClient, - subnetClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc2EIP, vpcQosParams.vpcNat1GwName, - vpcQosParams.vpc1EIPName, vpcQosParams.vpc1FIPName, vpcQosParams.vpc1Name, - vpcQosParams.vpc1SubnetName, lanIP, vpcQosParams.attachDefName) - }) - }) -}) - func init() { klog.SetOutput(ginkgo.GinkgoWriter) diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 1ed4cfbd515..dc8f32eafeb 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -11,6 +11,7 @@ import ( dockernetwork "github.com/moby/moby/api/types/network" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -1120,7 +1121,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectNotEmpty(eipCR.Status.V4Ip, "OvnEip should have V4 IP assigned") ginkgo.By("Step 3: Verifying finalizer is added to OvnEip") - framework.WaitUntil(time.Minute, func(_ context.Context) (bool, error) { + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { eipCR = ovnEipClient.Get(eipName) return eipCR != nil && len(eipCR.Finalizers) > 0, nil }, "OvnEip should have finalizer added") @@ -1159,7 +1160,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { ginkgo.By("Step 5: Deleting OvnEip and verifying cleanup") ovnEipClient.DeleteSync(eipName) - framework.WaitUntil(time.Minute, func(_ context.Context) (bool, error) { + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { _, err := f.KubeOVNClientSet.KubeovnV1().OvnEips().Get(context.Background(), eipName, metav1.GetOptions{}) return k8serrors.IsNotFound(err), nil }, "OvnEip should be deleted") @@ -1251,7 +1252,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { _ = ovnFipClient.CreateSync(fip) ginkgo.By("Step 5: Verifying EIP Status.Nat shows FIP usage") - framework.WaitUntil(time.Minute, func(_ context.Context) (bool, error) { + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { eipCR = ovnEipClient.Get(eipName) return eipCR != nil && strings.Contains(eipCR.Status.Nat, util.FipUsingEip), nil }, "EIP Status.Nat should contain 'fip'") @@ -1271,13 +1272,13 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { ginkgo.By("Step 8: Deleting FIP to unblock EIP deletion") ovnFipClient.DeleteSync(fipName) - framework.WaitUntil(time.Minute, func(_ context.Context) (bool, error) { + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { _, err := f.KubeOVNClientSet.KubeovnV1().OvnFips().Get(context.Background(), fipName, metav1.GetOptions{}) return k8serrors.IsNotFound(err), nil }, "FIP should be deleted") ginkgo.By("Step 9: Verifying EIP is now deleted after FIP removal") - framework.WaitUntil(time.Minute, func(_ context.Context) (bool, error) { + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { _, err := f.KubeOVNClientSet.KubeovnV1().OvnEips().Get(context.Background(), eipName, metav1.GetOptions{}) return k8serrors.IsNotFound(err), nil }, "EIP should be deleted after FIP is removed") diff --git a/test/e2e/vip/e2e_test.go b/test/e2e/vip/e2e_test.go index 1879d3148fd..a0ec0b1ed53 100644 --- a/test/e2e/vip/e2e_test.go +++ b/test/e2e/vip/e2e_test.go @@ -226,7 +226,8 @@ var _ = framework.Describe("[group:vip]", func() { ginkgo.By("4. Verify subnet status after VIP creation") afterCreateSubnet := subnetClient.Get(subnetName) - if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + switch afterCreateSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: // Verify IP count changed framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, "V4AvailableIPs should decrease by 1 after VIP creation") @@ -243,7 +244,7 @@ var _ = framework.Describe("[group:vip]", func() { vipIP := testVip.Status.V4ip framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V4UsingIPRange, vipIP), "VIP IP %s should be in V4UsingIPRange %s", vipIP, afterCreateSubnet.Status.V4UsingIPRange) - } else if afterCreateSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + case apiv1.ProtocolIPv6: // Verify IP count changed framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, "V6AvailableIPs should decrease by 1 after VIP creation") @@ -260,7 +261,7 @@ var _ = framework.Describe("[group:vip]", func() { vipIP := testVip.Status.V6ip framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V6UsingIPRange, vipIP), "VIP IP %s should be in V6UsingIPRange %s", vipIP, afterCreateSubnet.Status.V6UsingIPRange) - } else { + default: // Dual stack framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, "V4AvailableIPs should decrease by 1 after VIP creation") @@ -299,7 +300,8 @@ var _ = framework.Describe("[group:vip]", func() { ginkgo.By("7. Verify subnet status after VIP deletion") afterDeleteSubnet := subnetClient.Get(subnetName) - if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { + switch afterDeleteSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: // Verify IP count is restored framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, "V4AvailableIPs should increase by 1 after VIP deletion") @@ -317,7 +319,7 @@ var _ = framework.Describe("[group:vip]", func() { "V4AvailableIPs should return to initial value after VIP deletion") framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, "V4UsingIPs should return to initial value after VIP deletion") - } else if afterDeleteSubnet.Spec.Protocol == apiv1.ProtocolIPv6 { + case apiv1.ProtocolIPv6: // Verify IP count is restored framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, "V6AvailableIPs should increase by 1 after VIP deletion") @@ -335,7 +337,7 @@ var _ = framework.Describe("[group:vip]", func() { "V6AvailableIPs should return to initial value after VIP deletion") framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, "V6UsingIPs should return to initial value after VIP deletion") - } else { + default: // Dual stack framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, "V4AvailableIPs should increase by 1 after VIP deletion") @@ -377,7 +379,7 @@ var _ = framework.Describe("[group:vip]", func() { // Wait for finalizer to be added ginkgo.By("Waiting for VIP finalizer to be added") - for i := 0; i < 10; i++ { + for range 10 { countingVip = vipClient.Get(countingVipName) if len(countingVip.Finalizers) > 0 { break From 59f948415dd81970260af0f35b26b38fb054f590 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Fri, 12 Dec 2025 20:32:01 +0800 Subject: [PATCH 04/30] fix iptable eip qos e2e Signed-off-by: zbb88888 --- test/e2e/iptables-eip-qos/e2e_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/e2e/iptables-eip-qos/e2e_test.go b/test/e2e/iptables-eip-qos/e2e_test.go index 0e82db9bc6a..b1cb5341106 100644 --- a/test/e2e/iptables-eip-qos/e2e_test.go +++ b/test/e2e/iptables-eip-qos/e2e_test.go @@ -130,11 +130,13 @@ func setupNetworkAttachmentDefinition( } // Check if subnet already exists - existingSubnet := subnetClient.Get(externalNetworkName) - if existingSubnet == nil { + _, err = subnetClient.SubnetInterface.Get(context.TODO(), externalNetworkName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { // Subnet doesn't exist, create it macvlanSubnet := framework.MakeSubnet(externalNetworkName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", provider, excludeIPs, nil, nil) _ = subnetClient.CreateSync(macvlanSubnet) + } else { + framework.ExpectNoError(err, "getting subnet "+externalNetworkName) } } From bcbd1d0fa90d16f6cb2ab6bf1c387f30f58cb4bf Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Fri, 12 Dec 2025 20:43:54 +0800 Subject: [PATCH 05/30] =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=E5=B9=B6?= =?UTF-8?q?=E6=9B=B4=E6=96=B0iptables=20VPC=20NAT=E7=BD=91=E5=85=B3?= =?UTF-8?q?=E7=9A=84E2E=E6=B5=8B=E8=AF=95=E7=9B=AE=E6=A0=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- makefiles/e2e.mk | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/makefiles/e2e.mk b/makefiles/e2e.mk index 8e910be8714..8c623cd97d2 100644 --- a/makefiles/e2e.mk +++ b/makefiles/e2e.mk @@ -215,8 +215,8 @@ vpc-egress-gateway-e2e: ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v --timeout=30m \ --focus=CNI:Kube-OVN ./test/e2e/vpc-egress-gateway/vpc-egress-gateway.test -- $(TEST_BIN_ARGS) -.PHONY: iptables-vpc-nat-gw-conformance-e2e -iptables-vpc-nat-gw-conformance-e2e: +.PHONY: iptables-eip-conformance-e2e +iptables-eip-conformance-e2e: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw E2E_BRANCH=$(E2E_BRANCH) \ E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ @@ -233,6 +233,13 @@ iptables-eip-qos-conformance-e2e: ginkgo $(GINKGO_OUTPUT_OPT) --randomize-all -v \ --focus=CNI:Kube-OVN ./test/e2e/iptables-eip-qos/iptables-eip-qos.test -- $(TEST_BIN_ARGS) +.PHONY: iptables-vpc-nat-gw-conformance-e2e +iptables-vpc-nat-gw-conformance-e2e: + $(MAKE) iptables-eip-conformance-e2e + $(MAKE) iptables-eip-qos-conformance-e2e + + + .PHONY: ovn-vpc-nat-gw-conformance-e2e ovn-vpc-nat-gw-conformance-e2e: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ovn-vpc-nat-gw From a3e5913fef9833cc0c5ac959f4fc252ddce4b420 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Sat, 13 Dec 2025 01:05:03 +0800 Subject: [PATCH 06/30] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E6=97=A0=E9=BB=98=E8=AE=A4EIP=E7=9A=84VPC=20NAT=E7=BD=91?= =?UTF-8?q?=E5=85=B3=E5=88=9B=E5=BB=BA=E5=8A=9F=E8=83=BD=EF=BC=8C=E5=B9=B6?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/e2e/iptables-vpc-nat-gw/e2e_test.go | 138 ++++++++++++++++++++++- 1 file changed, 137 insertions(+), 1 deletion(-) diff --git a/test/e2e/iptables-vpc-nat-gw/e2e_test.go b/test/e2e/iptables-vpc-nat-gw/e2e_test.go index 2e1df53e624..2668da96fef 100644 --- a/test/e2e/iptables-vpc-nat-gw/e2e_test.go +++ b/test/e2e/iptables-vpc-nat-gw/e2e_test.go @@ -431,7 +431,7 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { cm, err := f.ClientSet.CoreV1().ConfigMaps(framework.KubeOvnNamespace).Get(context.Background(), vpcNatConfigName, metav1.GetOptions{}) framework.ExpectNoError(err) oldImage := cm.Data["image"] - cm.Data["image"] = "docker.io/kubeovn/vpc-nat-gateway:v1.12.18" + cm.Data["image"] = "docker.io/kubeovn/vpc-nat-gateway:v1.14.19" cm, err = f.ClientSet.CoreV1().ConfigMaps(framework.KubeOvnNamespace).Update(context.Background(), cm, metav1.UpdateOptions{}) framework.ExpectNoError(err) time.Sleep(3 * time.Second) @@ -958,6 +958,142 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { ginkgo.By("13. Test completed: IptablesEIP finalizer correctly blocks deletion when used by NAT rules") }) + + framework.ConformanceIt("Test VPC NAT Gateway with no IPAM NAD and noDefaultEIP", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + overlaySubnetV4Cidr := "10.0.5.0/24" + overlaySubnetV4Gw := "10.0.5.1" + lanIP := "10.0.5.254" + natgwQoS := "" + noIPAMNadName := "no-ipam-nad-" + framework.RandomSuffix() + noIPAMProvider := fmt.Sprintf("%s.%s", noIPAMNadName, framework.KubeOvnNamespace) + + ginkgo.By("1. Setting up NAD without IPAM and creating subnet using standard flow") + // Create NAD without IPAM section + ginkgo.By("Getting docker network " + dockerExtNet1Name) + network, err := docker.NetworkInspect(dockerExtNet1Name) + framework.ExpectNoError(err, "getting docker network "+dockerExtNet1Name) + + ginkgo.By("Creating network attachment definition without IPAM " + noIPAMNadName) + // NAD config without ipam - this is the key difference + attachConf := fmt.Sprintf(`{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "%s", + "mode": "bridge" + }`, net1NicName) + + attachNet := framework.MakeNetworkAttachmentDefinition(noIPAMNadName, framework.KubeOvnNamespace, attachConf) + nad := attachNetClient.Create(attachNet) + ginkgo.By("Got network attachment definition " + nad.Name) + + ginkgo.By("Creating underlay macvlan subnet " + noIPAMNadName) + var cidrV4, cidrV6, gatewayV4, gatewayV6 string + for _, config := range dockerExtNet1Network.IPAM.Config { + switch util.CheckProtocol(config.Subnet.Addr().String()) { + case apiv1.ProtocolIPv4: + if f.HasIPv4() { + cidrV4 = config.Subnet.String() + gatewayV4 = config.Gateway.String() + } + case apiv1.ProtocolIPv6: + if f.HasIPv6() { + cidrV6 = config.Subnet.String() + gatewayV6 = config.Gateway.String() + } + } + } + cidr := make([]string, 0, 2) + gateway := make([]string, 0, 2) + if f.HasIPv4() { + cidr = append(cidr, cidrV4) + gateway = append(gateway, gatewayV4) + } + if f.HasIPv6() { + cidr = append(cidr, cidrV6) + gateway = append(gateway, gatewayV6) + } + excludeIPs := make([]string, 0, len(network.Containers)*2) + for _, container := range network.Containers { + if container.IPv4Address.IsValid() && f.HasIPv4() { + excludeIPs = append(excludeIPs, container.IPv4Address.Addr().String()) + } + if container.IPv6Address.IsValid() && f.HasIPv6() { + excludeIPs = append(excludeIPs, container.IPv6Address.Addr().String()) + } + } + macvlanSubnet := framework.MakeSubnet(noIPAMNadName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", noIPAMProvider, excludeIPs, nil, nil) + _ = subnetClient.CreateSync(macvlanSubnet) + + ginkgo.By("2. Creating custom vpc " + vpcName) + vpc := framework.MakeVpc(vpcName, lanIP, false, false, nil) + _ = vpcClient.CreateSync(vpc) + + ginkgo.By("3. Creating custom overlay subnet " + overlaySubnetName) + overlaySubnet := framework.MakeSubnet(overlaySubnetName, "", overlaySubnetV4Cidr, overlaySubnetV4Gw, vpcName, "", nil, nil, nil) + _ = subnetClient.CreateSync(overlaySubnet) + + ginkgo.By("4. Creating custom vpc nat gw with noDefaultEIP=true " + vpcNatGwName) + vpcNatGw := framework.MakeVpcNatGatewayWithNoDefaultEIP(vpcNatGwName, vpcName, overlaySubnetName, lanIP, noIPAMNadName, natgwQoS, true) + _ = vpcNatGwClient.CreateSync(vpcNatGw, f.ClientSet) + + ginkgo.By("5. Verifying VPC NAT Gateway is created") + createdGw := vpcNatGwClient.Get(vpcNatGwName) + framework.ExpectNotNil(createdGw, "VPC NAT Gateway should be created") + framework.ExpectTrue(createdGw.Spec.NoDefaultEIP, "noDefaultEIP should be true") + + ginkgo.By("6. Verifying no default EIP is created") + time.Sleep(10 * time.Second) + eips, err := f.KubeOVNClientSet.KubeovnV1().IptablesEIPs().List(context.Background(), metav1.ListOptions{}) + framework.ExpectNoError(err, "Failed to list IptablesEIPs") + hasDefaultEIP := false + for _, eip := range eips.Items { + if eip.Spec.NatGwDp == vpcNatGwName { + hasDefaultEIP = true + break + } + } + framework.ExpectFalse(hasDefaultEIP, "No default EIP should be created when noDefaultEIP is true") + + ginkgo.By("7. Testing manual EIP creation") + eipName := "manual-eip-" + framework.RandomSuffix() + eip := framework.MakeIptablesEIP(eipName, "", "", "", vpcNatGwName, noIPAMNadName, "") + _ = iptablesEIPClient.CreateSync(eip) + + ginkgo.By("8. Verifying manually created EIP") + eipCR := waitForIptablesEIPReady(iptablesEIPClient, eipName, 60*time.Second) + framework.ExpectNotNil(eipCR, "Manual EIP should be created successfully") + framework.ExpectNotEmpty(eipCR.Status.IP, "Manual EIP should have IP assigned") + + ginkgo.By("9. Testing VIP and FIP with manual EIP") + vipName := "test-vip-no-ipam-" + framework.RandomSuffix() + vip := framework.MakeVip(f.Namespace.Name, vipName, overlaySubnetName, "", "", "") + _ = vipClient.CreateSync(vip) + vip = vipClient.Get(vipName) + + fipName := "test-fip-no-ipam-" + framework.RandomSuffix() + fip := framework.MakeIptablesFIPRule(fipName, eipName, vip.Status.V4ip) + _ = iptablesFIPClient.CreateSync(fip) + + ginkgo.By("10. Verifying FIP is created successfully") + createdFip := iptablesFIPClient.Get(fipName) + framework.ExpectNotNil(createdFip, "FIP should be created successfully") + framework.ExpectTrue(createdFip.Status.Ready, "FIP should be ready") + + ginkgo.By("11. Cleaning up resources") + iptablesFIPClient.DeleteSync(fipName) + vipClient.DeleteSync(vipName) + iptablesEIPClient.DeleteSync(eipName) + + vpcNatGwClient.DeleteSync(vpcNatGwName) + subnetClient.DeleteSync(overlaySubnetName) + subnetClient.DeleteSync(noIPAMNadName) + vpcClient.DeleteSync(vpcName) + attachNetClient.Delete(noIPAMNadName) + + ginkgo.By("12. Test completed: VPC NAT Gateway with no IPAM NAD and noDefaultEIP works correctly") + }) }) func init() { From 8573a56557d3741fd22d706aa28419def51bcb38 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Sat, 13 Dec 2025 01:05:27 +0800 Subject: [PATCH 07/30] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E6=97=A0=E9=BB=98=E8=AE=A4EIP=E7=9A=84VPC=20NAT=E7=BD=91?= =?UTF-8?q?=E5=85=B3=E5=88=9B=E5=BB=BA=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/e2e/framework/vpc-nat-gateway.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/e2e/framework/vpc-nat-gateway.go b/test/e2e/framework/vpc-nat-gateway.go index 0b2a547e1d6..1607df81003 100644 --- a/test/e2e/framework/vpc-nat-gateway.go +++ b/test/e2e/framework/vpc-nat-gateway.go @@ -222,3 +222,9 @@ func MakeVpcNatGateway(name, vpc, subnet, lanIP, externalSubnet, qosPolicyName s vpcNatGw.Spec.QoSPolicy = qosPolicyName return vpcNatGw } + +func MakeVpcNatGatewayWithNoDefaultEIP(name, vpc, subnet, lanIP, externalSubnet, qosPolicyName string, noDefaultEIP bool) *apiv1.VpcNatGateway { + vpcNatGw := MakeVpcNatGateway(name, vpc, subnet, lanIP, externalSubnet, qosPolicyName) + vpcNatGw.Spec.NoDefaultEIP = noDefaultEIP + return vpcNatGw +} From c3ec3659ba06d050bd1b78d75409c60c80714e95 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Sat, 13 Dec 2025 11:27:39 +0800 Subject: [PATCH 08/30] =?UTF-8?q?=E4=BC=98=E5=8C=96E2E=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=EF=BC=8C=E7=BB=9F=E4=B8=80=E9=9A=8F=E6=9C=BA=E5=90=8E=E7=BC=80?= =?UTF-8?q?=E7=94=9F=E6=88=90=E6=96=B9=E5=BC=8F=E5=B9=B6=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E7=AD=89=E5=BE=85=E9=80=BB=E8=BE=91=E4=BB=A5=E7=A1=AE=E4=BF=9D?= =?UTF-8?q?OVN=20EIP=E7=9A=84=E6=9C=80=E7=BB=88=E5=8C=96=E5=92=8C=E5=AD=90?= =?UTF-8?q?=E7=BD=91=E7=8A=B6=E6=80=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 78 ++++++++++++++++------------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index dc8f32eafeb..c438f229cb8 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -128,57 +128,58 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { // gw node is 2 means e2e HA cluster will have 2 gw nodes and a worker node // in this env, tcpdump gw nat flows will be more clear - noBfdVpcName = "no-bfd-vpc-" + framework.RandomSuffix() - bfdVpcName = "bfd-vpc-" + framework.RandomSuffix() + randomSuffix := framework.RandomSuffix() + noBfdVpcName = "no-bfd-vpc-" + randomSuffix + bfdVpcName = "bfd-vpc-" + randomSuffix // nats use ip crd name or vip crd - fipName = "fip-" + framework.RandomSuffix() - - countingEipName = "counting-eip-" + framework.RandomSuffix() - noBfdSubnetName = "no-bfd-subnet-" + framework.RandomSuffix() - noBfdExtraSubnetName = "no-bfd-extra-subnet-" + framework.RandomSuffix() - lrpEipSnatName = "lrp-eip-snat-" + framework.RandomSuffix() - lrpExtraEipSnatName = "lrp-extra-eip-snat-" + framework.RandomSuffix() - bfdSubnetName = "bfd-subnet-" + framework.RandomSuffix() - providerNetworkName = "external" - providerExtraNetworkName = "extra" - vlanName = "vlan-" + framework.RandomSuffix() - vlanExtraName = "vlan-extra-" + framework.RandomSuffix() - underlaySubnetName = "external" - underlayExtraSubnetName = "extra" + fipName = "fip-" + randomSuffix + + countingEipName = "counting-eip-" + randomSuffix + noBfdSubnetName = "no-bfd-subnet-" + randomSuffix + noBfdExtraSubnetName = "no-bfd-extra-subnet-" + randomSuffix + lrpEipSnatName = "lrp-eip-snat-" + randomSuffix + lrpExtraEipSnatName = "lrp-extra-eip-snat-" + randomSuffix + bfdSubnetName = "bfd-subnet-" + randomSuffix + providerNetworkName = "external-" + randomSuffix + providerExtraNetworkName = "extra-" + randomSuffix + vlanName = "vlan-" + randomSuffix + vlanExtraName = "vlan-extra-" + randomSuffix + underlaySubnetName = "external-" + randomSuffix + underlayExtraSubnetName = "extra-" + randomSuffix // sharing case - sharedVipName = "shared-vip-" + framework.RandomSuffix() - sharedEipDnatName = "shared-eip-dnat-" + framework.RandomSuffix() - sharedEipFipShoudOkName = "shared-eip-fip-should-ok-" + framework.RandomSuffix() - sharedEipFipShoudFailName = "shared-eip-fip-should-fail-" + framework.RandomSuffix() + sharedVipName = "shared-vip-" + randomSuffix + sharedEipDnatName = "shared-eip-dnat-" + randomSuffix + sharedEipFipShoudOkName = "shared-eip-fip-should-ok-" + randomSuffix + sharedEipFipShoudFailName = "shared-eip-fip-should-fail-" + randomSuffix // pod with fip - fipPodName = "fip-pod-" + framework.RandomSuffix() + fipPodName = "fip-pod-" + randomSuffix podEipName = fipPodName podFipName = fipPodName // pod with fip for extra external subnet - fipExtraPodName = "fip-extra-pod-" + framework.RandomSuffix() + fipExtraPodName = "fip-extra-pod-" + randomSuffix podExtraEipName = fipExtraPodName podExtraFipName = fipExtraPodName // fip use ip addr - ipFipVipName = "ip-fip-vip-" + framework.RandomSuffix() - ipFipEipName = "ip-fip-eip-" + framework.RandomSuffix() - ipFipName = "ip-fip-" + framework.RandomSuffix() + ipFipVipName = "ip-fip-vip-" + randomSuffix + ipFipEipName = "ip-fip-eip-" + randomSuffix + ipFipName = "ip-fip-" + randomSuffix // dnat use ip addr - ipDnatVipName = "ip-dnat-vip-" + framework.RandomSuffix() - ipDnatEipName = "ip-dnat-eip-" + framework.RandomSuffix() - ipDnatName = "ip-dnat-" + framework.RandomSuffix() + ipDnatVipName = "ip-dnat-vip-" + randomSuffix + ipDnatEipName = "ip-dnat-eip-" + randomSuffix + ipDnatName = "ip-dnat-" + randomSuffix // snat use ip cidr - cidrSnatEipName = "cidr-snat-eip-" + framework.RandomSuffix() - cidrSnatName = "cidr-snat-" + framework.RandomSuffix() - ipSnatVipName = "ip-snat-vip-" + framework.RandomSuffix() - ipSnatEipName = "ip-snat-eip-" + framework.RandomSuffix() - ipSnatName = "ip-snat-" + framework.RandomSuffix() + cidrSnatEipName = "cidr-snat-eip-" + randomSuffix + cidrSnatName = "cidr-snat-" + randomSuffix + ipSnatVipName = "ip-snat-vip-" + randomSuffix + ipSnatEipName = "ip-snat-eip-" + randomSuffix + ipSnatName = "ip-snat-" + randomSuffix if skip { ginkgo.Skip("underlay spec only runs on kind clusters") @@ -519,7 +520,13 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { ginkgo.By("Checking underlay vlan " + oldUnderlayExternalSubnet.Name) framework.ExpectEqual(oldUnderlayExternalSubnet.Spec.Vlan, vlanName) framework.ExpectNotEqual(oldUnderlayExternalSubnet.Spec.CIDRBlock, "") - time.Sleep(3 * time.Second) + ginkgo.By("Wait for ovn eip finalizer to be added") + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { + eipCR := ovnEipClient.Get(countingEipName) + return eipCR != nil && len(eipCR.Finalizers) > 0, nil + }, "OvnEip should have finalizer added") + ginkgo.By("Wait for subnet status to be updated after ovn eip creation") + time.Sleep(5 * time.Second) newUnerlayExternalSubnet := subnetClient.Get(underlaySubnetName) ginkgo.By("Check status using ovn eip for subnet " + underlaySubnetName) if newUnerlayExternalSubnet.Spec.Protocol == kubeovnv1.ProtocolIPv4 { @@ -536,7 +543,8 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { // delete counting eip oldUnderlayExternalSubnet = newUnerlayExternalSubnet ovnEipClient.DeleteSync(countingEipName) - time.Sleep(3 * time.Second) + ginkgo.By("Wait for subnet status to be updated after ovn eip deletion") + time.Sleep(5 * time.Second) newUnerlayExternalSubnet = subnetClient.Get(underlaySubnetName) if newUnerlayExternalSubnet.Spec.Protocol == kubeovnv1.ProtocolIPv4 { framework.ExpectEqual(oldUnderlayExternalSubnet.Status.V4AvailableIPs+1, newUnerlayExternalSubnet.Status.V4AvailableIPs) From 13709383eabf96d75912e59cb1dd7dfe16f34f4a Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Sat, 13 Dec 2025 17:58:14 +0800 Subject: [PATCH 09/30] fix ovn vpc e2e Signed-off-by: zbb88888 --- pkg/controller/ovn_dnat.go | 54 +++++++++++++++++++++- pkg/controller/ovn_eip.go | 36 +++++++++++---- pkg/controller/ovn_fip.go | 52 ++++++++++++++++++++- pkg/controller/ovn_snat.go | 52 ++++++++++++++++++++- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 72 +++++++++++++++++++++-------- 5 files changed, 232 insertions(+), 34 deletions(-) diff --git a/pkg/controller/ovn_dnat.go b/pkg/controller/ovn_dnat.go index 1647d98bc48..143bf70e6a3 100644 --- a/pkg/controller/ovn_dnat.go +++ b/pkg/controller/ovn_dnat.go @@ -34,8 +34,9 @@ func (c *Controller) enqueueUpdateOvnDnatRule(oldObj, newObj any) { // avoid delete twice return } - klog.Infof("enqueue del ovn dnat %s", key) - c.delOvnDnatRuleQueue.Add(key) + // DNAT with finalizer should be handled in updateOvnDnatRuleQueue + klog.Infof("enqueue update (deleting) ovn dnat %s", key) + c.updateOvnDnatRuleQueue.Add(key) return } oldDnat := oldObj.(*kubeovnv1.OvnDnatRule) @@ -297,6 +298,48 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error { klog.Error(err) return err } + + // Handle deletion first (for DNATs with finalizers) + if !cachedDnat.DeletionTimestamp.IsZero() { + klog.Infof("handle deleting ovn dnat %s", key) + if cachedDnat.Status.Vpc == "" { + // Already cleaned, just remove finalizer + if err = c.handleDelOvnDnatFinalizer(cachedDnat); err != nil { + klog.Errorf("failed to remove finalizer for ovn dnat %s, %v", cachedDnat.Name, err) + return err + } + return nil + } + + // ovn delete dnat + if cachedDnat.Status.V4Eip != "" && cachedDnat.Status.ExternalPort != "" { + if err = c.DelDnatRule(cachedDnat.Status.Vpc, cachedDnat.Name, + cachedDnat.Status.V4Eip, cachedDnat.Status.ExternalPort); err != nil { + klog.Errorf("failed to delete v4 dnat %s, %v", key, err) + return err + } + } + if cachedDnat.Status.V6Eip != "" && cachedDnat.Status.ExternalPort != "" { + if err = c.DelDnatRule(cachedDnat.Status.Vpc, cachedDnat.Name, + cachedDnat.Status.V6Eip, cachedDnat.Status.ExternalPort); err != nil { + klog.Errorf("failed to delete v6 dnat %s, %v", key, err) + return err + } + } + + // Remove finalizer + if err = c.handleDelOvnDnatFinalizer(cachedDnat); err != nil { + klog.Errorf("failed to remove finalizer for ovn dnat %s, %v", cachedDnat.Name, err) + return err + } + + // Reset eip + if cachedDnat.Spec.OvnEip != "" { + c.resetOvnEipQueue.Add(cachedDnat.Spec.OvnEip) + } + return nil + } + if !cachedDnat.Status.Ready { // create dnat only in add process, just check to error out here klog.Infof("wait ovn dnat %s to be ready only in the handle add process", cachedDnat.Name) @@ -655,5 +698,12 @@ func (c *Controller) handleDelOvnDnatFinalizer(cachedDnat *kubeovnv1.OvnDnatRule klog.Errorf("failed to remove finalizer from ovn dnat '%s', %v", cachedDnat.Name, err) return err } + + // Trigger associated EIP to recheck if it can be deleted now + if cachedDnat.Spec.OvnEip != "" { + klog.Infof("triggering eip %s update after dnat %s deletion", cachedDnat.Spec.OvnEip, cachedDnat.Name) + c.updateOvnEipQueue.Add(cachedDnat.Spec.OvnEip) + } + return nil } diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index 46ddd40a51e..89c7863a144 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -35,8 +35,9 @@ func (c *Controller) enqueueUpdateOvnEip(oldObj, newObj any) { // avoid delete eip twice return } - klog.Infof("enqueue del ovn eip %s", key) - c.delOvnEipQueue.Add(newEip) + // EIP with finalizer should be handled in updateOvnEipQueue + klog.Infof("enqueue update (deleting) ovn eip %s", key) + c.updateOvnEipQueue.Add(key) return } oldEip := oldObj.(*kubeovnv1.OvnEip) @@ -172,8 +173,9 @@ func (c *Controller) handleUpdateOvnEip(key string) error { return err } if nat != "" { - klog.Infof("ovn eip %s is still being used by NAT rules: %s, waiting for them to be deleted", key, nat) - return nil + err := fmt.Errorf("ovn eip %s is still being used by NAT rules: %s, waiting for them to be deleted", key, nat) + klog.Error(err) + return err } // Clean up resources before removing finalizer @@ -264,12 +266,28 @@ func (c *Controller) handleResetOvnEip(key string) error { } func (c *Controller) handleDelOvnEip(eip *kubeovnv1.OvnEip) error { - // Cleanup is now handled in handleUpdateOvnEip before finalizer removal - // This function is kept for compatibility with the delete queue - klog.V(3).Infof("ovn eip %s cleanup already done in update handler", eip.Name) + // This handles deletion of EIPs without finalizers (race condition or direct deletion) + // EIPs with finalizers are handled in handleUpdateOvnEip + klog.Infof("handle del ovn eip %s (without finalizer)", eip.Name) + + // Clean up resources if they still exist + if eip.Spec.Type == util.OvnEipTypeLSP { + if err := c.OVNNbClient.DeleteLogicalSwitchPort(eip.Name); err != nil { + klog.Errorf("failed to delete lsp %s, %v", eip.Name, err) + return err + } + } + if eip.Spec.Type == util.OvnEipTypeLRP { + if err := c.OVNNbClient.DeleteLogicalRouterPort(eip.Name); err != nil { + klog.Errorf("failed to delete lrp %s, %v", eip.Name, err) + return err + } + } + + // Release IP from IPAM + c.ipam.ReleaseAddressByPod(eip.Name, eip.Spec.ExternalSubnet) - // For OvnEips deleted without finalizer (race condition or direct deletion), - // we need to ensure subnet status is updated as a safety net. + // Ensure subnet status is updated if eip.Spec.ExternalSubnet != "" { c.updateSubnetStatusQueue.Add(eip.Spec.ExternalSubnet) } diff --git a/pkg/controller/ovn_fip.go b/pkg/controller/ovn_fip.go index 7c47cc2998e..e95193b3f26 100644 --- a/pkg/controller/ovn_fip.go +++ b/pkg/controller/ovn_fip.go @@ -35,8 +35,9 @@ func (c *Controller) enqueueUpdateOvnFip(oldObj, newObj any) { // avoid delete twice return } - klog.Infof("enqueue del ovn fip %s", key) - c.delOvnFipQueue.Add(key) + // FIP with finalizer should be handled in updateOvnFipQueue + klog.Infof("enqueue update (deleting) ovn fip %s", key) + c.updateOvnFipQueue.Add(key) return } oldFip := oldObj.(*kubeovnv1.OvnFip) @@ -267,6 +268,46 @@ func (c *Controller) handleUpdateOvnFip(key string) error { klog.Error(err) return err } + + // Handle deletion first (for FIPs with finalizers) + if !cachedFip.DeletionTimestamp.IsZero() { + klog.Infof("handle deleting ovn fip %s", key) + if cachedFip.Status.Vpc == "" { + // Already cleaned, just remove finalizer + if err = c.handleDelOvnFipFinalizer(cachedFip); err != nil { + klog.Errorf("failed to remove finalizer for ovn fip %s, %v", cachedFip.Name, err) + return err + } + return nil + } + + // ovn delete fip nat + if cachedFip.Status.V4Eip != "" && cachedFip.Status.V4Ip != "" { + if err = c.OVNNbClient.DeleteNat(cachedFip.Status.Vpc, ovnnb.NATTypeDNATAndSNAT, cachedFip.Status.V4Eip, cachedFip.Status.V4Ip); err != nil { + klog.Errorf("failed to delete v4 fip %s, %v", key, err) + return err + } + } + if cachedFip.Status.V6Eip != "" && cachedFip.Status.V6Ip != "" { + if err = c.OVNNbClient.DeleteNat(cachedFip.Status.Vpc, ovnnb.NATTypeDNATAndSNAT, cachedFip.Status.V6Eip, cachedFip.Status.V6Ip); err != nil { + klog.Errorf("failed to delete v6 fip %s, %v", key, err) + return err + } + } + + // Remove finalizer + if err = c.handleDelOvnFipFinalizer(cachedFip); err != nil { + klog.Errorf("failed to remove finalizer for ovn fip %s, %v", cachedFip.Name, err) + return err + } + + // Reset eip + if cachedFip.Spec.OvnEip != "" { + c.resetOvnEipQueue.Add(cachedFip.Spec.OvnEip) + } + return nil + } + if !cachedFip.Status.Ready { // create fip only in add process, just check to error out here klog.Infof("wait ovn fip %s to be ready only in the handle add process", cachedFip.Name) @@ -580,5 +621,12 @@ func (c *Controller) handleDelOvnFipFinalizer(cachedFip *kubeovnv1.OvnFip) error klog.Errorf("failed to remove finalizer from ovn fip '%s', %v", cachedFip.Name, err) return err } + + // Trigger associated EIP to recheck if it can be deleted now + if cachedFip.Spec.OvnEip != "" { + klog.Infof("triggering eip %s update after fip %s deletion", cachedFip.Spec.OvnEip, cachedFip.Name) + c.updateOvnEipQueue.Add(cachedFip.Spec.OvnEip) + } + return nil } diff --git a/pkg/controller/ovn_snat.go b/pkg/controller/ovn_snat.go index 11de6beed10..8fb40981f24 100644 --- a/pkg/controller/ovn_snat.go +++ b/pkg/controller/ovn_snat.go @@ -33,8 +33,9 @@ func (c *Controller) enqueueUpdateOvnSnatRule(oldObj, newObj any) { // avoid delete twice return } - klog.Infof("enqueue del ovn snat %s", key) - c.delOvnSnatRuleQueue.Add(key) + // SNAT with finalizer should be handled in updateOvnSnatRuleQueue + klog.Infof("enqueue update (deleting) ovn snat %s", key) + c.updateOvnSnatRuleQueue.Add(key) return } oldSnat := oldObj.(*kubeovnv1.OvnSnatRule) @@ -197,6 +198,46 @@ func (c *Controller) handleUpdateOvnSnatRule(key string) error { klog.Error(err) return err } + + // Handle deletion first (for SNATs with finalizers) + if !cachedSnat.DeletionTimestamp.IsZero() { + klog.Infof("handle deleting ovn snat %s", key) + if cachedSnat.Status.Vpc == "" { + // Already cleaned, just remove finalizer + if err = c.handleDelOvnSnatFinalizer(cachedSnat); err != nil { + klog.Errorf("failed to remove finalizer for ovn snat %s, %v", cachedSnat.Name, err) + return err + } + return nil + } + + // ovn delete snat + if cachedSnat.Status.V4Eip != "" && cachedSnat.Status.V4IpCidr != "" { + if err = c.OVNNbClient.DeleteNat(cachedSnat.Status.Vpc, ovnnb.NATTypeSNAT, cachedSnat.Status.V4Eip, cachedSnat.Status.V4IpCidr); err != nil { + klog.Errorf("failed to delete v4 snat %s, %v", key, err) + return err + } + } + if cachedSnat.Status.V6Eip != "" && cachedSnat.Status.V6IpCidr != "" { + if err = c.OVNNbClient.DeleteNat(cachedSnat.Status.Vpc, ovnnb.NATTypeSNAT, cachedSnat.Status.V6Eip, cachedSnat.Status.V6IpCidr); err != nil { + klog.Errorf("failed to delete v6 snat %s, %v", key, err) + return err + } + } + + // Remove finalizer + if err = c.handleDelOvnSnatFinalizer(cachedSnat); err != nil { + klog.Errorf("failed to remove finalizer for ovn snat %s, %v", cachedSnat.Name, err) + return err + } + + // Reset eip + if cachedSnat.Spec.OvnEip != "" { + c.resetOvnEipQueue.Add(cachedSnat.Spec.OvnEip) + } + return nil + } + if !cachedSnat.Status.Ready { klog.Infof("wait ovn snat %s to be ready only in the handle add process", cachedSnat.Name) return nil @@ -495,5 +536,12 @@ func (c *Controller) handleDelOvnSnatFinalizer(cachedSnat *kubeovnv1.OvnSnatRule klog.Errorf("failed to remove finalizer from ovn snat '%s', %v", cachedSnat.Name, err) return err } + + // Trigger associated EIP to recheck if it can be deleted now + if cachedSnat.Spec.OvnEip != "" { + klog.Infof("triggering eip %s update after snat %s deletion", cachedSnat.Spec.OvnEip, cachedSnat.Name) + c.updateOvnEipQueue.Add(cachedSnat.Spec.OvnEip) + } + return nil } diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index c438f229cb8..0120e0859bf 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -141,12 +141,14 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { lrpEipSnatName = "lrp-eip-snat-" + randomSuffix lrpExtraEipSnatName = "lrp-extra-eip-snat-" + randomSuffix bfdSubnetName = "bfd-subnet-" + randomSuffix - providerNetworkName = "external-" + randomSuffix - providerExtraNetworkName = "extra-" + randomSuffix + // provider network name has 12 bytes limit, use short prefix + providerNetworkName = "external" + providerExtraNetworkName = "extra" vlanName = "vlan-" + randomSuffix vlanExtraName = "vlan-extra-" + randomSuffix - underlaySubnetName = "external-" + randomSuffix - underlayExtraSubnetName = "extra-" + randomSuffix + // underlay subnet names use fixed names for global reuse + underlaySubnetName = "external" + underlayExtraSubnetName = "extra" // sharing case sharedVipName = "shared-vip-" + randomSuffix @@ -265,9 +267,16 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { itFn = func(exchangeLinkName bool, providerNetworkName string, linkMap map[string]*iproute.Link, bridgeIps *[]string) { ginkgo.GinkgoHelper() - ginkgo.By("Creating provider network " + providerNetworkName) - pn := makeProviderNetwork(providerNetworkName, exchangeLinkName, linkMap) - pn = providerNetworkClient.CreateSync(pn) + ginkgo.By("Getting or creating provider network " + providerNetworkName) + // Try to get existing provider network first + pn, err := providerNetworkClient.ProviderNetworkInterface.Get(context.Background(), providerNetworkName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // Provider network doesn't exist, create it + pn = makeProviderNetwork(providerNetworkName, exchangeLinkName, linkMap) + pn = providerNetworkClient.CreateSync(pn) + } else { + framework.ExpectNoError(err, "getting provider network "+providerNetworkName) + } ginkgo.By("Getting k8s nodes") k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) @@ -472,11 +481,17 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { exchangeLinkName := false itFn(exchangeLinkName, providerNetworkName, linkMap, &providerBridgeIps) - ginkgo.By("Creating underlay vlan " + vlanName) - vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) - _ = vlanClient.Create(vlan) + ginkgo.By("Getting or creating underlay vlan " + vlanName) + _, err = vlanClient.VlanInterface.Get(context.Background(), vlanName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // Vlan doesn't exist, create it + vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) + _ = vlanClient.Create(vlan) + } else { + framework.ExpectNoError(err, "getting vlan "+vlanName) + } - ginkgo.By("Creating underlay subnet " + underlaySubnetName) + ginkgo.By("Getting or creating underlay subnet " + underlaySubnetName) var cidrV4, cidrV6, gatewayV4, gatewayV6 string for _, config := range dockerNetwork.IPAM.Config { switch util.CheckProtocol(config.Subnet.String()) { @@ -513,8 +528,15 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { } vlanSubnetCidr := strings.Join(cidr, ",") vlanSubnetGw := strings.Join(gateway, ",") - underlaySubnet := framework.MakeSubnet(underlaySubnetName, vlanName, vlanSubnetCidr, vlanSubnetGw, "", "", excludeIPs, nil, nil) - oldUnderlayExternalSubnet := subnetClient.CreateSync(underlaySubnet) + var oldUnderlayExternalSubnet *kubeovnv1.Subnet + oldUnderlayExternalSubnet, err = subnetClient.SubnetInterface.Get(context.Background(), underlaySubnetName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // Subnet doesn't exist, create it + underlaySubnet := framework.MakeSubnet(underlaySubnetName, vlanName, vlanSubnetCidr, vlanSubnetGw, "", "", excludeIPs, nil, nil) + oldUnderlayExternalSubnet = subnetClient.CreateSync(underlaySubnet) + } else { + framework.ExpectNoError(err, "getting subnet "+underlaySubnetName) + } countingEip := makeOvnEip(countingEipName, underlaySubnetName, "", "", "", "") _ = ovnEipClient.CreateSync(countingEip) ginkgo.By("Checking underlay vlan " + oldUnderlayExternalSubnet.Name) @@ -703,11 +725,17 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectNoError(err, "getting extra docker network "+dockerExtraNetworkName) itFn(exchangeLinkName, providerExtraNetworkName, extraLinkMap, &extraProviderBridgeIps) - ginkgo.By("Creating underlay extra vlan " + vlanExtraName) - vlan = framework.MakeVlan(vlanExtraName, providerExtraNetworkName, 0) - _ = vlanClient.Create(vlan) + ginkgo.By("Getting or creating underlay extra vlan " + vlanExtraName) + _, err = vlanClient.VlanInterface.Get(context.Background(), vlanExtraName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // Vlan doesn't exist, create it + vlan := framework.MakeVlan(vlanExtraName, providerExtraNetworkName, 0) + _ = vlanClient.Create(vlan) + } else { + framework.ExpectNoError(err, "getting vlan "+vlanExtraName) + } - ginkgo.By("Creating extra underlay subnet " + underlayExtraSubnetName) + ginkgo.By("Getting or creating extra underlay subnet " + underlayExtraSubnetName) cidrV4, cidrV6, gatewayV4, gatewayV6 = "", "", "", "" for _, config := range dockerExtraNetwork.IPAM.Config { switch util.CheckProtocol(config.Subnet.String()) { @@ -745,8 +773,14 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { } extraVlanSubnetCidr := strings.Join(cidr, ",") extraVlanSubnetGw := strings.Join(gateway, ",") - underlayExtraSubnet := framework.MakeSubnet(underlayExtraSubnetName, vlanExtraName, extraVlanSubnetCidr, extraVlanSubnetGw, "", "", extraExcludeIPs, nil, nil) - _ = subnetClient.CreateSync(underlayExtraSubnet) + _, err = subnetClient.SubnetInterface.Get(context.Background(), underlayExtraSubnetName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // Subnet doesn't exist, create it + underlayExtraSubnet := framework.MakeSubnet(underlayExtraSubnetName, vlanExtraName, extraVlanSubnetCidr, extraVlanSubnetGw, "", "", extraExcludeIPs, nil, nil) + _ = subnetClient.CreateSync(underlayExtraSubnet) + } else { + framework.ExpectNoError(err, "getting subnet "+underlayExtraSubnetName) + } vlanExtraSubnet := subnetClient.Get(underlayExtraSubnetName) ginkgo.By("Checking extra underlay vlan " + vlanExtraSubnet.Name) framework.ExpectEqual(vlanExtraSubnet.Spec.Vlan, vlanExtraName) From ce7a4dcdb4187d5b5b4dc3bd8f7f61f1e7925369 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Sat, 13 Dec 2025 20:28:53 +0800 Subject: [PATCH 10/30] fix old finalizer Signed-off-by: zbb88888 --- pkg/controller/ip.go | 1 + pkg/controller/ippool.go | 1 + pkg/controller/ovn_dnat.go | 1 + pkg/controller/ovn_eip.go | 1 + pkg/controller/ovn_fip.go | 1 + pkg/controller/ovn_snat.go | 1 + pkg/controller/qos_policy.go | 2 ++ pkg/controller/subnet.go | 2 ++ pkg/controller/vip.go | 2 ++ pkg/controller/vpc.go | 2 ++ pkg/controller/vpc_egress_gateway.go | 2 ++ pkg/controller/vpc_nat_gw_eip.go | 2 ++ 12 files changed, 18 insertions(+) diff --git a/pkg/controller/ip.go b/pkg/controller/ip.go index 27a791fccb5..81868b8824a 100644 --- a/pkg/controller/ip.go +++ b/pkg/controller/ip.go @@ -320,6 +320,7 @@ func (c *Controller) handleAddOrUpdateIPFinalizer(cachedIP *kubeovnv1.IP) error } newIP := cachedIP.DeepCopy() + controllerutil.RemoveFinalizer(newIP, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newIP, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedIP, newIP) if err != nil { diff --git a/pkg/controller/ippool.go b/pkg/controller/ippool.go index d91c028b7ae..04c443f3fe4 100644 --- a/pkg/controller/ippool.go +++ b/pkg/controller/ippool.go @@ -215,6 +215,7 @@ func (c *Controller) handleAddIPPoolFinalizer(ippool *kubeovnv1.IPPool) error { } newIPPool := ippool.DeepCopy() + controllerutil.RemoveFinalizer(newIPPool, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newIPPool, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(ippool, newIPPool) if err != nil { diff --git a/pkg/controller/ovn_dnat.go b/pkg/controller/ovn_dnat.go index 143bf70e6a3..14bf14fb804 100644 --- a/pkg/controller/ovn_dnat.go +++ b/pkg/controller/ovn_dnat.go @@ -660,6 +660,7 @@ func (c *Controller) handleAddOvnDnatFinalizer(cachedDnat *kubeovnv1.OvnDnatRule err error ) + controllerutil.RemoveFinalizer(newDnat, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newDnat, util.KubeOVNControllerFinalizer) if patch, err = util.GenerateMergePatchPayload(cachedDnat, newDnat); err != nil { klog.Errorf("failed to generate patch payload for ovn dnat '%s', %v", cachedDnat.Name, err) diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index 89c7863a144..06eac63b492 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -530,6 +530,7 @@ func (c *Controller) handleAddOrUpdateOvnEipFinalizer(cachedEip *kubeovnv1.OvnEi return nil } newEip := cachedEip.DeepCopy() + controllerutil.RemoveFinalizer(newEip, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newEip, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedEip, newEip) if err != nil { diff --git a/pkg/controller/ovn_fip.go b/pkg/controller/ovn_fip.go index e95193b3f26..724f3767993 100644 --- a/pkg/controller/ovn_fip.go +++ b/pkg/controller/ovn_fip.go @@ -583,6 +583,7 @@ func (c *Controller) handleAddOvnFipFinalizer(cachedFip *kubeovnv1.OvnFip) error return nil } newFip := cachedFip.DeepCopy() + controllerutil.RemoveFinalizer(newFip, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newFip, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedFip, newFip) if err != nil { diff --git a/pkg/controller/ovn_snat.go b/pkg/controller/ovn_snat.go index 8fb40981f24..e2e5df5b1cb 100644 --- a/pkg/controller/ovn_snat.go +++ b/pkg/controller/ovn_snat.go @@ -499,6 +499,7 @@ func (c *Controller) handleAddOvnSnatFinalizer(cachedSnat *kubeovnv1.OvnSnatRule return nil } newSnat := cachedSnat.DeepCopy() + controllerutil.RemoveFinalizer(newSnat, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newSnat, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedSnat, newSnat) if err != nil { diff --git a/pkg/controller/qos_policy.go b/pkg/controller/qos_policy.go index 3b0f890c302..a2dc1912e82 100644 --- a/pkg/controller/qos_policy.go +++ b/pkg/controller/qos_policy.go @@ -164,6 +164,7 @@ func (c *Controller) handleDelQoSPoliciesFinalizer(key string) error { return nil } newQoSPolicies := cachedQoSPolicies.DeepCopy() + controllerutil.RemoveFinalizer(newQoSPolicies, util.DepreciatedFinalizerName) controllerutil.RemoveFinalizer(newQoSPolicies, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedQoSPolicies, newQoSPolicies) if err != nil { @@ -435,6 +436,7 @@ func (c *Controller) handleAddQoSPolicyFinalizer(key string) error { return nil } newQoSPolicy := cachedQoSPolicy.DeepCopy() + controllerutil.RemoveFinalizer(newQoSPolicy, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newQoSPolicy, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedQoSPolicy, newQoSPolicy) if err != nil { diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index d6f17d8c10c..9ce55204d6b 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -275,6 +275,7 @@ func (c *Controller) syncSubnetFinalizer(cl client.Client) error { func (c *Controller) handleSubnetFinalizer(subnet *kubeovnv1.Subnet) (*kubeovnv1.Subnet, bool, error) { if subnet.DeletionTimestamp.IsZero() && !slices.Contains(subnet.GetFinalizers(), util.KubeOVNControllerFinalizer) { newSubnet := subnet.DeepCopy() + controllerutil.RemoveFinalizer(newSubnet, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newSubnet, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(subnet, newSubnet) if err != nil { @@ -298,6 +299,7 @@ func (c *Controller) handleSubnetFinalizer(subnet *kubeovnv1.Subnet) (*kubeovnv1 u2oInterconnIP := subnet.Status.U2OInterconnectionIP if !subnet.DeletionTimestamp.IsZero() && (usingIPs == 0 || (usingIPs == 1 && u2oInterconnIP != "")) { newSubnet := subnet.DeepCopy() + controllerutil.RemoveFinalizer(newSubnet, util.DepreciatedFinalizerName) controllerutil.RemoveFinalizer(newSubnet, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(subnet, newSubnet) if err != nil { diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index ad9a9f5e270..82f8e182bed 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -519,6 +519,7 @@ func (c *Controller) handleAddOrUpdateVipFinalizer(key string) error { return nil } newVip := cachedVip.DeepCopy() + controllerutil.RemoveFinalizer(newVip, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newVip, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedVip, newVip) if err != nil { @@ -553,6 +554,7 @@ func (c *Controller) handleDelVipFinalizer(key string) error { return nil } newVip := cachedVip.DeepCopy() + controllerutil.RemoveFinalizer(newVip, util.DepreciatedFinalizerName) controllerutil.RemoveFinalizer(newVip, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedVip, newVip) if err != nil { diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index 6d712ee28ae..2b47a0323e5 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -1182,11 +1182,13 @@ func (c *Controller) formatVpc(vpc *kubeovnv1.Vpc) (*kubeovnv1.Vpc, error) { } if vpc.DeletionTimestamp.IsZero() && !slices.Contains(vpc.GetFinalizers(), util.KubeOVNControllerFinalizer) { + controllerutil.RemoveFinalizer(vpc, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(vpc, util.KubeOVNControllerFinalizer) changed = true } if !vpc.DeletionTimestamp.IsZero() && len(vpc.Status.Subnets) == 0 { + controllerutil.RemoveFinalizer(vpc, util.DepreciatedFinalizerName) controllerutil.RemoveFinalizer(vpc, util.KubeOVNControllerFinalizer) changed = true } diff --git a/pkg/controller/vpc_egress_gateway.go b/pkg/controller/vpc_egress_gateway.go index 558e6a95b82..09bc36da434 100644 --- a/pkg/controller/vpc_egress_gateway.go +++ b/pkg/controller/vpc_egress_gateway.go @@ -114,6 +114,7 @@ func (c *Controller) handleAddOrUpdateVpcEgressGateway(key string) error { return err } + controllerutil.RemoveFinalizer(gw, util.DepreciatedFinalizerName) if controllerutil.AddFinalizer(gw, util.KubeOVNControllerFinalizer) { updatedGateway, err := c.config.KubeOvnClient.KubeovnV1().VpcEgressGateways(gw.Namespace). Update(context.Background(), gw, metav1.UpdateOptions{}) @@ -981,6 +982,7 @@ func (c *Controller) handleDelVpcEgressGateway(key string) error { } gw := cachedGateway.DeepCopy() + controllerutil.RemoveFinalizer(gw, util.DepreciatedFinalizerName) if controllerutil.RemoveFinalizer(gw, util.KubeOVNControllerFinalizer) { if _, err = c.config.KubeOvnClient.KubeovnV1().VpcEgressGateways(gw.Namespace). Update(context.Background(), gw, metav1.UpdateOptions{}); err != nil { diff --git a/pkg/controller/vpc_nat_gw_eip.go b/pkg/controller/vpc_nat_gw_eip.go index 476e7109cb6..59d79ac0069 100644 --- a/pkg/controller/vpc_nat_gw_eip.go +++ b/pkg/controller/vpc_nat_gw_eip.go @@ -704,6 +704,7 @@ func (c *Controller) handleAddOrUpdateIptablesEipFinalizer(key string) error { return nil } newIptablesEip := cachedIptablesEip.DeepCopy() + controllerutil.RemoveFinalizer(newIptablesEip, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newIptablesEip, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedIptablesEip, newIptablesEip) if err != nil { @@ -739,6 +740,7 @@ func (c *Controller) handleDelIptablesEipFinalizer(key string) error { return nil } newIptablesEip := cachedIptablesEip.DeepCopy() + controllerutil.RemoveFinalizer(newIptablesEip, util.DepreciatedFinalizerName) controllerutil.RemoveFinalizer(newIptablesEip, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedIptablesEip, newIptablesEip) if err != nil { From 684d9b8a7c4cd37f493fefe6bc7d8707cc3ee3a2 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Sun, 14 Dec 2025 11:54:58 +0800 Subject: [PATCH 11/30] fix ovn vpc e2e Signed-off-by: zbb88888 --- test/e2e/framework/docker/network.go | 11 +++++++ test/e2e/ovn-vpc-nat-gw/e2e_test.go | 43 ++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/test/e2e/framework/docker/network.go b/test/e2e/framework/docker/network.go index 5eefb54d353..9e6a079f247 100644 --- a/test/e2e/framework/docker/network.go +++ b/test/e2e/framework/docker/network.go @@ -8,6 +8,7 @@ import ( "net" "net/netip" "strconv" + "strings" "github.com/moby/moby/api/types/network" "github.com/moby/moby/client" @@ -114,6 +115,16 @@ func NetworkCreate(name string, ipv6, skipIfExists bool) (*network.Inspect, erro defer cli.Close() if _, err = cli.NetworkCreate(context.Background(), name, options); err != nil { + // Handle race condition: if network was created between our check and create attempt + // Docker returns error like "network with name xxx already exists" + if skipIfExists && strings.Contains(err.Error(), "already exists") { + // Network already exists, retrieve and return it + network, getErr := getNetwork(name, false) + if getErr != nil { + return nil, getErr + } + return network, nil + } return nil, err } diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 0120e0859bf..48bd26efb0f 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -31,9 +31,9 @@ import ( "github.com/kubeovn/kube-ovn/test/e2e/framework/kind" ) -const dockerNetworkName = "kube-ovn-vlan" - -const dockerExtraNetworkName = "kube-ovn-extra-vlan" +// Docker network names will be initialized in init() with random suffix to avoid conflicts +var dockerNetworkName string +var dockerExtraNetworkName string func makeProviderNetwork(providerNetworkName string, exchangeLinkName bool, linkMap map[string]*iproute.Link) *kubeovnv1.ProviderNetwork { var defaultInterface string @@ -203,14 +203,14 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { if dockerNetwork == nil { ginkgo.By("Ensuring docker network " + dockerNetworkName + " exists") network, err := docker.NetworkCreate(dockerNetworkName, true, true) - framework.ExpectNoError(err, "creating docker network "+dockerNetworkName) + framework.ExpectNoError(err, "ensuring docker network "+dockerNetworkName+" exists") dockerNetwork = network } if dockerExtraNetwork == nil { ginkgo.By("Ensuring extra docker network " + dockerExtraNetworkName + " exists") network, err := docker.NetworkCreate(dockerExtraNetworkName, true, true) - framework.ExpectNoError(err, "creating extra docker network "+dockerExtraNetworkName) + framework.ExpectNoError(err, "ensuring extra docker network "+dockerExtraNetworkName+" exists") dockerExtraNetwork = network } @@ -459,16 +459,30 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectNoError(err, "timed out waiting for ovs bridge to disappear in node %s", node.Name()) } + if dockerExtraNetwork != nil { + ginkgo.By("Disconnecting nodes from the docker extra network") + if err = kind.NetworkDisconnect(dockerExtraNetwork.ID, nodes); err != nil { + framework.Logf("Warning: failed to disconnect nodes from extra network %s: %v", dockerExtraNetworkName, err) + } + + ginkgo.By("Deleting docker extra network " + dockerExtraNetworkName) + if err = docker.NetworkRemove(dockerExtraNetwork.ID); err != nil { + framework.Logf("Warning: failed to remove docker extra network %s: %v", dockerExtraNetworkName, err) + } + dockerExtraNetwork = nil + } + if dockerNetwork != nil { ginkgo.By("Disconnecting nodes from the docker network") - err = kind.NetworkDisconnect(dockerNetwork.ID, nodes) - framework.ExpectNoError(err, "disconnecting nodes from network "+dockerNetworkName) - } + if err = kind.NetworkDisconnect(dockerNetwork.ID, nodes); err != nil { + framework.Logf("Warning: failed to disconnect nodes from network %s: %v", dockerNetworkName, err) + } - if dockerExtraNetwork != nil { - ginkgo.By("Disconnecting nodes from the docker extra network") - err = kind.NetworkDisconnect(dockerExtraNetwork.ID, nodes) - framework.ExpectNoError(err, "disconnecting nodes from extra network "+dockerExtraNetworkName) + ginkgo.By("Deleting docker network " + dockerNetworkName) + if err = docker.NetworkRemove(dockerNetwork.ID); err != nil { + framework.Logf("Warning: failed to remove docker network %s: %v", dockerNetworkName, err) + } + dockerNetwork = nil } }) @@ -1335,6 +1349,11 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { }) func init() { + // Generate unique network names for this test run to avoid conflicts with previous runs + suffix := framework.RandomSuffix() + dockerNetworkName = "kube-ovn-vlan-" + suffix + dockerExtraNetworkName = "kube-ovn-extra-vlan-" + suffix + klog.SetOutput(ginkgo.GinkgoWriter) // Register flags. From 884ea0f8f8f6f29076958a25458dc53b81b48a2f Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Sun, 14 Dec 2025 14:34:49 +0800 Subject: [PATCH 12/30] fix ovn vpc nat e2e Signed-off-by: zbb88888 --- makefiles/ut.mk | 2 + pkg/ovs/ovn-nb-acl_test.go | 14 ++ test/e2e/framework/docker/network.go | 60 ++++- test/e2e/framework/docker/network_test.go | 274 ++++++++++++++++++++++ test/e2e/ovn-vpc-nat-gw/e2e_test.go | 21 +- 5 files changed, 364 insertions(+), 7 deletions(-) create mode 100644 test/e2e/framework/docker/network_test.go diff --git a/makefiles/ut.mk b/makefiles/ut.mk index 4ea06fa06cf..2ed2eccddbf 100644 --- a/makefiles/ut.mk +++ b/makefiles/ut.mk @@ -4,6 +4,8 @@ ut: ginkgo -mod=mod --show-node-events --poll-progress-after=60s $(GINKGO_OUTPUT_OPT) -v test/unittest go test -coverprofile=profile.cov $$(go list ./pkg/... | grep -vw '^github.com/kubeovn/kube-ovn/pkg/client') + @echo "Running e2e framework unit tests..." + go test -v ./test/e2e/framework/docker .PHONY: ovs-sandbox ovs-sandbox: clean-ovs-sandbox diff --git a/pkg/ovs/ovn-nb-acl_test.go b/pkg/ovs/ovn-nb-acl_test.go index fa4f2bd87c9..46451f890e4 100644 --- a/pkg/ovs/ovn-nb-acl_test.go +++ b/pkg/ovs/ovn-nb-acl_test.go @@ -2596,8 +2596,15 @@ func (suite *OvnClientTestSuite) testUpdateAnpRuleACLOps() { isIngress := true isBanp := false + // Clean up any existing port group from previous test runs + _ = nbClient.DeletePortGroup(pgName) + err := nbClient.CreatePortGroup(pgName, nil) require.NoError(t, err) + defer func() { + err := nbClient.DeletePortGroup(pgName) + require.NoError(t, err) + }() ops, err := nbClient.UpdateAnpRuleACLOps(pgName, asName, protocol, aclName, priority, aclAction, logACLActions, rulePorts, isIngress, isBanp) require.NoError(t, err) require.NotEmpty(t, ops) @@ -2616,8 +2623,15 @@ func (suite *OvnClientTestSuite) testUpdateAnpRuleACLOps() { isIngress := false isBanp := true + // Clean up any existing port group from previous test runs + _ = nbClient.DeletePortGroup(pgName) + err := nbClient.CreatePortGroup(pgName, nil) require.NoError(t, err) + defer func() { + err := nbClient.DeletePortGroup(pgName) + require.NoError(t, err) + }() ops, err := nbClient.UpdateAnpRuleACLOps(pgName, asName, protocol, aclName, priority, aclAction, logACLActions, rulePorts, isIngress, isBanp) require.NoError(t, err) require.NotEmpty(t, ops) diff --git a/test/e2e/framework/docker/network.go b/test/e2e/framework/docker/network.go index 9e6a079f247..04beb1a4d20 100644 --- a/test/e2e/framework/docker/network.go +++ b/test/e2e/framework/docker/network.go @@ -5,10 +5,12 @@ import ( "crypto/sha1" "encoding/binary" "fmt" + "math/rand" "net" "net/netip" "strconv" "strings" + "time" "github.com/moby/moby/api/types/network" "github.com/moby/moby/client" @@ -19,6 +21,41 @@ import ( const MTU = 1500 +// Base network for test networks: 172.28.0.0/16 +// This allows 256 /24 subnets (172.28.0.0/24 to 172.28.255.0/24) +const testNetworkBase = "172.28" + +// GenerateRandomSubnets generates N random /24 subnets within 172.28.0.0/16 +// Returns a slice of subnets in CIDR notation (e.g., ["172.28.123.0/24", "172.28.124.0/24"]) +// Subnets are guaranteed to be unique within the returned slice +func GenerateRandomSubnets(count int) []string { + if count <= 0 || count > 256 { + panic(fmt.Sprintf("invalid subnet count: %d (must be 1-256)", count)) + } + + // Use current nanosecond timestamp as seed for better randomness + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + + // Generate a random permutation of subnet indices + subnets := make([]string, count) + usedOctets := make(map[int]bool) + + for i := 0; i < count; i++ { + var octet int + // Find an unused octet + for { + octet = rng.Intn(256) + if !usedOctets[octet] { + usedOctets[octet] = true + break + } + } + subnets[i] = fmt.Sprintf("%s.%d.0/24", testNetworkBase, octet) + } + + return subnets +} + // https://github.com/kubernetes-sigs/kind/tree/main/pkg/cluster/internal/providers/docker/network.go#L313 // generateULASubnetFromName generate an IPv6 subnet based on the // name and Nth probing attempt @@ -72,7 +109,8 @@ func NetworkInspect(name string) (*network.Inspect, error) { return getNetwork(name, false) } -func NetworkCreate(name string, ipv6, skipIfExists bool) (*network.Inspect, error) { +// NetworkCreateWithSubnet creates a docker network with specified IPv4 subnet +func NetworkCreateWithSubnet(name string, ipv4Subnet string, ipv6, skipIfExists bool) (*network.Inspect, error) { if skipIfExists { network, err := getNetwork(name, true) if err != nil { @@ -94,6 +132,21 @@ func NetworkCreate(name string, ipv6, skipIfExists bool) (*network.Inspect, erro "com.docker.network.driver.mtu": strconv.Itoa(MTU), }, } + + // Add IPv4 subnet if specified + if ipv4Subnet != "" { + gateway, err := util.FirstIP(ipv4Subnet) + if err != nil { + return nil, fmt.Errorf("failed to get gateway for subnet %s: %v", ipv4Subnet, err) + } + config := network.IPAMConfig{ + Subnet: netip.MustParsePrefix(ipv4Subnet), + Gateway: netip.MustParseAddr(gateway), + } + options.IPAM.Config = append(options.IPAM.Config, config) + } + + // Add IPv6 subnet if enabled if ipv6 { options.EnableIPv6 = ptr.To(true) subnet := generateULASubnetFromName(name, 0) @@ -131,6 +184,11 @@ func NetworkCreate(name string, ipv6, skipIfExists bool) (*network.Inspect, erro return getNetwork(name, false) } +// NetworkCreate creates a docker network (backward compatible wrapper) +func NetworkCreate(name string, ipv6, skipIfExists bool) (*network.Inspect, error) { + return NetworkCreateWithSubnet(name, "", ipv6, skipIfExists) +} + func NetworkConnect(networkID, containerID string) error { cli, err := client.New(client.FromEnv) if err != nil { diff --git a/test/e2e/framework/docker/network_test.go b/test/e2e/framework/docker/network_test.go new file mode 100644 index 00000000000..7c854c46c07 --- /dev/null +++ b/test/e2e/framework/docker/network_test.go @@ -0,0 +1,274 @@ +package docker + +import ( + "fmt" + "regexp" + "strings" + "testing" +) + +func TestGenerateRandomSubnets(t *testing.T) { + tests := []struct { + name string + count int + expectPanic bool + validateFunc func([]string) error + }{ + { + name: "generate 1 subnet", + count: 1, + validateFunc: func(subnets []string) error { + if len(subnets) != 1 { + return fmt.Errorf("expected 1 subnet, got %d", len(subnets)) + } + return nil + }, + }, + { + name: "generate 2 subnets", + count: 2, + validateFunc: func(subnets []string) error { + if len(subnets) != 2 { + return fmt.Errorf("expected 2 subnets, got %d", len(subnets)) + } + return nil + }, + }, + { + name: "generate 5 subnets", + count: 5, + validateFunc: func(subnets []string) error { + if len(subnets) != 5 { + return fmt.Errorf("expected 5 subnets, got %d", len(subnets)) + } + return nil + }, + }, + { + name: "generate maximum (256) subnets", + count: 256, + validateFunc: func(subnets []string) error { + if len(subnets) != 256 { + return fmt.Errorf("expected 256 subnets, got %d", len(subnets)) + } + return nil + }, + }, + { + name: "invalid count: 0", + count: 0, + expectPanic: true, + }, + { + name: "invalid count: negative", + count: -1, + expectPanic: true, + }, + { + name: "invalid count: > 256", + count: 257, + expectPanic: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.expectPanic { + defer func() { + if r := recover(); r == nil { + t.Errorf("expected panic but didn't get one") + } + }() + GenerateRandomSubnets(tt.count) + return + } + + subnets := GenerateRandomSubnets(tt.count) + + if tt.validateFunc != nil { + if err := tt.validateFunc(subnets); err != nil { + t.Errorf("validation failed: %v", err) + } + } + + // Common validations for all non-panic cases + if err := validateSubnetFormat(subnets); err != nil { + t.Errorf("subnet format validation failed: %v", err) + } + + if err := validateSubnetUniqueness(subnets); err != nil { + t.Errorf("subnet uniqueness validation failed: %v", err) + } + + if err := validateSubnetRange(subnets); err != nil { + t.Errorf("subnet range validation failed: %v", err) + } + }) + } +} + +func TestGenerateRandomSubnetsUniqueness(t *testing.T) { + // Test that multiple invocations produce different results + iterations := 10 + allSubnets := make(map[string]int) + + for i := 0; i < iterations; i++ { + subnets := GenerateRandomSubnets(2) + for _, subnet := range subnets { + allSubnets[subnet]++ + } + } + + // We expect to see some variety (not all the same) + if len(allSubnets) < 3 { + t.Errorf("expected at least 3 different subnets across %d iterations, got %d: %v", + iterations, len(allSubnets), allSubnets) + } + + t.Logf("Generated %d unique subnets across %d iterations", len(allSubnets), iterations) +} + +func TestGenerateRandomSubnetsDistribution(t *testing.T) { + // Test randomness distribution + count := 100 + subnets := make([]string, count) + for i := 0; i < count; i++ { + result := GenerateRandomSubnets(1) + subnets[i] = result[0] + } + + uniqueSubnets := make(map[string]bool) + for _, subnet := range subnets { + uniqueSubnets[subnet] = true + } + + uniqueCount := len(uniqueSubnets) + // Expect at least 50% uniqueness in 100 iterations + minExpected := count / 2 + if uniqueCount < minExpected { + t.Errorf("poor randomness: expected at least %d unique subnets in %d iterations, got %d", + minExpected, count, uniqueCount) + } + + t.Logf("Randomness check: %d unique subnets out of %d iterations (%.1f%%)", + uniqueCount, count, float64(uniqueCount)/float64(count)*100) +} + +func TestGenerateRandomSubnetsNoCollision(t *testing.T) { + // Test that generating 2 subnets never produces collision within a single call + iterations := 100 + for i := 0; i < iterations; i++ { + subnets := GenerateRandomSubnets(2) + if subnets[0] == subnets[1] { + t.Errorf("iteration %d: collision detected: %s == %s", i, subnets[0], subnets[1]) + } + } + t.Logf("No collisions detected in %d iterations", iterations) +} + +func TestSubnetFormatConsistency(t *testing.T) { + // Test that all generated subnets follow the expected format + subnets := GenerateRandomSubnets(10) + expectedPattern := regexp.MustCompile(`^172\.28\.\d{1,3}\.0/24$`) + + for i, subnet := range subnets { + if !expectedPattern.MatchString(subnet) { + t.Errorf("subnet %d has invalid format: %s", i, subnet) + } + + // Verify the third octet is in valid range (0-255) + parts := strings.Split(strings.TrimSuffix(subnet, "/24"), ".") + if len(parts) != 4 { + t.Errorf("subnet %d has invalid structure: %s", i, subnet) + continue + } + + var thirdOctet int + if _, err := fmt.Sscanf(parts[2], "%d", &thirdOctet); err != nil { + t.Errorf("subnet %d has invalid third octet: %s", i, parts[2]) + continue + } + + if thirdOctet < 0 || thirdOctet > 255 { + t.Errorf("subnet %d has third octet out of range: %d", i, thirdOctet) + } + } +} + +// Helper functions for validation + +func validateSubnetFormat(subnets []string) error { + pattern := regexp.MustCompile(`^172\.28\.\d{1,3}\.0/24$`) + for i, subnet := range subnets { + if !pattern.MatchString(subnet) { + return fmt.Errorf("subnet %d has invalid format: %s (expected 172.28.X.0/24)", i, subnet) + } + } + return nil +} + +func validateSubnetUniqueness(subnets []string) error { + seen := make(map[string]bool) + for i, subnet := range subnets { + if seen[subnet] { + return fmt.Errorf("duplicate subnet found at index %d: %s", i, subnet) + } + seen[subnet] = true + } + return nil +} + +func validateSubnetRange(subnets []string) error { + for i, subnet := range subnets { + parts := strings.Split(strings.TrimSuffix(subnet, "/24"), ".") + if len(parts) != 4 { + return fmt.Errorf("subnet %d has invalid structure: %s", i, subnet) + } + + // Validate base (172.28) + if parts[0] != "172" || parts[1] != "28" { + return fmt.Errorf("subnet %d has invalid base: %s.%s (expected 172.28)", i, parts[0], parts[1]) + } + + // Validate third octet (0-255) + var thirdOctet int + if _, err := fmt.Sscanf(parts[2], "%d", &thirdOctet); err != nil { + return fmt.Errorf("subnet %d has invalid third octet: %s", i, parts[2]) + } + if thirdOctet < 0 || thirdOctet > 255 { + return fmt.Errorf("subnet %d has third octet out of range: %d (must be 0-255)", i, thirdOctet) + } + + // Validate fourth octet (must be 0) + if parts[3] != "0" { + return fmt.Errorf("subnet %d has invalid fourth octet: %s (expected 0)", i, parts[3]) + } + } + return nil +} + +// Benchmark tests + +func BenchmarkGenerateRandomSubnets1(b *testing.B) { + for i := 0; i < b.N; i++ { + GenerateRandomSubnets(1) + } +} + +func BenchmarkGenerateRandomSubnets2(b *testing.B) { + for i := 0; i < b.N; i++ { + GenerateRandomSubnets(2) + } +} + +func BenchmarkGenerateRandomSubnets10(b *testing.B) { + for i := 0; i < b.N; i++ { + GenerateRandomSubnets(10) + } +} + +func BenchmarkGenerateRandomSubnets100(b *testing.B) { + for i := 0; i < b.N; i++ { + GenerateRandomSubnets(100) + } +} diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 48bd26efb0f..af760eab2f7 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -31,9 +31,11 @@ import ( "github.com/kubeovn/kube-ovn/test/e2e/framework/kind" ) -// Docker network names will be initialized in init() with random suffix to avoid conflicts +// Docker network configurations will be initialized in init() with random names and subnets var dockerNetworkName string var dockerExtraNetworkName string +var dockerNetworkSubnet string +var dockerExtraNetworkSubnet string func makeProviderNetwork(providerNetworkName string, exchangeLinkName bool, linkMap map[string]*iproute.Link) *kubeovnv1.ProviderNetwork { var defaultInterface string @@ -201,15 +203,15 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { } if dockerNetwork == nil { - ginkgo.By("Ensuring docker network " + dockerNetworkName + " exists") - network, err := docker.NetworkCreate(dockerNetworkName, true, true) + ginkgo.By("Ensuring docker network " + dockerNetworkName + " with subnet " + dockerNetworkSubnet + " exists") + network, err := docker.NetworkCreateWithSubnet(dockerNetworkName, dockerNetworkSubnet, true, true) framework.ExpectNoError(err, "ensuring docker network "+dockerNetworkName+" exists") dockerNetwork = network } if dockerExtraNetwork == nil { - ginkgo.By("Ensuring extra docker network " + dockerExtraNetworkName + " exists") - network, err := docker.NetworkCreate(dockerExtraNetworkName, true, true) + ginkgo.By("Ensuring extra docker network " + dockerExtraNetworkName + " with subnet " + dockerExtraNetworkSubnet + " exists") + network, err := docker.NetworkCreateWithSubnet(dockerExtraNetworkName, dockerExtraNetworkSubnet, true, true) framework.ExpectNoError(err, "ensuring extra docker network "+dockerExtraNetworkName+" exists") dockerExtraNetwork = network } @@ -1349,11 +1351,18 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { }) func init() { - // Generate unique network names for this test run to avoid conflicts with previous runs + // Generate unique network names and subnets for this test run + // This avoids conflicts with other parallel test runs on the same host suffix := framework.RandomSuffix() dockerNetworkName = "kube-ovn-vlan-" + suffix dockerExtraNetworkName = "kube-ovn-extra-vlan-" + suffix + // Generate random /24 subnets within 172.28.0.0/16 + // This allows up to 256 parallel test runs on the same host + subnets := docker.GenerateRandomSubnets(2) + dockerNetworkSubnet = subnets[0] + dockerExtraNetworkSubnet = subnets[1] + klog.SetOutput(ginkgo.GinkgoWriter) // Register flags. From 3ee1d7089e703b36e525f45deb3fd30b908eab6b Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Sun, 14 Dec 2025 14:55:38 +0800 Subject: [PATCH 13/30] fix lint Signed-off-by: zbb88888 --- test/e2e/framework/docker/network.go | 4 ++-- test/e2e/framework/docker/network_test.go | 8 ++++---- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 10 ++++++---- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/test/e2e/framework/docker/network.go b/test/e2e/framework/docker/network.go index 04beb1a4d20..9f33a38378b 100644 --- a/test/e2e/framework/docker/network.go +++ b/test/e2e/framework/docker/network.go @@ -110,7 +110,7 @@ func NetworkInspect(name string) (*network.Inspect, error) { } // NetworkCreateWithSubnet creates a docker network with specified IPv4 subnet -func NetworkCreateWithSubnet(name string, ipv4Subnet string, ipv6, skipIfExists bool) (*network.Inspect, error) { +func NetworkCreateWithSubnet(name, ipv4Subnet string, ipv6, skipIfExists bool) (*network.Inspect, error) { if skipIfExists { network, err := getNetwork(name, true) if err != nil { @@ -137,7 +137,7 @@ func NetworkCreateWithSubnet(name string, ipv4Subnet string, ipv6, skipIfExists if ipv4Subnet != "" { gateway, err := util.FirstIP(ipv4Subnet) if err != nil { - return nil, fmt.Errorf("failed to get gateway for subnet %s: %v", ipv4Subnet, err) + return nil, fmt.Errorf("failed to get gateway for subnet %s: %w", ipv4Subnet, err) } config := network.IPAMConfig{ Subnet: netip.MustParsePrefix(ipv4Subnet), diff --git a/test/e2e/framework/docker/network_test.go b/test/e2e/framework/docker/network_test.go index 7c854c46c07..6c6094b27a6 100644 --- a/test/e2e/framework/docker/network_test.go +++ b/test/e2e/framework/docker/network_test.go @@ -9,10 +9,10 @@ import ( func TestGenerateRandomSubnets(t *testing.T) { tests := []struct { - name string - count int - expectPanic bool - validateFunc func([]string) error + name string + count int + expectPanic bool + validateFunc func([]string) error }{ { name: "generate 1 subnet", diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index af760eab2f7..c37e06dd01c 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -32,10 +32,12 @@ import ( ) // Docker network configurations will be initialized in init() with random names and subnets -var dockerNetworkName string -var dockerExtraNetworkName string -var dockerNetworkSubnet string -var dockerExtraNetworkSubnet string +var ( + dockerNetworkName string + dockerExtraNetworkName string + dockerNetworkSubnet string + dockerExtraNetworkSubnet string +) func makeProviderNetwork(providerNetworkName string, exchangeLinkName bool, linkMap map[string]*iproute.Link) *kubeovnv1.ProviderNetwork { var defaultInterface string From b7a6ca2bf453da606796fdf4db0900823016f861 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Sun, 14 Dec 2025 18:35:01 +0800 Subject: [PATCH 14/30] add test Signed-off-by: zbb88888 --- pkg/controller/controller_test.go | 2 + pkg/controller/external_gw_test.go | 382 +++++++++++++++++++++++++++++ 2 files changed, 384 insertions(+) create mode 100644 pkg/controller/external_gw_test.go diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 7a6bcdc5057..2457fdbdca7 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -117,6 +117,7 @@ func newFakeControllerWithOptions(t *testing.T, opts *FakeControllerOptions) (*f serviceInformer := kubeInformerFactory.Core().V1().Services() namespaceInformer := kubeInformerFactory.Core().V1().Namespaces() podInformer := kubeInformerFactory.Core().V1().Pods() + configMapInformer := kubeInformerFactory.Core().V1().ConfigMaps() nadInformerFactory := nadinformers.NewSharedInformerFactory(nadClient, 0) nadInformer := nadInformerFactory.K8sCniCncfIo().V1().NetworkAttachmentDefinitions() @@ -149,6 +150,7 @@ func newFakeControllerWithOptions(t *testing.T, opts *FakeControllerOptions) (*f subnetSynced: alwaysReady, netAttachLister: nadInformer.Lister(), netAttachSynced: alwaysReady, + configMapsLister: configMapInformer.Lister(), OVNNbClient: mockOvnClient, syncVirtualPortsQueue: newTypedRateLimitingQueue[string]("SyncVirtualPort", nil), } diff --git a/pkg/controller/external_gw_test.go b/pkg/controller/external_gw_test.go new file mode 100644 index 00000000000..a35a37ed4de --- /dev/null +++ b/pkg/controller/external_gw_test.go @@ -0,0 +1,382 @@ +package controller + +import ( + "context" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + kubeovnfake "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/fake" + kubeovninformers "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// Helper function to create a test controller with optional initial objects +func newTestController(subnets []*kubeovnv1.Subnet, configMaps []*corev1.ConfigMap) *Controller { + // Create clientsets + var kubeObjects []runtime.Object + for _, cm := range configMaps { + kubeObjects = append(kubeObjects, cm) + } + kubeClient := fake.NewSimpleClientset(kubeObjects...) + + var kubeovnObjects []runtime.Object + for _, subnet := range subnets { + kubeovnObjects = append(kubeovnObjects, subnet) + } + kubeOvnClient := kubeovnfake.NewSimpleClientset(kubeovnObjects...) + + // Create informer factories + kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, 0) + kubeovnInformerFactory := kubeovninformers.NewSharedInformerFactory(kubeOvnClient, 0) + + config := &Configuration{ + KubeClient: kubeClient, + KubeOvnClient: kubeOvnClient, + ExternalGatewaySwitch: "external", // Default name + ExternalGatewayConfigNS: "kube-system", + } + + controller := &Controller{ + config: config, + configMapsLister: kubeInformerFactory.Core().V1().ConfigMaps().Lister(), + subnetsLister: kubeovnInformerFactory.Kubeovn().V1().Subnets().Lister(), + } + + // Start informers and wait for cache sync + stopCh := make(chan struct{}) + defer close(stopCh) + + kubeInformerFactory.Start(stopCh) + kubeovnInformerFactory.Start(stopCh) + + kubeInformerFactory.WaitForCacheSync(stopCh) + kubeovnInformerFactory.WaitForCacheSync(stopCh) + + // Give informers time to sync + time.Sleep(100 * time.Millisecond) + + return controller +} + +// Test Scenario 1: Default "external" subnet does NOT exist, ConfigMap NOT specified +// Expected: Return default name "external" (will fail later with subnet not found) +func TestGetExternalGatewaySwitch_Scenario1_DefaultNotExist_ConfigMapNotSpecified(t *testing.T) { + c := newTestController(nil, nil) + + result, err := c.getExternalGatewaySwitch() + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + + expected := "external" + if result != expected { + t.Errorf("expected %s, got %s", expected, result) + } +} + +// Test Scenario 2: Default "external" subnet EXISTS, ConfigMap NOT specified +// Expected: Use default "external" +func TestGetExternalGatewaySwitch_Scenario2_DefaultExists_ConfigMapNotSpecified(t *testing.T) { + defaultSubnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "external", + }, + } + + c := newTestController([]*kubeovnv1.Subnet{defaultSubnet}, nil) + + result, err := c.getExternalGatewaySwitch() + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + + expected := "external" + if result != expected { + t.Errorf("expected %s, got %s", expected, result) + } +} + +// Test Scenario 3: Default "external" subnet does NOT exist, ConfigMap specifies "custom-ext" +// Expected: Use "custom-ext" +func TestGetExternalGatewaySwitch_Scenario3_DefaultNotExist_ConfigMapSpecifiedDifferent(t *testing.T) { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.ExternalGatewayConfig, + Namespace: "kube-system", + }, + Data: map[string]string{ + "enable-external-gw": "true", + "external-gw-switch": "custom-ext", + }, + } + + c := newTestController(nil, []*corev1.ConfigMap{configMap}) + + result, err := c.getExternalGatewaySwitch() + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + + expected := "custom-ext" + if result != expected { + t.Errorf("expected %s, got %s", expected, result) + } +} + +// Test Scenario 4: Default "external" subnet EXISTS, ConfigMap specifies "custom-ext" (different) +// Expected: ERROR - configuration conflict +func TestGetExternalGatewaySwitch_Scenario4_DefaultExists_ConfigMapSpecifiedDifferent(t *testing.T) { + defaultSubnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "external", + }, + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.ExternalGatewayConfig, + Namespace: "kube-system", + }, + Data: map[string]string{ + "enable-external-gw": "true", + "external-gw-switch": "custom-ext", + }, + } + + c := newTestController([]*kubeovnv1.Subnet{defaultSubnet}, []*corev1.ConfigMap{configMap}) + + _, err := c.getExternalGatewaySwitch() + if err == nil { + t.Error("expected error due to configuration conflict, but got nil") + return + } + + expectedErrMsg := "configuration conflict" + if !strings.Contains(err.Error(), expectedErrMsg) { + t.Errorf("expected error message to contain '%s', got: %v", expectedErrMsg, err) + } +} + +// Test Scenario 5: Default "external" subnet EXISTS, ConfigMap specifies "external" (same name) +// Expected: Use default "external" (no conflict) +func TestGetExternalGatewaySwitch_Scenario5_DefaultExists_ConfigMapSpecifiedSame(t *testing.T) { + defaultSubnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "external", + }, + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.ExternalGatewayConfig, + Namespace: "kube-system", + }, + Data: map[string]string{ + "enable-external-gw": "true", + "external-gw-switch": "external", + }, + } + + c := newTestController([]*kubeovnv1.Subnet{defaultSubnet}, []*corev1.ConfigMap{configMap}) + + result, err := c.getExternalGatewaySwitch() + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + + expected := "external" + if result != expected { + t.Errorf("expected %s, got %s", expected, result) + } +} + +// Test ConfigMap disabled: enable-external-gw = false +// Expected: Return default regardless of ConfigMap content +func TestGetExternalGatewaySwitch_ConfigMapDisabled(t *testing.T) { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.ExternalGatewayConfig, + Namespace: "kube-system", + }, + Data: map[string]string{ + "enable-external-gw": "false", + "external-gw-switch": "custom-ext", // Should be ignored + }, + } + + c := newTestController(nil, []*corev1.ConfigMap{configMap}) + + result, err := c.getExternalGatewaySwitch() + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + + expected := "external" + if result != expected { + t.Errorf("expected %s (should ignore ConfigMap when disabled), got %s", expected, result) + } +} + +// Test getExternalGatewaySwitchWithConfigMap directly +func TestGetExternalGatewaySwitchWithConfigMap_AllScenarios(t *testing.T) { + tests := []struct { + name string + defaultExists bool + configSwitch string + expectedResult string + expectError bool + errorMsgContains string + }{ + { + name: "Scenario 1: default not exist, config not specified", + defaultExists: false, + configSwitch: "", + expectedResult: "external", + expectError: false, + }, + { + name: "Scenario 2: default exists, config not specified", + defaultExists: true, + configSwitch: "", + expectedResult: "external", + expectError: false, + }, + { + name: "Scenario 3: default not exist, config specified", + defaultExists: false, + configSwitch: "custom-ext", + expectedResult: "custom-ext", + expectError: false, + }, + { + name: "Scenario 4: default exists, config specified different", + defaultExists: true, + configSwitch: "custom-ext", + expectedResult: "", + expectError: true, + errorMsgContains: "configuration conflict", + }, + { + name: "Scenario 5: default exists, config specified same", + defaultExists: true, + configSwitch: "external", + expectedResult: "external", + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var subnets []*kubeovnv1.Subnet + if tt.defaultExists { + subnets = []*kubeovnv1.Subnet{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "external", + }, + }, + } + } + + c := newTestController(subnets, nil) + + configData := map[string]string{} + if tt.configSwitch != "" { + configData["external-gw-switch"] = tt.configSwitch + } + + result, err := c.getExternalGatewaySwitchWithConfigMap(configData) + + if tt.expectError { + if err == nil { + t.Error("expected error but got nil") + return + } + if tt.errorMsgContains != "" && !strings.Contains(err.Error(), tt.errorMsgContains) { + t.Errorf("expected error containing '%s', got: %v", tt.errorMsgContains, err) + } + } else { + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + if result != tt.expectedResult { + t.Errorf("expected result %s, got %s", tt.expectedResult, result) + } + } + }) + } +} + +// Test with actual Kubernetes client operations +func TestGetExternalGatewaySwitch_WithClientOperations(t *testing.T) { + kubeClient := fake.NewSimpleClientset() + kubeOvnClient := kubeovnfake.NewSimpleClientset() + + kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, 0) + kubeovnInformerFactory := kubeovninformers.NewSharedInformerFactory(kubeOvnClient, 0) + + config := &Configuration{ + KubeClient: kubeClient, + KubeOvnClient: kubeOvnClient, + ExternalGatewaySwitch: "external", + ExternalGatewayConfigNS: "kube-system", + } + + controller := &Controller{ + config: config, + configMapsLister: kubeInformerFactory.Core().V1().ConfigMaps().Lister(), + subnetsLister: kubeovnInformerFactory.Kubeovn().V1().Subnets().Lister(), + } + + stopCh := make(chan struct{}) + defer close(stopCh) + + kubeInformerFactory.Start(stopCh) + kubeovnInformerFactory.Start(stopCh) + + // Create subnet after informer starts + subnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "external", + }, + } + _, err := kubeOvnClient.KubeovnV1().Subnets().Create(context.Background(), subnet, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create subnet: %v", err) + } + + // Create ConfigMap + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.ExternalGatewayConfig, + Namespace: "kube-system", + }, + Data: map[string]string{ + "enable-external-gw": "true", + "external-gw-switch": "custom-ext", + }, + } + _, err = kubeClient.CoreV1().ConfigMaps("kube-system").Create(context.Background(), configMap, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create configmap: %v", err) + } + + // Wait for informer cache sync + time.Sleep(200 * time.Millisecond) + + // This should return error due to conflict + _, err = controller.getExternalGatewaySwitch() + if err == nil { + t.Error("expected configuration conflict error, got nil") + } else if !strings.Contains(err.Error(), "configuration conflict") { + t.Errorf("expected configuration conflict error, got: %v", err) + } +} From efb6738ea0df1336ae7804ffac279ece45a8acb0 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Sun, 14 Dec 2025 19:07:49 +0800 Subject: [PATCH 15/30] simple default external subnet Signed-off-by: zbb88888 --- pkg/controller/external_gw.go | 85 +++++++++++++++++++++++--- pkg/controller/external_vpc.go | 23 ++++--- pkg/controller/vpc.go | 95 ++++++++++++++++++++++------- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 67 ++++++++++---------- 4 files changed, 193 insertions(+), 77 deletions(-) diff --git a/pkg/controller/external_gw.go b/pkg/controller/external_gw.go index 34c07f20658..fb95f7341a4 100644 --- a/pkg/controller/external_gw.go +++ b/pkg/controller/external_gw.go @@ -145,8 +145,16 @@ func (c *Controller) establishExternalGateway(config map[string]string) error { klog.Errorf("failed to get gateway chassis, %v", err) return err } + + // Get external gateway switch using centralized logic + externalGwSwitch, err := c.getExternalGatewaySwitchWithConfigMap(config) + if err != nil { + klog.Errorf("failed to get external gateway switch: %v", err) + return err + } + var lrpIP, lrpMac string - lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch) + lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, externalGwSwitch) lrp, err := c.OVNNbClient.GetLogicalRouterPort(lrpName, true) if err != nil { klog.Errorf("failed to get lrp %s, %v", lrpName, err) @@ -159,7 +167,7 @@ func (c *Controller) establishExternalGateway(config map[string]string) error { lrpMac = lrp.MAC lrpIP = lrp.Networks[0] case config["nic-ip"] == "": - if lrpIP, lrpMac, err = c.createDefaultVpcLrpEip(); err != nil { + if lrpIP, lrpMac, err = c.createDefaultVpcLrpEip(externalGwSwitch); err != nil { klog.Errorf("failed to create ovn eip for default vpc lrp: %v", err) return err } @@ -168,22 +176,22 @@ func (c *Controller) establishExternalGateway(config map[string]string) error { lrpMac = config["nic-mac"] } - if err := c.OVNNbClient.CreateGatewayLogicalSwitch(c.config.ExternalGatewaySwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIP, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil { - klog.Errorf("failed to create external gateway switch %s: %v", c.config.ExternalGatewaySwitch, err) + if err := c.OVNNbClient.CreateGatewayLogicalSwitch(externalGwSwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIP, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil { + klog.Errorf("failed to create external gateway switch %s: %v", externalGwSwitch, err) return err } return nil } -func (c *Controller) createDefaultVpcLrpEip() (string, string, error) { - cachedSubnet, err := c.subnetsLister.Get(c.config.ExternalGatewaySwitch) +func (c *Controller) createDefaultVpcLrpEip(externalGwSwitch string) (string, string, error) { + cachedSubnet, err := c.subnetsLister.Get(externalGwSwitch) if err != nil { - klog.Errorf("failed to get subnet %s, %v", c.config.ExternalGatewaySwitch, err) + klog.Errorf("failed to get subnet %s, %v", externalGwSwitch, err) return "", "", err } needCreateEip := false - lrpEipName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch) + lrpEipName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, externalGwSwitch) cachedEip, err := c.ovnEipsLister.Get(lrpEipName) if err != nil { if !k8serrors.IsNotFound(err) { @@ -203,12 +211,12 @@ func (c *Controller) createDefaultVpcLrpEip() (string, string, error) { } } else { var v6ip string - v4ip, v6ip, mac, err = c.acquireIPAddress(c.config.ExternalGatewaySwitch, lrpEipName, lrpEipName) + v4ip, v6ip, mac, err = c.acquireIPAddress(externalGwSwitch, lrpEipName, lrpEipName) if err != nil { klog.Errorf("failed to acquire ip address for default vpc lrp %s, %v", lrpEipName, err) return "", "", err } - if err := c.createOrUpdateOvnEipCR(lrpEipName, c.config.ExternalGatewaySwitch, v4ip, v6ip, mac, util.OvnEipTypeLRP); err != nil { + if err := c.createOrUpdateOvnEipCR(lrpEipName, externalGwSwitch, v4ip, v6ip, mac, util.OvnEipTypeLRP); err != nil { klog.Errorf("failed to create ovn lrp eip %s, %v", lrpEipName, err) return "", "", err } @@ -221,6 +229,63 @@ func (c *Controller) createDefaultVpcLrpEip() (string, string, error) { return v4ipCidr, mac, nil } +// getExternalGatewaySwitchWithConfigMap determines which external gateway switch to use. +// Logic (mutually exclusive modes): +// 1. If default "external" subnet exists: +// - ConfigMap not specified or same name -> use default "external" +// - ConfigMap specifies different name -> ERROR (configuration conflict) +// +// 2. If default "external" subnet does NOT exist: +// - ConfigMap specifies a name -> use ConfigMap name +// - ConfigMap not specified -> return default name (will fail later with subnet not found) +func (c *Controller) getExternalGatewaySwitchWithConfigMap(configData map[string]string) (string, error) { + defaultSwitch := c.config.ExternalGatewaySwitch + configSwitch := configData["external-gw-switch"] + + // Check if default subnet exists + _, err := c.subnetsLister.Get(defaultSwitch) + defaultExists := err == nil + + if defaultExists { + // Default subnet exists - MUST use it (backward compatibility) + if configSwitch != "" && configSwitch != defaultSwitch { + // Configuration conflict: both modes specified + return "", fmt.Errorf("configuration conflict: default external subnet %s exists, but ConfigMap specifies different subnet %s. Please use only one mode: either remove the default subnet or remove the ConfigMap setting", defaultSwitch, configSwitch) + } + return defaultSwitch, nil + } + + // Default subnet does NOT exist - check ConfigMap + if configSwitch != "" { + // Use ConfigMap specified subnet + return configSwitch, nil + } + + // Neither default nor ConfigMap specified subnet available + return defaultSwitch, nil // Return default name anyway for error messages +} + +// getExternalGatewaySwitch returns the external gateway switch name by reading from ConfigMap +// Logic: default subnet exists -> use default; default not exists + ConfigMap specified -> use ConfigMap +func (c *Controller) getExternalGatewaySwitch() (string, error) { + cm, err := c.configMapsLister.ConfigMaps(c.config.ExternalGatewayConfigNS).Get(util.ExternalGatewayConfig) + if err != nil { + if k8serrors.IsNotFound(err) { + // ConfigMap doesn't exist, use default + return c.config.ExternalGatewaySwitch, nil + } + return "", fmt.Errorf("failed to get ConfigMap %s: %w", util.ExternalGatewayConfig, err) + } + + // Check if ConfigMap is enabled + if cm.Data["enable-external-gw"] == "false" { + return c.config.ExternalGatewaySwitch, nil + } + + // Use the centralized logic + return c.getExternalGatewaySwitchWithConfigMap(cm.Data) +} + func (c *Controller) getGatewayChassis(config map[string]string) ([]string, error) { chassises := []string{} nodes, err := c.nodesLister.List(externalGatewayNodeSelector) diff --git a/pkg/controller/external_vpc.go b/pkg/controller/external_vpc.go index 877d3cf08bb..6992016bc92 100644 --- a/pkg/controller/external_vpc.go +++ b/pkg/controller/external_vpc.go @@ -27,24 +27,31 @@ func (c *Controller) syncExternalVpc() { return } for _, cachedVpc := range vpcs { - vpc := cachedVpc.DeepCopy() - if _, ok := logicalRouters[vpc.Name]; ok { + vpcName := cachedVpc.Name + if _, ok := logicalRouters[vpcName]; ok { + // Get the latest VPC object before updating status to avoid conflicts + vpc, err := c.config.KubeOvnClient.KubeovnV1().Vpcs().Get(context.Background(), vpcName, metav1.GetOptions{}) + if err != nil { + klog.Errorf("failed to get latest vpc %s: %v", vpcName, err) + continue + } + // update vpc status subnet list vpc.Status.Subnets = []string{} - for _, ls := range logicalRouters[vpc.Name].LogicalSwitches { + for _, ls := range logicalRouters[vpcName].LogicalSwitches { vpc.Status.Subnets = append(vpc.Status.Subnets, ls.Name) } _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().UpdateStatus(context.Background(), vpc, metav1.UpdateOptions{}) if err != nil { - klog.Errorf("failed to update vpc %s status: %v", vpc.Name, err) + klog.Errorf("failed to update vpc %s status: %v", vpcName, err) continue } - delete(logicalRouters, vpc.Name) + delete(logicalRouters, vpcName) } else { - klog.Infof("external vpc %s has no ovn logical router, delete it", vpc.Name) - err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Delete(context.Background(), vpc.Name, metav1.DeleteOptions{}) + klog.Infof("external vpc %s has no ovn logical router, delete it", vpcName) + err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Delete(context.Background(), vpcName, metav1.DeleteOptions{}) if err != nil { - klog.Errorf("failed to delete vpc %s: %v", vpc.Name, err) + klog.Errorf("failed to delete vpc %s: %v", vpcName, err) continue } } diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index 2b47a0323e5..b2d4cb01acd 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -129,14 +129,16 @@ func (c *Controller) handleDelVpc(vpc *kubeovnv1.Vpc) error { return err } - if err := c.handleDelVpcExternalSubnet(vpc.Name, c.config.ExternalGatewaySwitch); err != nil { - klog.Errorf("failed to delete external connection for vpc %s, error %v", vpc.Name, err) + // Delete both ConfigMap-specified and default external gateway switches + // to ensure cleanup even if configuration changed between creation and deletion + if err := c.deleteVpcExternalSubnets(vpc.Name); err != nil { + klog.Errorf("failed to delete external connections for vpc %s: %v", vpc.Name, err) return err } for _, subnet := range vpc.Status.ExtraExternalSubnets { klog.Infof("disconnect external network %s to vpc %s", subnet, vpc.Name) - if err := c.handleDelVpcExternalSubnet(vpc.Name, subnet); err != nil { + if err := c.handleDelVpcRes2ExternalSubnet(vpc.Name, subnet); err != nil { klog.Error(err) return err } @@ -349,20 +351,28 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { return err } + // Determine which external gateway switch to use + // Logic: default subnet exists -> use default; default not exists + ConfigMap specified -> use ConfigMap + externalGwSwitch, err := c.getExternalGatewaySwitch() + if err != nil { + klog.Warningf("failed to get external gateway switch: %v", err) + externalGwSwitch = c.config.ExternalGatewaySwitch // fallback to default + } + var externalSubnet *kubeovnv1.Subnet externalSubnetExist := false externalSubnetGW := "" if c.config.EnableEipSnat { - externalSubnet, err = c.subnetsLister.Get(c.config.ExternalGatewaySwitch) + externalSubnet, err = c.subnetsLister.Get(externalGwSwitch) if err != nil { - klog.Warningf("enable-eip-snat need external subnet %s to be exist: %v", c.config.ExternalGatewaySwitch, err) + klog.Warningf("enable-eip-snat need external subnet %s to exist: %v", externalGwSwitch, err) } else { if !externalSubnet.Spec.LogicalGateway { // logical gw external subnet can not access external externalSubnetExist = true externalSubnetGW = externalSubnet.Spec.Gateway } else { - klog.Infof("default external subnet %s using logical gw", c.config.ExternalGatewaySwitch) + klog.Infof("external subnet %s using logical gw", externalGwSwitch) } } } @@ -416,7 +426,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { nextHop := cm.Data["external-gw-addr"] if nextHop == "" { if !externalSubnetExist { - err = fmt.Errorf("failed to get external subnet %s", c.config.ExternalGatewaySwitch) + err = fmt.Errorf("failed to get external subnet %s", externalGwSwitch) klog.Error(err) return err } @@ -602,7 +612,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { } if vpc.Spec.EnableExternal || vpc.Status.EnableExternal { - if err = c.handleUpdateVpcExternal(cachedVpc, custVpcEnableExternalEcmp, externalSubnetExist, externalSubnetGW); err != nil { + if err = c.handleUpdateVpcExternal(cachedVpc, externalGwSwitch, custVpcEnableExternalEcmp, externalSubnetExist, externalSubnetGW); err != nil { klog.Errorf("failed to handle update external subnet for vpc %s, %v", key, err) return err } @@ -613,17 +623,25 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { klog.Error(err) return err } + + // Get the latest VPC object before updating status to avoid conflicts + latestVpc, err := c.config.KubeOvnClient.KubeovnV1().Vpcs().Get(context.Background(), key, metav1.GetOptions{}) + if err != nil { + klog.Errorf("failed to get latest vpc %s: %v", key, err) + return err + } + if vpc.Spec.BFDPort == nil || !vpc.Spec.BFDPort.Enabled { - vpc.Status.BFDPort = kubeovnv1.BFDPortStatus{} + latestVpc.Status.BFDPort = kubeovnv1.BFDPortStatus{} } else { - vpc.Status.BFDPort = kubeovnv1.BFDPortStatus{ + latestVpc.Status.BFDPort = kubeovnv1.BFDPortStatus{ Name: bfdPortName, IP: vpc.Spec.BFDPort.IP, Nodes: bfdPortNodes, } } if _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs(). - UpdateStatus(context.Background(), vpc, metav1.UpdateOptions{}); err != nil { + UpdateStatus(context.Background(), latestVpc, metav1.UpdateOptions{}); err != nil { klog.Error(err) return err } @@ -631,7 +649,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { return nil } -func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, custVpcEnableExternalEcmp, defaultExternalSubnetExist bool, externalSubnetGW string) error { +func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, externalGwSwitch string, custVpcEnableExternalEcmp, defaultExternalSubnetExist bool, externalSubnetGW string) error { if c.config.EnableEipSnat && vpc.Name == util.DefaultVpc { klog.Infof("external_gw handle ovn default external gw %s", vpc.Name) return nil @@ -652,14 +670,14 @@ func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, custVpcEnableEx if !vpc.Spec.EnableExternal && vpc.Status.EnableExternal { // just del all external subnets connection - klog.Infof("disconnect default external subnet %s to vpc %s", c.config.ExternalGatewaySwitch, vpc.Name) - if err := c.handleDelVpcExternalSubnet(vpc.Name, c.config.ExternalGatewaySwitch); err != nil { - klog.Errorf("failed to delete external subnet %s connection for vpc %s, error %v", c.config.ExternalGatewaySwitch, vpc.Name, err) + klog.Infof("disconnect default external subnet %s to vpc %s", externalGwSwitch, vpc.Name) + if err := c.handleDelVpcRes2ExternalSubnet(vpc.Name, externalGwSwitch); err != nil { + klog.Errorf("failed to delete external subnet %s connection for vpc %s, error %v", externalGwSwitch, vpc.Name, err) return err } for _, subnet := range vpc.Status.ExtraExternalSubnets { klog.Infof("disconnect external subnet %s to vpc %s", subnet, vpc.Name) - if err := c.handleDelVpcExternalSubnet(vpc.Name, subnet); err != nil { + if err := c.handleDelVpcRes2ExternalSubnet(vpc.Name, subnet); err != nil { klog.Errorf("failed to delete external subnet %s connection for vpc %s, error %v", subnet, vpc.Name, err) return err } @@ -671,9 +689,9 @@ func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, custVpcEnableEx // just add external connection if vpc.Spec.ExtraExternalSubnets == nil && defaultExternalSubnetExist { // only connect default external subnet - klog.Infof("connect default external subnet %s with vpc %s", c.config.ExternalGatewaySwitch, vpc.Name) - if err := c.handleAddVpcExternalSubnet(vpc.Name, c.config.ExternalGatewaySwitch); err != nil { - klog.Errorf("failed to add external subnet %s connection for vpc %s, error %v", c.config.ExternalGatewaySwitch, vpc.Name, err) + klog.Infof("connect default external subnet %s with vpc %s", externalGwSwitch, vpc.Name) + if err := c.handleAddVpcExternalSubnet(vpc.Name, externalGwSwitch); err != nil { + klog.Errorf("failed to add external subnet %s connection for vpc %s, error %v", externalGwSwitch, vpc.Name, err) return err } } @@ -703,7 +721,7 @@ func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, custVpcEnableEx for _, subnet := range vpc.Status.ExtraExternalSubnets { if !slices.Contains(vpc.Spec.ExtraExternalSubnets, subnet) { klog.Infof("disconnect external subnet %s to vpc %s", subnet, vpc.Name) - if err := c.handleDelVpcExternalSubnet(vpc.Name, subnet); err != nil { + if err := c.handleDelVpcRes2ExternalSubnet(vpc.Name, subnet); err != nil { klog.Errorf("failed to delete external subnet %s connection for vpc %s, error %v", subnet, vpc.Name, err) return err } @@ -716,7 +734,7 @@ func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, custVpcEnableEx // create bfd between lrp and physical switch gw // bfd status down means current lrp binding chassis node external nic lost external network connectivity // should switch lrp to another node - lrpEipName := fmt.Sprintf("%s-%s", vpc.Name, c.config.ExternalGatewaySwitch) + lrpEipName := fmt.Sprintf("%s-%s", vpc.Name, externalGwSwitch) v4ExtGw, _ := util.SplitStringIP(externalSubnetGW) // TODO: dualstack if _, err := c.OVNNbClient.CreateBFD(lrpEipName, v4ExtGw, c.config.BfdMinRx, c.config.BfdMinTx, c.config.BfdDetectMult, nil); err != nil { @@ -735,7 +753,7 @@ func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, custVpcEnableEx } if !vpc.Spec.EnableBfd && vpc.Status.EnableBfd { - lrpEipName := fmt.Sprintf("%s-%s", vpc.Name, c.config.ExternalGatewaySwitch) + lrpEipName := fmt.Sprintf("%s-%s", vpc.Name, externalGwSwitch) if err := c.OVNNbClient.DeleteBFDByDstIP(lrpEipName, ""); err != nil { klog.Error(err) return err @@ -1427,7 +1445,7 @@ func (c *Controller) handleDeleteVpcStaticRoute(key string) error { return nil } -func (c *Controller) handleDelVpcExternalSubnet(key, subnet string) error { +func (c *Controller) handleDelVpcRes2ExternalSubnet(key, subnet string) error { lspName := fmt.Sprintf("%s-%s", subnet, key) lrpName := fmt.Sprintf("%s-%s", key, subnet) klog.Infof("delete vpc lrp %s", lrpName) @@ -1514,3 +1532,34 @@ func (c *Controller) updateVpcExternalStatus(key string, enableExternal bool) er return nil } + +// deleteVpcExternalSubnets deletes both ConfigMap-specified and default external gateway switches +// to ensure complete cleanup even if configuration changed between VPC creation and deletion +func (c *Controller) deleteVpcExternalSubnets(vpcName string) error { + // Try to get ConfigMap-specified switch and delete it + externalGwSwitch, err := c.getExternalGatewaySwitch() + if err != nil { + klog.Warningf("failed to get external gateway switch during deletion: %v", err) + // Fallback to default if we can't determine ConfigMap setting + externalGwSwitch = c.config.ExternalGatewaySwitch + } + + var anyErr error + if err := c.handleDelVpcRes2ExternalSubnet(vpcName, externalGwSwitch); err != nil { + klog.Errorf("failed to delete external connection %s for vpc %s: %v", externalGwSwitch, vpcName, err) + anyErr = err + } + + // Also try to delete default switch if it's different from ConfigMap-specified + // This ensures cleanup even if configuration changed + if externalGwSwitch != c.config.ExternalGatewaySwitch { + if err := c.handleDelVpcRes2ExternalSubnet(vpcName, c.config.ExternalGatewaySwitch); err != nil { + klog.Errorf("failed to delete default external connection %s for vpc %s: %v", c.config.ExternalGatewaySwitch, vpcName, err) + if anyErr == nil { + anyErr = err + } + } + } + + return anyErr +} diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index c37e06dd01c..1b7c9d04c5d 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -77,6 +77,23 @@ func makeOvnDnat(name, ovnEip, ipType, ipName, vpc, v4Ip, internalPort, external return framework.MakeOvnDnatRule(name, ovnEip, ipType, ipName, vpc, v4Ip, internalPort, externalPort, protocol) } +func makeExternalGatewayConfigMap(gwNodes, switchName string, cidr []string, gwType string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.ExternalGatewayConfig, + Namespace: framework.KubeOvnNamespace, + }, + Data: map[string]string{ + "enable-external-gw": "true", + "external-gw-nodes": gwNodes, + "external-gw-switch": switchName, + "type": gwType, + "external-gw-nic": "eth1", + "external-gw-addr": strings.Join(cidr, ","), + }, + } +} + var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { f := framework.NewDefaultFramework("ovn-vpc-nat-gw") @@ -133,6 +150,8 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { // in this env, tcpdump gw nat flows will be more clear randomSuffix := framework.RandomSuffix() + // Use first 6 digits of random suffix for provider network names (12 byte limit) + shortSuffix := randomSuffix[:6] noBfdVpcName = "no-bfd-vpc-" + randomSuffix bfdVpcName = "bfd-vpc-" + randomSuffix @@ -145,14 +164,16 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { lrpEipSnatName = "lrp-eip-snat-" + randomSuffix lrpExtraEipSnatName = "lrp-extra-eip-snat-" + randomSuffix bfdSubnetName = "bfd-subnet-" + randomSuffix - // provider network name has 12 bytes limit, use short prefix + // Default external network must use fixed "external" name for global reuse + // kube-ovn-cni creates default external subnet with name "external" when enable-eip-snat providerNetworkName = "external" - providerExtraNetworkName = "extra" - vlanName = "vlan-" + randomSuffix - vlanExtraName = "vlan-extra-" + randomSuffix - // underlay subnet names use fixed names for global reuse underlaySubnetName = "external" - underlayExtraSubnetName = "extra" + vlanName = "vlan" + // Extra network uses random suffix to test multi-external-subnet scenarios + // provider network name has 12 bytes limit, use short suffix + providerExtraNetworkName = "extra-" + shortSuffix + underlayExtraSubnetName = "extra-" + shortSuffix + vlanExtraName = "vlan-extra-" + randomSuffix // sharing case sharedVipName = "shared-vip-" + randomSuffix @@ -600,20 +621,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { externalGwNodes := strings.Join(gwNodeNames, ",") ginkgo.By("Creating config map ovn-external-gw-config for centralized case") - cmData := map[string]string{ - "enable-external-gw": "true", - "external-gw-nodes": externalGwNodes, - "type": kubeovnv1.GWCentralizedType, - "external-gw-nic": "eth1", - "external-gw-addr": strings.Join(cidr, ","), - } - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: util.ExternalGatewayConfig, - Namespace: framework.KubeOvnNamespace, - }, - Data: cmData, - } + configMap := makeExternalGatewayConfigMap(externalGwNodes, underlaySubnetName, cidr, kubeovnv1.GWCentralizedType) _, err = cs.CoreV1().ConfigMaps(framework.KubeOvnNamespace).Create(context.Background(), configMap, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create") @@ -1000,21 +1008,8 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { _ = podClient.CreateSync(pod) } ginkgo.By("3. Updating config map ovn-external-gw-config for distributed case") - cmData = map[string]string{ - "enable-external-gw": "true", - "external-gw-nodes": externalGwNodes, - "type": kubeovnv1.GWDistributedType, - "external-gw-nic": "eth1", - "external-gw-addr": strings.Join(cidr, ","), - } // TODO:// external-gw-nodes could be auto managed by recognizing gw chassis node which has the external-gw-nic - configMap = &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: util.ExternalGatewayConfig, - Namespace: framework.KubeOvnNamespace, - }, - Data: cmData, - } + configMap = makeExternalGatewayConfigMap(externalGwNodes, underlaySubnetName, cidr, kubeovnv1.GWDistributedType) _, err = cs.CoreV1().ConfigMaps(framework.KubeOvnNamespace).Update(context.Background(), configMap, metav1.UpdateOptions{}) framework.ExpectNoError(err, "failed to update ConfigMap") @@ -1356,8 +1351,8 @@ func init() { // Generate unique network names and subnets for this test run // This avoids conflicts with other parallel test runs on the same host suffix := framework.RandomSuffix() - dockerNetworkName = "kube-ovn-vlan-" + suffix - dockerExtraNetworkName = "kube-ovn-extra-vlan-" + suffix + dockerNetworkName = "kube-ovn-vlan0-" + suffix + dockerExtraNetworkName = "kube-ovn-extra-vlan0-" + suffix // Generate random /24 subnets within 172.28.0.0/16 // This allows up to 256 parallel test runs on the same host From c308438dfde24c26307fd8ee266af3c5df296543 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Sun, 14 Dec 2025 20:47:56 +0800 Subject: [PATCH 16/30] make sure create it self Signed-off-by: zbb88888 --- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 92 ++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 9 deletions(-) diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 1b7c9d04c5d..eba4492b01e 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -293,15 +293,68 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { ginkgo.GinkgoHelper() ginkgo.By("Getting or creating provider network " + providerNetworkName) - // Try to get existing provider network first - pn, err := providerNetworkClient.ProviderNetworkInterface.Get(context.Background(), providerNetworkName, metav1.GetOptions{}) - if err != nil && k8serrors.IsNotFound(err) { - // Provider network doesn't exist, create it - pn = makeProviderNetwork(providerNetworkName, exchangeLinkName, linkMap) - pn = providerNetworkClient.CreateSync(pn) - } else { - framework.ExpectNoError(err, "getting provider network "+providerNetworkName) - } + // Wait for existing provider network from other tests to be cleaned up, or create new one + var pn *kubeovnv1.ProviderNetwork + framework.WaitUntil(5*time.Second, 5*time.Minute, func(_ context.Context) (bool, error) { + existingPN, err := providerNetworkClient.ProviderNetworkInterface.Get(context.Background(), providerNetworkName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // Provider network doesn't exist, try to create it + newPN := makeProviderNetwork(providerNetworkName, exchangeLinkName, linkMap) + pn, err = providerNetworkClient.ProviderNetworkInterface.Create(context.Background(), newPN, metav1.CreateOptions{}) + if err != nil { + if k8serrors.IsAlreadyExists(err) { + // Another test created it concurrently, retry in next iteration + return false, nil + } + return false, err + } + // Successfully created, wait for it to be ready + if !providerNetworkClient.WaitToBeReady(pn.Name, 2*time.Minute) { + return false, fmt.Errorf("provider network %s is not ready", pn.Name) + } + pn = providerNetworkClient.Get(pn.Name) + return true, nil + } else if err != nil { + return false, err + } + + // Provider network exists, check if it's being deleted (has deletion timestamp) + if existingPN.DeletionTimestamp != nil { + // It's being deleted by another test, wait for it to be fully removed + klog.Infof("Provider network %s is being deleted, waiting for cleanup...", providerNetworkName) + return false, nil + } + + // Provider network exists and is not being deleted, reuse it + pn = existingPN + return true, nil + }, fmt.Sprintf("waiting for provider network %s to be available", providerNetworkName)) + + ginkgo.By("Waiting for provider network node labels to be set correctly") + framework.WaitUntil(2*time.Second, 2*time.Minute, func(_ context.Context) (bool, error) { + k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) + if err != nil { + return false, err + } + for _, node := range k8sNodes.Items { + link := linkMap[node.Name] + interfaceLabel := fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, providerNetworkName) + readyLabel := fmt.Sprintf(util.ProviderNetworkReadyTemplate, providerNetworkName) + mtuLabel := fmt.Sprintf(util.ProviderNetworkMtuTemplate, providerNetworkName) + + // Check if labels are set correctly + if node.Labels[interfaceLabel] != link.IfName { + return false, nil + } + if node.Labels[readyLabel] != "true" { + return false, nil + } + if node.Labels[mtuLabel] != strconv.Itoa(link.Mtu) { + return false, nil + } + } + return true, nil + }, "waiting for provider network node labels to be set correctly") ginkgo.By("Getting k8s nodes") k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) @@ -484,6 +537,27 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectNoError(err, "timed out waiting for ovs bridge to disappear in node %s", node.Name()) } + ginkgo.By("Waiting for provider network node labels to be cleaned up") + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { + k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) + if err != nil { + return false, err + } + for _, node := range k8sNodes.Items { + // Check if provider network labels still exist + interfaceLabel := fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, providerNetworkName) + if _, exists := node.Labels[interfaceLabel]; exists { + return false, nil + } + // Check extra provider network labels + extraInterfaceLabel := fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, providerExtraNetworkName) + if _, exists := node.Labels[extraInterfaceLabel]; exists { + return false, nil + } + } + return true, nil + }, "waiting for provider network node labels to be cleaned up") + if dockerExtraNetwork != nil { ginkgo.By("Disconnecting nodes from the docker extra network") if err = kind.NetworkDisconnect(dockerExtraNetwork.ID, nodes); err != nil { From 0450134b22c8aa96c41f57fca11bb6cbdb87c7d7 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Mon, 15 Dec 2025 00:12:12 +0800 Subject: [PATCH 17/30] rollback Signed-off-by: zbb88888 --- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 115 +++------------------------- 1 file changed, 10 insertions(+), 105 deletions(-) diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index eba4492b01e..7bb52aa3171 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -292,69 +292,9 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { itFn = func(exchangeLinkName bool, providerNetworkName string, linkMap map[string]*iproute.Link, bridgeIps *[]string) { ginkgo.GinkgoHelper() - ginkgo.By("Getting or creating provider network " + providerNetworkName) - // Wait for existing provider network from other tests to be cleaned up, or create new one - var pn *kubeovnv1.ProviderNetwork - framework.WaitUntil(5*time.Second, 5*time.Minute, func(_ context.Context) (bool, error) { - existingPN, err := providerNetworkClient.ProviderNetworkInterface.Get(context.Background(), providerNetworkName, metav1.GetOptions{}) - if err != nil && k8serrors.IsNotFound(err) { - // Provider network doesn't exist, try to create it - newPN := makeProviderNetwork(providerNetworkName, exchangeLinkName, linkMap) - pn, err = providerNetworkClient.ProviderNetworkInterface.Create(context.Background(), newPN, metav1.CreateOptions{}) - if err != nil { - if k8serrors.IsAlreadyExists(err) { - // Another test created it concurrently, retry in next iteration - return false, nil - } - return false, err - } - // Successfully created, wait for it to be ready - if !providerNetworkClient.WaitToBeReady(pn.Name, 2*time.Minute) { - return false, fmt.Errorf("provider network %s is not ready", pn.Name) - } - pn = providerNetworkClient.Get(pn.Name) - return true, nil - } else if err != nil { - return false, err - } - - // Provider network exists, check if it's being deleted (has deletion timestamp) - if existingPN.DeletionTimestamp != nil { - // It's being deleted by another test, wait for it to be fully removed - klog.Infof("Provider network %s is being deleted, waiting for cleanup...", providerNetworkName) - return false, nil - } - - // Provider network exists and is not being deleted, reuse it - pn = existingPN - return true, nil - }, fmt.Sprintf("waiting for provider network %s to be available", providerNetworkName)) - - ginkgo.By("Waiting for provider network node labels to be set correctly") - framework.WaitUntil(2*time.Second, 2*time.Minute, func(_ context.Context) (bool, error) { - k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) - if err != nil { - return false, err - } - for _, node := range k8sNodes.Items { - link := linkMap[node.Name] - interfaceLabel := fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, providerNetworkName) - readyLabel := fmt.Sprintf(util.ProviderNetworkReadyTemplate, providerNetworkName) - mtuLabel := fmt.Sprintf(util.ProviderNetworkMtuTemplate, providerNetworkName) - - // Check if labels are set correctly - if node.Labels[interfaceLabel] != link.IfName { - return false, nil - } - if node.Labels[readyLabel] != "true" { - return false, nil - } - if node.Labels[mtuLabel] != strconv.Itoa(link.Mtu) { - return false, nil - } - } - return true, nil - }, "waiting for provider network node labels to be set correctly") + ginkgo.By("Creating provider network " + providerNetworkName) + pn := makeProviderNetwork(providerNetworkName, exchangeLinkName, linkMap) + pn = providerNetworkClient.CreateSync(pn) ginkgo.By("Getting k8s nodes") k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) @@ -537,51 +477,16 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectNoError(err, "timed out waiting for ovs bridge to disappear in node %s", node.Name()) } - ginkgo.By("Waiting for provider network node labels to be cleaned up") - framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { - k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) - if err != nil { - return false, err - } - for _, node := range k8sNodes.Items { - // Check if provider network labels still exist - interfaceLabel := fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, providerNetworkName) - if _, exists := node.Labels[interfaceLabel]; exists { - return false, nil - } - // Check extra provider network labels - extraInterfaceLabel := fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, providerExtraNetworkName) - if _, exists := node.Labels[extraInterfaceLabel]; exists { - return false, nil - } - } - return true, nil - }, "waiting for provider network node labels to be cleaned up") - - if dockerExtraNetwork != nil { - ginkgo.By("Disconnecting nodes from the docker extra network") - if err = kind.NetworkDisconnect(dockerExtraNetwork.ID, nodes); err != nil { - framework.Logf("Warning: failed to disconnect nodes from extra network %s: %v", dockerExtraNetworkName, err) - } - - ginkgo.By("Deleting docker extra network " + dockerExtraNetworkName) - if err = docker.NetworkRemove(dockerExtraNetwork.ID); err != nil { - framework.Logf("Warning: failed to remove docker extra network %s: %v", dockerExtraNetworkName, err) - } - dockerExtraNetwork = nil - } - if dockerNetwork != nil { ginkgo.By("Disconnecting nodes from the docker network") - if err = kind.NetworkDisconnect(dockerNetwork.ID, nodes); err != nil { - framework.Logf("Warning: failed to disconnect nodes from network %s: %v", dockerNetworkName, err) - } + err = kind.NetworkDisconnect(dockerNetwork.ID, nodes) + framework.ExpectNoError(err, "disconnecting nodes from network "+dockerNetworkName) + } - ginkgo.By("Deleting docker network " + dockerNetworkName) - if err = docker.NetworkRemove(dockerNetwork.ID); err != nil { - framework.Logf("Warning: failed to remove docker network %s: %v", dockerNetworkName, err) - } - dockerNetwork = nil + if dockerExtraNetwork != nil { + ginkgo.By("Disconnecting nodes from the docker extra network") + err = kind.NetworkDisconnect(dockerExtraNetwork.ID, nodes) + framework.ExpectNoError(err, "disconnecting nodes from extra network "+dockerExtraNetworkName) } }) From 184c48767e26ed7ecf4188b6f48bfb09f55d4c8d Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Mon, 15 Dec 2025 00:39:04 +0800 Subject: [PATCH 18/30] debug provider network deleting process Signed-off-by: zbb88888 --- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 7bb52aa3171..3fe2bcf2251 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -466,6 +466,40 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { ginkgo.By("Deleting provider extra network " + providerExtraNetworkName) providerNetworkClient.DeleteSync(providerExtraNetworkName) + ginkgo.By("Confirming provider networks are fully deleted") + startTime := time.Now() + checkCount := 0 + framework.WaitUntil(2*time.Second, 2*time.Minute, func(_ context.Context) (bool, error) { + checkCount++ + elapsed := time.Since(startTime) + + pn, err := providerNetworkClient.ProviderNetworkInterface.Get(context.Background(), providerNetworkName, metav1.GetOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return false, err + } + if err == nil { + // Provider network still exists + framework.Logf("Warning: provider network %s still exists after %v (check #%d), DeletionTimestamp: %v", + providerNetworkName, elapsed, checkCount, pn.DeletionTimestamp) + return false, nil + } + + extraPn, err := providerNetworkClient.ProviderNetworkInterface.Get(context.Background(), providerExtraNetworkName, metav1.GetOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return false, err + } + if err == nil { + // Extra provider network still exists + framework.Logf("Warning: provider extra network %s still exists after %v (check #%d), DeletionTimestamp: %v", + providerExtraNetworkName, elapsed, checkCount, extraPn.DeletionTimestamp) + return false, nil + } + + // Both provider networks are deleted + framework.Logf("Provider networks fully deleted after %v (%d checks)", elapsed, checkCount) + return true, nil + }, "waiting for provider networks to be fully deleted") + ginkgo.By("Getting nodes") nodes, err := kind.ListNodes(clusterName, "") framework.ExpectNoError(err, "getting nodes in cluster") From e5093a1a407e6c1512b78bd655cc8c02b507bfa3 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Mon, 15 Dec 2025 01:02:47 +0800 Subject: [PATCH 19/30] =?UTF-8?q?=E4=B8=B2=E8=A1=8C=E6=89=A7=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zbb88888 --- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 3fe2bcf2251..09300667fca 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -94,7 +94,7 @@ func makeExternalGatewayConfigMap(gwNodes, switchName string, cidr []string, gwT } } -var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { +var _ = framework.Serial("[group:ovn-vpc-nat-gw]", func() { f := framework.NewDefaultFramework("ovn-vpc-nat-gw") var skip bool From 523b38e913356f5e4dd264c315f0b3d8b1a24d70 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Mon, 15 Dec 2025 01:04:17 +0800 Subject: [PATCH 20/30] =?UTF-8?q?=E4=B8=B2=E8=A1=8C=E6=89=A7=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zbb88888 --- test/e2e/ovn-vpc-nat-gw/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 09300667fca..e8dd1f3c435 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -94,7 +94,7 @@ func makeExternalGatewayConfigMap(gwNodes, switchName string, cidr []string, gwT } } -var _ = framework.Serial("[group:ovn-vpc-nat-gw]", func() { +var _ = framework.SerialDescribe("[group:ovn-vpc-nat-gw]", func() { f := framework.NewDefaultFramework("ovn-vpc-nat-gw") var skip bool From 68874bbad389c599c7eb60e157ec0d7e05e9486f Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Mon, 15 Dec 2025 10:53:17 +0800 Subject: [PATCH 21/30] fix webhook outdated update subnet Signed-off-by: zbb88888 --- test/e2e/webhook/subnet/subnet.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/webhook/subnet/subnet.go b/test/e2e/webhook/subnet/subnet.go index e08a2b1eae7..43a2d6c10c2 100644 --- a/test/e2e/webhook/subnet/subnet.go +++ b/test/e2e/webhook/subnet/subnet.go @@ -98,6 +98,7 @@ var _ = framework.Describe("[group:webhook-subnet]", func() { subnet = subnetClient.CreateSync(subnet) ginkgo.By("Validating vpc can be changed from empty to ovn-cluster") + subnet = subnetClient.Get(subnetName) modifiedSubnet := subnet.DeepCopy() modifiedSubnet.Spec.Vpc = util.DefaultVpc _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) From b3fa18a3855ea7013b0db42ed6d84d7470f96079 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Mon, 15 Dec 2025 12:07:45 +0800 Subject: [PATCH 22/30] make sure ovs br exist and then add its nic Signed-off-by: zbb88888 --- pkg/daemon/ovs_linux.go | 12 ++++++ test/e2e/ovn-vpc-nat-gw/e2e_test.go | 62 ++++++++++++++--------------- test/e2e/webhook/subnet/subnet.go | 2 +- 3 files changed, 42 insertions(+), 34 deletions(-) diff --git a/pkg/daemon/ovs_linux.go b/pkg/daemon/ovs_linux.go index 1410ed5cfee..463d5faaaef 100644 --- a/pkg/daemon/ovs_linux.go +++ b/pkg/daemon/ovs_linux.go @@ -1491,6 +1491,18 @@ func (c *Controller) configProviderNic(nicName, brName string, trunks []string) var mtu int if !isUserspaceDP { + // Check if bridge exists before attempting to add port + output, err := ovs.Exec("list-br") + if err != nil { + klog.Errorf("failed to list OVS bridges: %v, %q", err, output) + return 0, err + } + if !slices.Contains(strings.Split(output, "\n"), brName) { + err := fmt.Errorf("bridge %s does not exist", brName) + klog.Error(err) + return 0, err + } + mtu, err = c.transferAddrsAndRoutes(nicName, brName, false) if err != nil { klog.Errorf("failed to transfer addresses and routes from %s to %s: %v", nicName, brName, err) diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index e8dd1f3c435..dcfd67dca4f 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -533,17 +533,15 @@ var _ = framework.SerialDescribe("[group:ovn-vpc-nat-gw]", func() { exchangeLinkName := false itFn(exchangeLinkName, providerNetworkName, linkMap, &providerBridgeIps) - ginkgo.By("Getting or creating underlay vlan " + vlanName) + ginkgo.By("Verifying vlan " + vlanName + " does not exist from previous test") _, err = vlanClient.VlanInterface.Get(context.Background(), vlanName, metav1.GetOptions{}) - if err != nil && k8serrors.IsNotFound(err) { - // Vlan doesn't exist, create it - vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) - _ = vlanClient.Create(vlan) - } else { - framework.ExpectNoError(err, "getting vlan "+vlanName) - } + framework.ExpectTrue(k8serrors.IsNotFound(err), "vlan %s should not exist, AfterEach cleanup may have failed", vlanName) + + ginkgo.By("Creating underlay vlan " + vlanName) + vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) + _ = vlanClient.Create(vlan) - ginkgo.By("Getting or creating underlay subnet " + underlaySubnetName) + ginkgo.By("Preparing subnet configuration for " + underlaySubnetName) var cidrV4, cidrV6, gatewayV4, gatewayV6 string for _, config := range dockerNetwork.IPAM.Config { switch util.CheckProtocol(config.Subnet.String()) { @@ -580,15 +578,15 @@ var _ = framework.SerialDescribe("[group:ovn-vpc-nat-gw]", func() { } vlanSubnetCidr := strings.Join(cidr, ",") vlanSubnetGw := strings.Join(gateway, ",") + + ginkgo.By("Verifying subnet " + underlaySubnetName + " does not exist from previous test") + _, err = subnetClient.SubnetInterface.Get(context.Background(), underlaySubnetName, metav1.GetOptions{}) + framework.ExpectTrue(k8serrors.IsNotFound(err), "subnet %s should not exist, AfterEach cleanup may have failed", underlaySubnetName) + + ginkgo.By("Creating underlay subnet " + underlaySubnetName) var oldUnderlayExternalSubnet *kubeovnv1.Subnet - oldUnderlayExternalSubnet, err = subnetClient.SubnetInterface.Get(context.Background(), underlaySubnetName, metav1.GetOptions{}) - if err != nil && k8serrors.IsNotFound(err) { - // Subnet doesn't exist, create it - underlaySubnet := framework.MakeSubnet(underlaySubnetName, vlanName, vlanSubnetCidr, vlanSubnetGw, "", "", excludeIPs, nil, nil) - oldUnderlayExternalSubnet = subnetClient.CreateSync(underlaySubnet) - } else { - framework.ExpectNoError(err, "getting subnet "+underlaySubnetName) - } + underlaySubnet := framework.MakeSubnet(underlaySubnetName, vlanName, vlanSubnetCidr, vlanSubnetGw, "", "", excludeIPs, nil, nil) + oldUnderlayExternalSubnet = subnetClient.CreateSync(underlaySubnet) countingEip := makeOvnEip(countingEipName, underlaySubnetName, "", "", "", "") _ = ovnEipClient.CreateSync(countingEip) ginkgo.By("Checking underlay vlan " + oldUnderlayExternalSubnet.Name) @@ -764,17 +762,15 @@ var _ = framework.SerialDescribe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectNoError(err, "getting extra docker network "+dockerExtraNetworkName) itFn(exchangeLinkName, providerExtraNetworkName, extraLinkMap, &extraProviderBridgeIps) - ginkgo.By("Getting or creating underlay extra vlan " + vlanExtraName) + ginkgo.By("Verifying extra vlan " + vlanExtraName + " does not exist from previous test") _, err = vlanClient.VlanInterface.Get(context.Background(), vlanExtraName, metav1.GetOptions{}) - if err != nil && k8serrors.IsNotFound(err) { - // Vlan doesn't exist, create it - vlan := framework.MakeVlan(vlanExtraName, providerExtraNetworkName, 0) - _ = vlanClient.Create(vlan) - } else { - framework.ExpectNoError(err, "getting vlan "+vlanExtraName) - } + framework.ExpectTrue(k8serrors.IsNotFound(err), "vlan %s should not exist, AfterEach cleanup may have failed", vlanExtraName) + + ginkgo.By("Creating underlay extra vlan " + vlanExtraName) + vlan = framework.MakeVlan(vlanExtraName, providerExtraNetworkName, 0) + _ = vlanClient.Create(vlan) - ginkgo.By("Getting or creating extra underlay subnet " + underlayExtraSubnetName) + ginkgo.By("Preparing extra subnet configuration for " + underlayExtraSubnetName) cidrV4, cidrV6, gatewayV4, gatewayV6 = "", "", "", "" for _, config := range dockerExtraNetwork.IPAM.Config { switch util.CheckProtocol(config.Subnet.String()) { @@ -812,14 +808,14 @@ var _ = framework.SerialDescribe("[group:ovn-vpc-nat-gw]", func() { } extraVlanSubnetCidr := strings.Join(cidr, ",") extraVlanSubnetGw := strings.Join(gateway, ",") + + ginkgo.By("Verifying extra subnet " + underlayExtraSubnetName + " does not exist from previous test") _, err = subnetClient.SubnetInterface.Get(context.Background(), underlayExtraSubnetName, metav1.GetOptions{}) - if err != nil && k8serrors.IsNotFound(err) { - // Subnet doesn't exist, create it - underlayExtraSubnet := framework.MakeSubnet(underlayExtraSubnetName, vlanExtraName, extraVlanSubnetCidr, extraVlanSubnetGw, "", "", extraExcludeIPs, nil, nil) - _ = subnetClient.CreateSync(underlayExtraSubnet) - } else { - framework.ExpectNoError(err, "getting subnet "+underlayExtraSubnetName) - } + framework.ExpectTrue(k8serrors.IsNotFound(err), "subnet %s should not exist, AfterEach cleanup may have failed", underlayExtraSubnetName) + + ginkgo.By("Creating extra underlay subnet " + underlayExtraSubnetName) + underlayExtraSubnet := framework.MakeSubnet(underlayExtraSubnetName, vlanExtraName, extraVlanSubnetCidr, extraVlanSubnetGw, "", "", extraExcludeIPs, nil, nil) + _ = subnetClient.CreateSync(underlayExtraSubnet) vlanExtraSubnet := subnetClient.Get(underlayExtraSubnetName) ginkgo.By("Checking extra underlay vlan " + vlanExtraSubnet.Name) framework.ExpectEqual(vlanExtraSubnet.Spec.Vlan, vlanExtraName) diff --git a/test/e2e/webhook/subnet/subnet.go b/test/e2e/webhook/subnet/subnet.go index 43a2d6c10c2..e8333b35038 100644 --- a/test/e2e/webhook/subnet/subnet.go +++ b/test/e2e/webhook/subnet/subnet.go @@ -95,7 +95,7 @@ var _ = framework.Describe("[group:webhook-subnet]", func() { f.SkipVersionPriorTo(1, 15, "vpc cannot be set to non-ovn-cluster on update is not supported before 1.15.0") ginkgo.By("Creating subnet " + subnetName) subnet := framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil) - subnet = subnetClient.CreateSync(subnet) + subnetClient.CreateSync(subnet) ginkgo.By("Validating vpc can be changed from empty to ovn-cluster") subnet = subnetClient.Get(subnetName) From b8d5d606a97c14c988730caa04a51d6a7c4eb1be Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Mon, 15 Dec 2025 18:28:27 +0800 Subject: [PATCH 23/30] fix as self review Signed-off-by: zbb88888 --- pkg/controller/external_gw.go | 76 +++++++++++++++++++----------- pkg/controller/external_gw_test.go | 64 ++++++++++++++++--------- pkg/controller/ip.go | 42 +++++++++-------- pkg/controller/ovn_eip.go | 17 +++++-- pkg/controller/vip.go | 21 +++++---- pkg/controller/vpc.go | 51 +++++++++----------- pkg/controller/vpc_nat_gw_eip.go | 17 +++++-- 7 files changed, 174 insertions(+), 114 deletions(-) diff --git a/pkg/controller/external_gw.go b/pkg/controller/external_gw.go index fb95f7341a4..687cf0cddbf 100644 --- a/pkg/controller/external_gw.go +++ b/pkg/controller/external_gw.go @@ -230,51 +230,71 @@ func (c *Controller) createDefaultVpcLrpEip(externalGwSwitch string) (string, st } // getExternalGatewaySwitchWithConfigMap determines which external gateway switch to use. -// Logic (mutually exclusive modes): -// 1. If default "external" subnet exists: -// - ConfigMap not specified or same name -> use default "external" -// - ConfigMap specifies different name -> ERROR (configuration conflict) -// -// 2. If default "external" subnet does NOT exist: -// - ConfigMap specifies a name -> use ConfigMap name -// - ConfigMap not specified -> return default name (will fail later with subnet not found) +// Two modes: +// - Traditional mode: c.config.ExternalGatewaySwitch (OVN LogicalSwitch, NOT managed by Subnet CRD) +// - ConfigMap mode: User-specified subnet in ConfigMap (managed by Subnet CRD) +// Logic: +// 1. ConfigMap not specified -> use default (traditional mode) +// 2. ConfigMap same as default -> use default (traditional mode) +// 3. ConfigMap different from default -> check conflict and verify ConfigMap subnet exists func (c *Controller) getExternalGatewaySwitchWithConfigMap(configData map[string]string) (string, error) { + configMapSwitch := configData["external-gw-switch"] defaultSwitch := c.config.ExternalGatewaySwitch - configSwitch := configData["external-gw-switch"] - // Check if default subnet exists - _, err := c.subnetsLister.Get(defaultSwitch) - defaultExists := err == nil + // 1. ConfigMap not specified -> use default + if configMapSwitch == "" { + return defaultSwitch, nil + } - if defaultExists { - // Default subnet exists - MUST use it (backward compatibility) - if configSwitch != "" && configSwitch != defaultSwitch { - // Configuration conflict: both modes specified - return "", fmt.Errorf("configuration conflict: default external subnet %s exists, but ConfigMap specifies different subnet %s. Please use only one mode: either remove the default subnet or remove the ConfigMap setting", defaultSwitch, configSwitch) - } + // 2. ConfigMap specified same as default -> use default + if configMapSwitch == defaultSwitch { return defaultSwitch, nil } - // Default subnet does NOT exist - check ConfigMap - if configSwitch != "" { - // Use ConfigMap specified subnet - return configSwitch, nil + // 3. ConfigMap specified different from default + // Check if default logical switch exists in OVN (configuration conflict) + // Note: c.config.ExternalGatewaySwitch is OVN LogicalSwitch, NOT Subnet CRD + exists, err := c.OVNNbClient.LogicalSwitchExists(defaultSwitch) + if err != nil { + klog.Errorf("failed to check if default logical switch %s exists: %v", defaultSwitch, err) + return "", err + } + if exists { + // Default logical switch exists - conflict + err := fmt.Errorf("configuration conflict: default external logical switch %s exists, but ConfigMap specifies different subnet %s. Please use only one mode: either remove the default logical switch or remove the ConfigMap setting", defaultSwitch, configMapSwitch) + klog.Error(err) + return "", err + } + + // Default subnet does not exist, verify ConfigMap-specified subnet exists + _, err = c.subnetsLister.Get(configMapSwitch) + if err != nil { + if k8serrors.IsNotFound(err) { + err := fmt.Errorf("ConfigMap specifies external subnet %s, but it does not exist. Please create the subnet first or update the ConfigMap", configMapSwitch) + klog.Error(err) + return "", err + } + err := fmt.Errorf("failed to get subnet %s from lister: %w", configMapSwitch, err) + klog.Error(err) + return "", err } - // Neither default nor ConfigMap specified subnet available - return defaultSwitch, nil // Return default name anyway for error messages + return configMapSwitch, nil } -// getExternalGatewaySwitch returns the external gateway switch name by reading from ConfigMap -// Logic: default subnet exists -> use default; default not exists + ConfigMap specified -> use ConfigMap -func (c *Controller) getExternalGatewaySwitch() (string, error) { +// getConfigDefaultExternalSwitch determines which (from config or configmap) external gateway switch to use +// ConfigMap not specified -> use default; +// default not exists + ConfigMap specified -> use ConfigMap +func (c *Controller) getConfigDefaultExternalSwitch() (string, error) { cm, err := c.configMapsLister.ConfigMaps(c.config.ExternalGatewayConfigNS).Get(util.ExternalGatewayConfig) if err != nil { if k8serrors.IsNotFound(err) { // ConfigMap doesn't exist, use default return c.config.ExternalGatewaySwitch, nil } - return "", fmt.Errorf("failed to get ConfigMap %s: %w", util.ExternalGatewayConfig, err) + err = fmt.Errorf("failed to get ConfigMap %s: %w", util.ExternalGatewayConfig, err) + klog.Error(err) + return "", err } // Check if ConfigMap is enabled diff --git a/pkg/controller/external_gw_test.go b/pkg/controller/external_gw_test.go index a35a37ed4de..e696e60b859 100644 --- a/pkg/controller/external_gw_test.go +++ b/pkg/controller/external_gw_test.go @@ -6,12 +6,14 @@ import ( "testing" "time" + "go.uber.org/mock/gomock" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" + mockovs "github.com/kubeovn/kube-ovn/mocks/pkg/ovs" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" kubeovnfake "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/fake" kubeovninformers "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions" @@ -19,7 +21,8 @@ import ( ) // Helper function to create a test controller with optional initial objects -func newTestController(subnets []*kubeovnv1.Subnet, configMaps []*corev1.ConfigMap) *Controller { +// defaultLSExists controls whether the default logical switch "external" exists in OVN +func newTestController(t *testing.T, subnets []*kubeovnv1.Subnet, configMaps []*corev1.ConfigMap, defaultLSExists bool) *Controller { // Create clientsets var kubeObjects []runtime.Object for _, cm := range configMaps { @@ -44,10 +47,16 @@ func newTestController(subnets []*kubeovnv1.Subnet, configMaps []*corev1.ConfigM ExternalGatewayConfigNS: "kube-system", } + // Create mock OVN client + mockOvnClient := mockovs.NewMockNbClient(gomock.NewController(t)) + // Set expectation for LogicalSwitchExists based on defaultLSExists parameter + mockOvnClient.EXPECT().LogicalSwitchExists("external").Return(defaultLSExists, nil).AnyTimes() + controller := &Controller{ config: config, configMapsLister: kubeInformerFactory.Core().V1().ConfigMaps().Lister(), subnetsLister: kubeovnInformerFactory.Kubeovn().V1().Subnets().Lister(), + OVNNbClient: mockOvnClient, } // Start informers and wait for cache sync @@ -69,9 +78,9 @@ func newTestController(subnets []*kubeovnv1.Subnet, configMaps []*corev1.ConfigM // Test Scenario 1: Default "external" subnet does NOT exist, ConfigMap NOT specified // Expected: Return default name "external" (will fail later with subnet not found) func TestGetExternalGatewaySwitch_Scenario1_DefaultNotExist_ConfigMapNotSpecified(t *testing.T) { - c := newTestController(nil, nil) + c := newTestController(t, nil, nil, false) - result, err := c.getExternalGatewaySwitch() + result, err := c.getConfigDefaultExternalSwitch() if err != nil { t.Errorf("expected no error, got: %v", err) } @@ -91,9 +100,9 @@ func TestGetExternalGatewaySwitch_Scenario2_DefaultExists_ConfigMapNotSpecified( }, } - c := newTestController([]*kubeovnv1.Subnet{defaultSubnet}, nil) + c := newTestController(t, []*kubeovnv1.Subnet{defaultSubnet}, nil, true) - result, err := c.getExternalGatewaySwitch() + result, err := c.getConfigDefaultExternalSwitch() if err != nil { t.Errorf("expected no error, got: %v", err) } @@ -107,6 +116,13 @@ func TestGetExternalGatewaySwitch_Scenario2_DefaultExists_ConfigMapNotSpecified( // Test Scenario 3: Default "external" subnet does NOT exist, ConfigMap specifies "custom-ext" // Expected: Use "custom-ext" func TestGetExternalGatewaySwitch_Scenario3_DefaultNotExist_ConfigMapSpecifiedDifferent(t *testing.T) { + // Create the custom-ext subnet that ConfigMap references + customSubnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-ext", + }, + } + configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: util.ExternalGatewayConfig, @@ -118,9 +134,9 @@ func TestGetExternalGatewaySwitch_Scenario3_DefaultNotExist_ConfigMapSpecifiedDi }, } - c := newTestController(nil, []*corev1.ConfigMap{configMap}) + c := newTestController(t, []*kubeovnv1.Subnet{customSubnet}, []*corev1.ConfigMap{configMap}, false) - result, err := c.getExternalGatewaySwitch() + result, err := c.getConfigDefaultExternalSwitch() if err != nil { t.Errorf("expected no error, got: %v", err) } @@ -151,9 +167,9 @@ func TestGetExternalGatewaySwitch_Scenario4_DefaultExists_ConfigMapSpecifiedDiff }, } - c := newTestController([]*kubeovnv1.Subnet{defaultSubnet}, []*corev1.ConfigMap{configMap}) + c := newTestController(t, []*kubeovnv1.Subnet{defaultSubnet}, []*corev1.ConfigMap{configMap}, true) - _, err := c.getExternalGatewaySwitch() + _, err := c.getConfigDefaultExternalSwitch() if err == nil { t.Error("expected error due to configuration conflict, but got nil") return @@ -185,9 +201,9 @@ func TestGetExternalGatewaySwitch_Scenario5_DefaultExists_ConfigMapSpecifiedSame }, } - c := newTestController([]*kubeovnv1.Subnet{defaultSubnet}, []*corev1.ConfigMap{configMap}) + c := newTestController(t, []*kubeovnv1.Subnet{defaultSubnet}, []*corev1.ConfigMap{configMap}, true) - result, err := c.getExternalGatewaySwitch() + result, err := c.getConfigDefaultExternalSwitch() if err != nil { t.Errorf("expected no error, got: %v", err) } @@ -212,9 +228,9 @@ func TestGetExternalGatewaySwitch_ConfigMapDisabled(t *testing.T) { }, } - c := newTestController(nil, []*corev1.ConfigMap{configMap}) + c := newTestController(t, nil, []*corev1.ConfigMap{configMap}, false) - result, err := c.getExternalGatewaySwitch() + result, err := c.getConfigDefaultExternalSwitch() if err != nil { t.Errorf("expected no error, got: %v", err) } @@ -276,17 +292,16 @@ func TestGetExternalGatewaySwitchWithConfigMap_AllScenarios(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var subnets []*kubeovnv1.Subnet - if tt.defaultExists { - subnets = []*kubeovnv1.Subnet{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "external", - }, + if tt.configSwitch != "" && tt.configSwitch != "external" { + // Add the custom subnet referenced by ConfigMap + subnets = append(subnets, &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: tt.configSwitch, }, - } + }) } - c := newTestController(subnets, nil) + c := newTestController(t, subnets, nil, tt.defaultExists) configData := map[string]string{} if tt.configSwitch != "" { @@ -330,10 +345,15 @@ func TestGetExternalGatewaySwitch_WithClientOperations(t *testing.T) { ExternalGatewayConfigNS: "kube-system", } + // Create mock OVN client - default logical switch exists (conflict case) + mockOvnClient := mockovs.NewMockNbClient(gomock.NewController(t)) + mockOvnClient.EXPECT().LogicalSwitchExists("external").Return(true, nil).AnyTimes() + controller := &Controller{ config: config, configMapsLister: kubeInformerFactory.Core().V1().ConfigMaps().Lister(), subnetsLister: kubeovnInformerFactory.Kubeovn().V1().Subnets().Lister(), + OVNNbClient: mockOvnClient, } stopCh := make(chan struct{}) @@ -373,7 +393,7 @@ func TestGetExternalGatewaySwitch_WithClientOperations(t *testing.T) { time.Sleep(200 * time.Millisecond) // This should return error due to conflict - _, err = controller.getExternalGatewaySwitch() + _, err = controller.getConfigDefaultExternalSwitch() if err == nil { t.Error("expected configuration conflict error, got nil") } else if !strings.Contains(err.Error(), "configuration conflict") { diff --git a/pkg/controller/ip.go b/pkg/controller/ip.go index 81868b8824a..d74e5ccee9f 100644 --- a/pkg/controller/ip.go +++ b/pkg/controller/ip.go @@ -155,6 +155,18 @@ func (c *Controller) handleAddReservedIP(key string) error { return nil } + // Add finalizer FIRST before any resource allocation to prevent IP leak + if err = c.handleAddOrUpdateIPFinalizer(ip); err != nil { + klog.Errorf("failed to add finalizer for ip, %v", err) + return err + } + // Re-fetch the ip after adding finalizer (resourceVersion may have changed) + ip, err = c.ipsLister.Get(key) + if err != nil { + klog.Errorf("failed to get ip after adding finalizer, %v", err) + return err + } + // not handle add the ip, which created in pod process, lsp created before ip lsp, err := c.OVNNbClient.GetLogicalSwitchPort(portName, true) if err != nil { @@ -162,13 +174,8 @@ func (c *Controller) handleAddReservedIP(key string) error { return err } if lsp != nil { - // port already exists means the ip already created - // but we still need to ensure finalizer is added - klog.V(3).Infof("ip %s is ready, checking finalizer", portName) - if err = c.handleAddOrUpdateIPFinalizer(ip); err != nil { - klog.Errorf("failed to handle add or update finalizer for ip %s: %v", ip.Name, err) - return err - } + // port already exists means the ip already created, finalizer already added above + klog.V(3).Infof("ip %s is ready, finalizer already added", portName) return nil } @@ -212,12 +219,6 @@ func (c *Controller) handleAddReservedIP(key string) error { } } - // Handle add or update finalizer after IP is created/updated - if err = c.handleAddOrUpdateIPFinalizer(ip); err != nil { - klog.Errorf("failed to handle add or update finalizer for ip %s: %v", ip.Name, err) - return err - } - return nil } @@ -231,12 +232,7 @@ func (c *Controller) handleUpdateIP(key string) error { return err } - // Handle add or update finalizer - if err = c.handleAddOrUpdateIPFinalizer(cachedIP); err != nil { - klog.Errorf("failed to handle add or update finalizer for ip %s: %v", key, err) - return err - } - + // Handle deletion first if !cachedIP.DeletionTimestamp.IsZero() { klog.Infof("handle deleting ip %s", cachedIP.Name) subnet, err := c.subnetsLister.Get(cachedIP.Spec.Subnet) @@ -277,7 +273,15 @@ func (c *Controller) handleUpdateIP(key string) error { klog.Errorf("failed to handle del ip finalizer %v", err) return err } + return nil } + + // Non-deletion case: ensure finalizer is added + if err = c.handleAddOrUpdateIPFinalizer(cachedIP); err != nil { + klog.Errorf("failed to handle add or update finalizer for ip %s: %v", key, err) + return err + } + return nil } diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index 06eac63b492..e0701fde89c 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -90,6 +90,19 @@ func (c *Controller) handleAddOvnEip(key string) error { return nil } klog.Infof("handle add ovn eip %s", cachedEip.Name) + + // Add finalizer FIRST before any resource allocation to prevent IP leak + if err = c.handleAddOrUpdateOvnEipFinalizer(cachedEip); err != nil { + klog.Errorf("failed to add finalizer for ovn eip, %v", err) + return err + } + // Re-fetch the eip after adding finalizer (resourceVersion may have changed) + cachedEip, err = c.ovnEipsLister.Get(key) + if err != nil { + klog.Errorf("failed to get ovn eip after adding finalizer, %v", err) + return err + } + var v4ip, v6ip, mac, subnetName string subnetName = cachedEip.Spec.ExternalSubnet if subnetName == "" { @@ -144,10 +157,6 @@ func (c *Controller) handleAddOvnEip(key string) error { return err } } - if err = c.handleAddOrUpdateOvnEipFinalizer(cachedEip); err != nil { - klog.Errorf("failed to add finalizer for ovn eip, %v", err) - return err - } return nil } diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index 82f8e182bed..f235e9232a8 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -80,6 +80,19 @@ func (c *Controller) handleAddVirtualIP(key string) error { return nil } klog.V(3).Infof("handle add vip %s", key) + + // Add finalizer FIRST before any resource allocation to prevent IP leak + if err := c.handleAddOrUpdateVipFinalizer(key); err != nil { + klog.Errorf("failed to add finalizer for vip, %v", err) + return err + } + // Re-fetch the vip after adding finalizer (resourceVersion may have changed) + cachedVip, err = c.virtualIpsLister.Get(key) + if err != nil { + klog.Errorf("failed to get vip after adding finalizer, %v", err) + return err + } + vip := cachedVip.DeepCopy() var sourceV4Ip, sourceV6Ip, v4ip, v6ip, mac, subnetName string subnetName = vip.Spec.Subnet @@ -160,10 +173,6 @@ func (c *Controller) handleAddVirtualIP(key string) error { if vip.Spec.Type == util.KubeHostVMVip { // vm use the vip as its real ip klog.Infof("created host network pod vm ip %s", key) - if err = c.handleAddOrUpdateVipFinalizer(key); err != nil { - klog.Errorf("failed to handle add or update vip finalizer %v", err) - return err - } return nil } if err := c.handleUpdateVirtualParents(key); err != nil { @@ -171,10 +180,6 @@ func (c *Controller) handleAddVirtualIP(key string) error { klog.Error(err) return err } - if err = c.handleAddOrUpdateVipFinalizer(key); err != nil { - klog.Errorf("failed to handle add or update vip finalizer %v", err) - return err - } return nil } diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index b2d4cb01acd..8477ea41323 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -129,10 +129,9 @@ func (c *Controller) handleDelVpc(vpc *kubeovnv1.Vpc) error { return err } - // Delete both ConfigMap-specified and default external gateway switches - // to ensure cleanup even if configuration changed between creation and deletion - if err := c.deleteVpcExternalSubnets(vpc.Name); err != nil { - klog.Errorf("failed to delete external connections for vpc %s: %v", vpc.Name, err) + // Delete connection to default external network + if err := c.deleteVpc2ExternalConnection(vpc.Name); err != nil { + klog.Errorf("failed to delete default external connection for vpc %s: %v", vpc.Name, err) return err } @@ -353,7 +352,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { // Determine which external gateway switch to use // Logic: default subnet exists -> use default; default not exists + ConfigMap specified -> use ConfigMap - externalGwSwitch, err := c.getExternalGatewaySwitch() + externalGwSwitch, err := c.getConfigDefaultExternalSwitch() if err != nil { klog.Warningf("failed to get external gateway switch: %v", err) externalGwSwitch = c.config.ExternalGatewaySwitch // fallback to default @@ -1533,31 +1532,27 @@ func (c *Controller) updateVpcExternalStatus(key string, enableExternal bool) er return nil } -// deleteVpcExternalSubnets deletes both ConfigMap-specified and default external gateway switches -// to ensure complete cleanup even if configuration changed between VPC creation and deletion -func (c *Controller) deleteVpcExternalSubnets(vpcName string) error { - // Try to get ConfigMap-specified switch and delete it - externalGwSwitch, err := c.getExternalGatewaySwitch() - if err != nil { - klog.Warningf("failed to get external gateway switch during deletion: %v", err) - // Fallback to default if we can't determine ConfigMap setting - externalGwSwitch = c.config.ExternalGatewaySwitch +// deleteVpc2ExternalConnection deletes VPC connections to external networks +// Deletes both ConfigMap-specified and default connections to ensure complete cleanup +// even if configuration changed during VPC lifecycle +func (c *Controller) deleteVpc2ExternalConnection(vpcName string) error { + var anyErr error + + // Try to delete ConfigMap-specified connection if exists + cm, err := c.configMapsLister.ConfigMaps(c.config.ExternalGatewayConfigNS).Get(util.ExternalGatewayConfig) + if err == nil && cm.Data["external-gw-switch"] != "" { + configSwitch := cm.Data["external-gw-switch"] + if err := c.handleDelVpcRes2ExternalSubnet(vpcName, configSwitch); err != nil { + klog.Errorf("failed to delete ConfigMap-specified connection %s for vpc %s: %v", configSwitch, vpcName, err) + anyErr = err + } } - var anyErr error - if err := c.handleDelVpcRes2ExternalSubnet(vpcName, externalGwSwitch); err != nil { - klog.Errorf("failed to delete external connection %s for vpc %s: %v", externalGwSwitch, vpcName, err) - anyErr = err - } - - // Also try to delete default switch if it's different from ConfigMap-specified - // This ensures cleanup even if configuration changed - if externalGwSwitch != c.config.ExternalGatewaySwitch { - if err := c.handleDelVpcRes2ExternalSubnet(vpcName, c.config.ExternalGatewaySwitch); err != nil { - klog.Errorf("failed to delete default external connection %s for vpc %s: %v", c.config.ExternalGatewaySwitch, vpcName, err) - if anyErr == nil { - anyErr = err - } + // Always try to delete default connection + if err := c.handleDelVpcRes2ExternalSubnet(vpcName, c.config.ExternalGatewaySwitch); err != nil { + klog.Errorf("failed to delete default connection %s for vpc %s: %v", c.config.ExternalGatewaySwitch, vpcName, err) + if anyErr == nil { + anyErr = err } } diff --git a/pkg/controller/vpc_nat_gw_eip.go b/pkg/controller/vpc_nat_gw_eip.go index 59d79ac0069..d1e4bdd81a3 100644 --- a/pkg/controller/vpc_nat_gw_eip.go +++ b/pkg/controller/vpc_nat_gw_eip.go @@ -85,6 +85,18 @@ func (c *Controller) handleAddIptablesEip(key string) error { return nil } + // Add finalizer FIRST before any resource allocation to prevent IP leak + if err := c.handleAddOrUpdateIptablesEipFinalizer(key); err != nil { + klog.Errorf("failed to add finalizer for iptables eip, %v", err) + return err + } + // Re-fetch the eip after adding finalizer (resourceVersion may have changed) + cachedEip, err = c.iptablesEipsLister.Get(key) + if err != nil { + klog.Errorf("failed to get iptables eip after adding finalizer, %v", err) + return err + } + subnets, err := c.subnetsLister.List(labels.Everything()) if err != nil { klog.Errorf("failed to list subnets: %v", err) @@ -154,11 +166,6 @@ func (c *Controller) handleAddIptablesEip(key string) error { return err } - if err = c.handleAddOrUpdateIptablesEipFinalizer(key); err != nil { - klog.Errorf("failed to handle add or update finalizer for eip %s, %v", key, err) - return err - } - return nil } From 9bd30600f21551b60eb2a79b0af97d71df630702 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Mon, 15 Dec 2025 21:01:12 +0800 Subject: [PATCH 24/30] =?UTF-8?q?=E4=BC=98=E5=8C=96IP=E5=92=8CEIP=E5=A4=84?= =?UTF-8?q?=E7=90=86=E9=80=BB=E8=BE=91=EF=BC=8C=E7=A7=BB=E9=99=A4=E5=86=97?= =?UTF-8?q?=E4=BD=99=E7=9A=84finalizer=E6=B7=BB=E5=8A=A0=E6=AD=A5=E9=AA=A4?= =?UTF-8?q?=EF=BC=8C=E7=A1=AE=E4=BF=9D=E5=9C=A8=E6=89=80=E6=9C=89=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E5=AE=8C=E6=88=90=E5=90=8E=E6=9B=B4=E6=96=B0=E5=AD=90?= =?UTF-8?q?=E7=BD=91=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/controller/ip.go | 28 ++++++++----------------- pkg/controller/ovn_eip.go | 35 +++++++++++++++++--------------- pkg/controller/vip.go | 34 +++++++++++++++---------------- pkg/controller/vpc_nat_gw_eip.go | 34 ++++++++++++++++--------------- 4 files changed, 63 insertions(+), 68 deletions(-) diff --git a/pkg/controller/ip.go b/pkg/controller/ip.go index d74e5ccee9f..ea3e670073e 100644 --- a/pkg/controller/ip.go +++ b/pkg/controller/ip.go @@ -155,18 +155,6 @@ func (c *Controller) handleAddReservedIP(key string) error { return nil } - // Add finalizer FIRST before any resource allocation to prevent IP leak - if err = c.handleAddOrUpdateIPFinalizer(ip); err != nil { - klog.Errorf("failed to add finalizer for ip, %v", err) - return err - } - // Re-fetch the ip after adding finalizer (resourceVersion may have changed) - ip, err = c.ipsLister.Get(key) - if err != nil { - klog.Errorf("failed to get ip after adding finalizer, %v", err) - return err - } - // not handle add the ip, which created in pod process, lsp created before ip lsp, err := c.OVNNbClient.GetLogicalSwitchPort(portName, true) if err != nil { @@ -219,6 +207,9 @@ func (c *Controller) handleAddReservedIP(key string) error { } } + // Trigger subnet status update after all operations complete + // At this point: IPAM allocated, IP CR created with labels+finalizer + c.updateSubnetStatusQueue.Add(subnet.Name) return nil } @@ -318,10 +309,6 @@ func (c *Controller) handleAddOrUpdateIPFinalizer(cachedIP *kubeovnv1.IP) error // IP is being deleted, don't handle finalizer add/update return nil } - if len(cachedIP.GetFinalizers()) != 0 { - // Finalizer already exists - return nil - } newIP := cachedIP.DeepCopy() controllerutil.RemoveFinalizer(newIP, util.DepreciatedFinalizerName) @@ -340,8 +327,9 @@ func (c *Controller) handleAddOrUpdateIPFinalizer(cachedIP *kubeovnv1.IP) error return err } - // Trigger subnet status update after finalizer is added - // This ensures subnet status reflects the new IP allocation + // Trigger subnet status update after finalizer is processed as a fallback + // This handles cases where finalizer was not added during creation + // AddFinalizer is idempotent, so this is safe even if finalizer already exists c.updateSubnetStatusQueue.Add(cachedIP.Spec.Subnet) for _, as := range cachedIP.Spec.AttachSubnets { c.updateSubnetStatusQueue.Add(as) @@ -469,9 +457,11 @@ func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, } v4IP, v6IP := util.SplitStringIP(ip) if ipCR == nil { + // Create CR with finalizer and labels all at once ipCR = &kubeovnv1.IP{ ObjectMeta: metav1.ObjectMeta{ - Name: ipName, + Name: ipName, + Finalizers: []string{util.KubeOVNControllerFinalizer}, Labels: map[string]string{ util.SubnetNameLabel: subnetName, util.NodeNameLabel: nodeName, diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index e0701fde89c..f34e8ee741e 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -91,18 +91,6 @@ func (c *Controller) handleAddOvnEip(key string) error { } klog.Infof("handle add ovn eip %s", cachedEip.Name) - // Add finalizer FIRST before any resource allocation to prevent IP leak - if err = c.handleAddOrUpdateOvnEipFinalizer(cachedEip); err != nil { - klog.Errorf("failed to add finalizer for ovn eip, %v", err) - return err - } - // Re-fetch the eip after adding finalizer (resourceVersion may have changed) - cachedEip, err = c.ovnEipsLister.Get(key) - if err != nil { - klog.Errorf("failed to get ovn eip after adding finalizer, %v", err) - return err - } - var v4ip, v6ip, mac, subnetName string subnetName = cachedEip.Spec.ExternalSubnet if subnetName == "" { @@ -157,6 +145,10 @@ func (c *Controller) handleAddOvnEip(key string) error { return err } } + + // Trigger subnet status update after all operations complete + // At this point: IPAM allocated, OvnEip CR created with labels+status+finalizer + c.updateSubnetStatusQueue.Add(subnet.Name) return nil } @@ -308,9 +300,11 @@ func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usageT cachedEip, err := c.ovnEipsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { + // Create CR with finalizer, labels and status all at once _, err := c.config.KubeOvnClient.KubeovnV1().OvnEips().Create(context.Background(), &kubeovnv1.OvnEip{ ObjectMeta: metav1.ObjectMeta{ - Name: key, + Name: key, + Finalizers: []string{util.KubeOVNControllerFinalizer}, Labels: map[string]string{ util.SubnetNameLabel: subnet, util.OvnEipTypeLabel: usageType, @@ -325,6 +319,14 @@ func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usageT MacAddress: mac, Type: usageType, }, + Status: kubeovnv1.OvnEipStatus{ + V4Ip: v4ip, + V6Ip: v6ip, + MacAddress: mac, + Type: usageType, + Nat: "", + Ready: false, + }, }, metav1.CreateOptions{}) if err != nil { err := fmt.Errorf("failed to create crd ovn eip '%s', %w", key, err) @@ -535,7 +537,7 @@ func (c *Controller) syncOvnEipFinalizer(cl client.Client) error { } func (c *Controller) handleAddOrUpdateOvnEipFinalizer(cachedEip *kubeovnv1.OvnEip) error { - if !cachedEip.DeletionTimestamp.IsZero() || len(cachedEip.GetFinalizers()) != 0 { + if !cachedEip.DeletionTimestamp.IsZero() { return nil } newEip := cachedEip.DeepCopy() @@ -555,8 +557,9 @@ func (c *Controller) handleAddOrUpdateOvnEipFinalizer(cachedEip *kubeovnv1.OvnEi return err } - // Trigger subnet status update after finalizer is added - // This ensures subnet status reflects the new OVN EIP allocation + // Trigger subnet status update after finalizer is processed as a fallback + // This handles cases where finalizer was not added during creation + // AddFinalizer is idempotent, so this is safe even if finalizer already exists c.updateSubnetStatusQueue.Add(cachedEip.Spec.ExternalSubnet) return nil } diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index f235e9232a8..0793b6d82fc 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -81,18 +81,6 @@ func (c *Controller) handleAddVirtualIP(key string) error { } klog.V(3).Infof("handle add vip %s", key) - // Add finalizer FIRST before any resource allocation to prevent IP leak - if err := c.handleAddOrUpdateVipFinalizer(key); err != nil { - klog.Errorf("failed to add finalizer for vip, %v", err) - return err - } - // Re-fetch the vip after adding finalizer (resourceVersion may have changed) - cachedVip, err = c.virtualIpsLister.Get(key) - if err != nil { - klog.Errorf("failed to get vip after adding finalizer, %v", err) - return err - } - vip := cachedVip.DeepCopy() var sourceV4Ip, sourceV6Ip, v4ip, v6ip, mac, subnetName string subnetName = vip.Spec.Subnet @@ -180,6 +168,10 @@ func (c *Controller) handleAddVirtualIP(key string) error { klog.Error(err) return err } + + // Trigger subnet status update after all operations complete + // At this point: IPAM allocated, VIP CR created with labels+status+finalizer + c.updateSubnetStatusQueue.Add(subnetName) return nil } @@ -374,14 +366,16 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string vipCR, err := c.virtualIpsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { + // Create CR with finalizer, labels and status all at once if _, err := c.config.KubeOvnClient.KubeovnV1().Vips().Create(context.Background(), &kubeovnv1.Vip{ ObjectMeta: metav1.ObjectMeta{ - Name: key, + Name: key, + Namespace: ns, + Finalizers: []string{util.KubeOVNControllerFinalizer}, Labels: map[string]string{ util.SubnetNameLabel: subnet, util.IPReservedLabel: "", }, - Namespace: ns, }, Spec: kubeovnv1.VipSpec{ Namespace: ns, @@ -390,6 +384,11 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string V6ip: v6ip, MacAddress: mac, }, + Status: kubeovnv1.VipStatus{ + V4ip: v4ip, + V6ip: v6ip, + Mac: mac, + }, }, metav1.CreateOptions{}); err != nil { err := fmt.Errorf("failed to create crd vip '%s', %w", key, err) klog.Error(err) @@ -520,7 +519,7 @@ func (c *Controller) handleAddOrUpdateVipFinalizer(key string) error { klog.Error(err) return err } - if !cachedVip.DeletionTimestamp.IsZero() || len(cachedVip.GetFinalizers()) != 0 { + if !cachedVip.DeletionTimestamp.IsZero() { return nil } newVip := cachedVip.DeepCopy() @@ -540,8 +539,9 @@ func (c *Controller) handleAddOrUpdateVipFinalizer(key string) error { return err } - // Trigger subnet status update after finalizer is added - // This ensures subnet status reflects the new VIP allocation + // Trigger subnet status update after finalizer is processed as a fallback + // This handles cases where finalizer was not added during creation + // AddFinalizer is idempotent, so this is safe even if finalizer already exists c.updateSubnetStatusQueue.Add(cachedVip.Spec.Subnet) return nil } diff --git a/pkg/controller/vpc_nat_gw_eip.go b/pkg/controller/vpc_nat_gw_eip.go index d1e4bdd81a3..a9ebbea8f61 100644 --- a/pkg/controller/vpc_nat_gw_eip.go +++ b/pkg/controller/vpc_nat_gw_eip.go @@ -85,18 +85,6 @@ func (c *Controller) handleAddIptablesEip(key string) error { return nil } - // Add finalizer FIRST before any resource allocation to prevent IP leak - if err := c.handleAddOrUpdateIptablesEipFinalizer(key); err != nil { - klog.Errorf("failed to add finalizer for iptables eip, %v", err) - return err - } - // Re-fetch the eip after adding finalizer (resourceVersion may have changed) - cachedEip, err = c.iptablesEipsLister.Get(key) - if err != nil { - klog.Errorf("failed to get iptables eip after adding finalizer, %v", err) - return err - } - subnets, err := c.subnetsLister.List(labels.Everything()) if err != nil { klog.Errorf("failed to list subnets: %v", err) @@ -166,6 +154,9 @@ func (c *Controller) handleAddIptablesEip(key string) error { return err } + // Trigger subnet status update after all operations complete + // At this point: IPAM allocated, IptablesEIP CR created with labels+status+finalizer + c.updateSubnetStatusQueue.Add(subnet.Name) return nil } @@ -610,9 +601,11 @@ func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, ext externalNetwork := util.GetExternalNetwork(cachedEip.Spec.ExternalSubnet) if needCreate { klog.V(3).Infof("create eip cr %s", key) + // Create CR with finalizer, labels and status all at once _, err := c.config.KubeOvnClient.KubeovnV1().IptablesEIPs().Create(context.Background(), &kubeovnv1.IptablesEIP{ ObjectMeta: metav1.ObjectMeta{ - Name: key, + Name: key, + Finalizers: []string{util.KubeOVNControllerFinalizer}, Labels: map[string]string{ util.SubnetNameLabel: externalNet, util.EipV4IpLabel: v4ip, @@ -624,6 +617,14 @@ func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, ext V6ip: v6ip, MacAddress: mac, NatGwDp: natGwDp, + QoSPolicy: qos, + }, + Status: kubeovnv1.IptablesEIPStatus{ + IP: v4ip, + Ready: true, + QoSPolicy: qos, + Nat: "", + Redo: "", }, }, metav1.CreateOptions{}) if err != nil { @@ -707,7 +708,7 @@ func (c *Controller) handleAddOrUpdateIptablesEipFinalizer(key string) error { klog.Error(err) return err } - if !cachedIptablesEip.DeletionTimestamp.IsZero() || len(cachedIptablesEip.GetFinalizers()) != 0 { + if !cachedIptablesEip.DeletionTimestamp.IsZero() { return nil } newIptablesEip := cachedIptablesEip.DeepCopy() @@ -727,8 +728,9 @@ func (c *Controller) handleAddOrUpdateIptablesEipFinalizer(key string) error { return err } - // Trigger subnet status update after finalizer is added - // This ensures subnet status reflects the new Iptables EIP allocation + // Trigger subnet status update after finalizer is processed as a fallback + // This handles cases where finalizer was not added during creation + // AddFinalizer is idempotent, so this is safe even if finalizer already exists externalNetwork := util.GetExternalNetwork(cachedIptablesEip.Spec.ExternalSubnet) c.updateSubnetStatusQueue.Add(externalNetwork) return nil From 5c5a31130a7d2628046ef21e8f50ec94e2842c4a Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Mon, 15 Dec 2025 22:55:44 +0800 Subject: [PATCH 25/30] =?UTF-8?q?=E5=9C=A8=E5=88=9B=E5=BB=BA=E6=88=96?= =?UTF-8?q?=E6=9B=B4=E6=96=B0CR=E5=90=8E=EF=BC=8C=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=BB=B6=E8=BF=9F=E4=BB=A5=E8=A7=A6=E5=8F=91=E5=AD=90=E7=BD=91?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/controller/ip.go | 9 ++++++++- pkg/controller/ovn_eip.go | 3 +++ pkg/controller/vip.go | 4 ++++ pkg/controller/vpc_nat_gw_eip.go | 6 ++++-- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pkg/controller/ip.go b/pkg/controller/ip.go index ea3e670073e..030fc9b2eef 100644 --- a/pkg/controller/ip.go +++ b/pkg/controller/ip.go @@ -9,6 +9,7 @@ import ( "net" "reflect" "strings" + "time" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -484,7 +485,7 @@ func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, PodType: podType, }, } - if _, err = c.config.KubeOvnClient.KubeovnV1().IPs().Create(context.Background(), ipCR, metav1.CreateOptions{}); err != nil { + if ipCR, err = c.config.KubeOvnClient.KubeovnV1().IPs().Create(context.Background(), ipCR, metav1.CreateOptions{}); err != nil { errMsg := fmt.Errorf("failed to create ip CR %s: %w", ipName, err) klog.Error(errMsg) return errMsg @@ -523,6 +524,12 @@ func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, return err } } + // Trigger subnet status update after CR creation with finalizer + time.Sleep(1 * time.Second) + c.updateSubnetStatusQueue.Add(ipCR.Spec.Subnet) + for _, as := range ipCR.Spec.AttachSubnets { + c.updateSubnetStatusQueue.Add(as) + } return nil } diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index f34e8ee741e..a4418ef6a21 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -409,6 +409,9 @@ func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usageT } } } + // Trigger subnet status update after CR creation or update + time.Sleep(1 * time.Second) + c.updateSubnetStatusQueue.Add(subnet) return nil } diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index 0793b6d82fc..3d3794a0c17 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -7,6 +7,7 @@ import ( "fmt" "slices" "strings" + "time" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -432,6 +433,9 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string } } } + // Trigger subnet status update after CR creation or update + time.Sleep(1 * time.Second) + c.updateSubnetStatusQueue.Add(subnet) return nil } diff --git a/pkg/controller/vpc_nat_gw_eip.go b/pkg/controller/vpc_nat_gw_eip.go index a9ebbea8f61..4955449589d 100644 --- a/pkg/controller/vpc_nat_gw_eip.go +++ b/pkg/controller/vpc_nat_gw_eip.go @@ -598,7 +598,6 @@ func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, ext return err } } - externalNetwork := util.GetExternalNetwork(cachedEip.Spec.ExternalSubnet) if needCreate { klog.V(3).Infof("create eip cr %s", key) // Create CR with finalizer, labels and status all at once @@ -639,7 +638,7 @@ func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, ext if eip.Labels == nil { eip.Labels = make(map[string]string) } - eip.Labels[util.SubnetNameLabel] = externalNetwork + eip.Labels[util.SubnetNameLabel] = externalNet eip.Labels[util.VpcNatGatewayNameLabel] = natGwDp eip.Labels[util.EipV4IpLabel] = v4ip if eip.Spec.QoSPolicy != "" { @@ -685,6 +684,9 @@ func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, ext return err } } + // Trigger subnet status update after all operations complete + time.Sleep(1 * time.Second) + c.updateSubnetStatusQueue.Add(externalNet) return nil } From baa100f6722c01cc13540b087c02b6c3026c3f1e Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Tue, 16 Dec 2025 00:29:20 +0800 Subject: [PATCH 26/30] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=AD=90=E7=BD=91?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E6=9B=B4=E6=96=B0=E5=BB=B6=E8=BF=9F=EF=BC=8C?= =?UTF-8?q?=E7=A1=AE=E4=BF=9DAPI=E6=9C=8D=E5=8A=A1=E5=99=A8=E5=AE=8C?= =?UTF-8?q?=E6=88=90finalizer=E7=A7=BB=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/controller/ip.go | 4 +++- pkg/controller/ovn_eip.go | 4 +++- pkg/controller/vip.go | 4 +++- pkg/controller/vpc_nat_gw_eip.go | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/controller/ip.go b/pkg/controller/ip.go index 030fc9b2eef..49b5318cd7a 100644 --- a/pkg/controller/ip.go +++ b/pkg/controller/ip.go @@ -361,6 +361,8 @@ func (c *Controller) handleDelIPFinalizer(cachedIP *kubeovnv1.IP) error { // Trigger subnet status update after finalizer is removed // This ensures subnet status reflects the IP release + // Add delay to ensure API server completes the finalizer removal + time.Sleep(300 * time.Millisecond) c.updateSubnetStatusQueue.Add(cachedIP.Spec.Subnet) for _, as := range cachedIP.Spec.AttachSubnets { c.updateSubnetStatusQueue.Add(as) @@ -525,7 +527,7 @@ func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, } } // Trigger subnet status update after CR creation with finalizer - time.Sleep(1 * time.Second) + time.Sleep(300 * time.Millisecond) c.updateSubnetStatusQueue.Add(ipCR.Spec.Subnet) for _, as := range ipCR.Spec.AttachSubnets { c.updateSubnetStatusQueue.Add(as) diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index a4418ef6a21..e7a5c133cc9 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -410,7 +410,7 @@ func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usageT } } // Trigger subnet status update after CR creation or update - time.Sleep(1 * time.Second) + time.Sleep(300 * time.Millisecond) c.updateSubnetStatusQueue.Add(subnet) return nil } @@ -602,6 +602,8 @@ func (c *Controller) handleDelOvnEipFinalizer(cachedEip *kubeovnv1.OvnEip) error // Trigger subnet status update after finalizer is removed // This ensures subnet status reflects the IP release + // Add delay to ensure API server completes the finalizer removal + time.Sleep(300 * time.Millisecond) c.updateSubnetStatusQueue.Add(cachedEip.Spec.ExternalSubnet) return nil } diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index 3d3794a0c17..a4f44d4ea83 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -434,7 +434,7 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string } } // Trigger subnet status update after CR creation or update - time.Sleep(1 * time.Second) + time.Sleep(300 * time.Millisecond) c.updateSubnetStatusQueue.Add(subnet) return nil } @@ -581,6 +581,8 @@ func (c *Controller) handleDelVipFinalizer(key string) error { // Trigger subnet status update after finalizer is removed // This ensures subnet status reflects the IP release + // Add delay to ensure API server completes the finalizer removal + time.Sleep(300 * time.Millisecond) c.updateSubnetStatusQueue.Add(cachedVip.Spec.Subnet) return nil } diff --git a/pkg/controller/vpc_nat_gw_eip.go b/pkg/controller/vpc_nat_gw_eip.go index 4955449589d..52390e461c5 100644 --- a/pkg/controller/vpc_nat_gw_eip.go +++ b/pkg/controller/vpc_nat_gw_eip.go @@ -685,7 +685,7 @@ func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, ext } } // Trigger subnet status update after all operations complete - time.Sleep(1 * time.Second) + time.Sleep(300 * time.Millisecond) c.updateSubnetStatusQueue.Add(externalNet) return nil } @@ -769,6 +769,8 @@ func (c *Controller) handleDelIptablesEipFinalizer(key string) error { // Trigger subnet status update after finalizer is removed // This ensures subnet status reflects the IP release + // Add delay to ensure API server completes the finalizer removal + time.Sleep(300 * time.Millisecond) externalNetwork := util.GetExternalNetwork(cachedIptablesEip.Spec.ExternalSubnet) c.updateSubnetStatusQueue.Add(externalNetwork) return nil From c2bacde52a8becc12ed5129ef78901ae659854dc Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Tue, 16 Dec 2025 11:09:18 +0800 Subject: [PATCH 27/30] fix webhook Signed-off-by: zbb88888 --- test/e2e/webhook/subnet/subnet.go | 47 +++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/test/e2e/webhook/subnet/subnet.go b/test/e2e/webhook/subnet/subnet.go index e8333b35038..c017ec10c5f 100644 --- a/test/e2e/webhook/subnet/subnet.go +++ b/test/e2e/webhook/subnet/subnet.go @@ -4,6 +4,7 @@ import ( "context" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" "github.com/onsi/ginkgo/v2" @@ -97,25 +98,37 @@ var _ = framework.Describe("[group:webhook-subnet]", func() { subnet := framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil) subnetClient.CreateSync(subnet) + // TODO: Use Patch instead of Update to modify only the Spec.Vpc field. + // This would avoid resource conflicts caused by concurrent status updates from the controller. + // Example: subnetClient.Patch(subnet, modifiedSubnet, framework.Timeout()) ginkgo.By("Validating vpc can be changed from empty to ovn-cluster") - subnet = subnetClient.Get(subnetName) - modifiedSubnet := subnet.DeepCopy() - modifiedSubnet.Spec.Vpc = util.DefaultVpc - _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + subnet = subnetClient.Get(subnetName) + modifiedSubnet := subnet.DeepCopy() + modifiedSubnet.Spec.Vpc = util.DefaultVpc + _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + return err + }) framework.ExpectNoError(err) ginkgo.By("Validating vpc cannot be changed from ovn-cluster to another value") - subnet = subnetClient.Get(subnetName) - modifiedSubnet = subnet.DeepCopy() - modifiedSubnet.Spec.Vpc = "test-vpc" - _, err = subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + subnet = subnetClient.Get(subnetName) + modifiedSubnet := subnet.DeepCopy() + modifiedSubnet.Spec.Vpc = "test-vpc" + _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + return err + }) framework.ExpectError(err, "vpc can only be changed from empty to ovn-cluster") ginkgo.By("Validating vpc cannot be changed from ovn-cluster to empty") - subnet = subnetClient.Get(subnetName) - modifiedSubnet = subnet.DeepCopy() - modifiedSubnet.Spec.Vpc = "" - _, err = subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + subnet = subnetClient.Get(subnetName) + modifiedSubnet := subnet.DeepCopy() + modifiedSubnet.Spec.Vpc = "" + _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + return err + }) framework.ExpectError(err, "vpc can only be changed from empty to ovn-cluster") }) @@ -126,9 +139,13 @@ var _ = framework.Describe("[group:webhook-subnet]", func() { subnet = subnetClient.CreateSync(subnet) ginkgo.By("Validating vpc cannot be changed from empty to non-ovn-cluster value") - modifiedSubnet := subnet.DeepCopy() - modifiedSubnet.Spec.Vpc = "test-vpc" - _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + subnet = subnetClient.Get(subnetName) + modifiedSubnet := subnet.DeepCopy() + modifiedSubnet.Spec.Vpc = "test-vpc" + _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + return err + }) framework.ExpectError(err, "vpc can only be changed from empty to ovn-cluster") }) }) From 5651048555dfee6fcf7eca0bd3a203e1689d20be Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Tue, 16 Dec 2025 15:28:36 +0800 Subject: [PATCH 28/30] =?UTF-8?q?=E4=BC=98=E5=8C=96OVN=E5=AE=A2=E6=88=B7?= =?UTF-8?q?=E7=AB=AF=E5=88=9B=E5=BB=BA=E9=80=BB=E8=BE=91=EF=BC=8C=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E9=87=8D=E8=AF=95=E6=9C=BA=E5=88=B6=E4=BB=A5=E6=8F=90?= =?UTF-8?q?=E9=AB=98=E8=BF=9E=E6=8E=A5=E6=88=90=E5=8A=9F=E7=8E=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/ovs/ovn.go | 64 ++++++++++++++++++--------- test/e2e/framework/ovn_address_set.go | 49 ++++++++++++++++---- 2 files changed, 85 insertions(+), 28 deletions(-) diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index 8e003344aaa..75bab91f307 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -63,7 +63,7 @@ func NewLegacyClient(timeout int) *LegacyClient { func NewDynamicOvnNbClient( ovnNbAddr string, - ovnNbTimeout, ovsDbConTimeout, ovsDbInactivityTimeout int, + ovnNbTimeout, ovsDbConTimeout, ovsDbInactivityTimeout, maxRetry int, tables ...string, ) (*OVNNbClient, map[string]model.Model, error) { dbModel, err := model.NewClientDBModel(ovnnb.DatabaseName, nil) @@ -71,16 +71,28 @@ func NewDynamicOvnNbClient( return nil, nil, fmt.Errorf("failed to create client db model: %w", err) } - nbClient, err := ovsclient.NewOvsDbClient( - ovsclient.NBDB, - ovnNbAddr, - dbModel, - nil, - ovsDbConTimeout, - ovsDbInactivityTimeout, - ) - if err != nil { - return nil, nil, fmt.Errorf("failed to create initial ovsdb client to fetch schema: %w", err) + // First connection to fetch schema with retry + var nbClient client.Client + try := 0 + for { + nbClient, err = ovsclient.NewOvsDbClient( + ovsclient.NBDB, + ovnNbAddr, + dbModel, + nil, + ovsDbConTimeout, + ovsDbInactivityTimeout, + ) + if err != nil { + klog.Errorf("failed to create initial OVN NB client to fetch schema: %v", err) + } else { + break + } + if try >= maxRetry { + return nil, nil, fmt.Errorf("failed to create initial ovsdb client to fetch schema after %d retries: %w", maxRetry, err) + } + time.Sleep(2 * time.Second) + try++ } schemaTables := nbClient.Schema().Tables @@ -117,15 +129,27 @@ func NewDynamicOvnNbClient( return nil, nil, fmt.Errorf("failed to create dynamic client db model: %w", err) } - if nbClient, err = ovsclient.NewOvsDbClient( - ovsclient.NBDB, - ovnNbAddr, - dbModel, - monitors, - ovsDbConTimeout, - ovsDbInactivityTimeout, - ); err != nil { - return nil, nil, fmt.Errorf("failed to create dynamic ovsdb client: %w", err) + // Second connection with dynamic model and retry + try = 0 + for { + nbClient, err = ovsclient.NewOvsDbClient( + ovsclient.NBDB, + ovnNbAddr, + dbModel, + monitors, + ovsDbConTimeout, + ovsDbInactivityTimeout, + ) + if err != nil { + klog.Errorf("failed to create dynamic OVN NB client: %v", err) + } else { + break + } + if try >= maxRetry { + return nil, nil, fmt.Errorf("failed to create dynamic ovsdb client after %d retries: %w", maxRetry, err) + } + time.Sleep(2 * time.Second) + try++ } c := &OVNNbClient{ diff --git a/test/e2e/framework/ovn_address_set.go b/test/e2e/framework/ovn_address_set.go index 1211e51ad73..88749c6f8d2 100644 --- a/test/e2e/framework/ovn_address_set.go +++ b/test/e2e/framework/ovn_address_set.go @@ -176,15 +176,48 @@ func getOVNNbClient(tables ...string) (*ovs.OVNNbClient, map[string]model.Model) }) ExpectNoError(ovnnbAddrErr) - client, models, err := ovs.NewDynamicOvnNbClient( - ovnnbAddr, - ovnNbTimeoutSeconds, - ovsdbConnTimeout, - ovsdbInactivityTimeout, - tables..., - ) - ExpectNoError(err) + var client *ovs.OVNNbClient + var models map[string]model.Model + var err error + + // Retry the entire client creation and connection verification process + for try := 0; try <= ovnClientMaxRetry; try++ { + client, models, err = ovs.NewDynamicOvnNbClient( + ovnnbAddr, + ovnNbTimeoutSeconds, + ovsdbConnTimeout, + ovsdbInactivityTimeout, + ovnClientMaxRetry, + tables..., + ) + if err != nil { + if try < ovnClientMaxRetry { + Logf("Failed to create OVN NB client (attempt %d/%d): %v, retrying...", try+1, ovnClientMaxRetry+1, err) + time.Sleep(2 * time.Second) + continue + } + break + } + + // Verify the connection is actually working by checking if we're connected + connected := client.Connected() + if !connected { + client.Close() + err = fmt.Errorf("client created but not connected") + if try < ovnClientMaxRetry { + Logf("OVN NB client not connected (attempt %d/%d), retrying...", try+1, ovnClientMaxRetry+1) + time.Sleep(2 * time.Second) + continue + } + break + } + + // Connection verified, return the client + return client, models + } + + ExpectNoError(err) return client, models } From e90f7c1c30ad3ae9f4fdd6a935f6791467577f4e Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Tue, 16 Dec 2025 15:37:28 +0800 Subject: [PATCH 29/30] fix err Signed-off-by: zbb88888 --- test/e2e/framework/ovn_address_set.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/framework/ovn_address_set.go b/test/e2e/framework/ovn_address_set.go index 88749c6f8d2..18453c3fd93 100644 --- a/test/e2e/framework/ovn_address_set.go +++ b/test/e2e/framework/ovn_address_set.go @@ -204,7 +204,7 @@ func getOVNNbClient(tables ...string) (*ovs.OVNNbClient, map[string]model.Model) if !connected { client.Close() - err = fmt.Errorf("client created but not connected") + err = errors.New("client created but not connected") if try < ovnClientMaxRetry { Logf("OVN NB client not connected (attempt %d/%d), retrying...", try+1, ovnClientMaxRetry+1) time.Sleep(2 * time.Second) From 7973f46a5d917c087c99bc782a4b99e4605d3670 Mon Sep 17 00:00:00 2001 From: zbb88888 Date: Wed, 17 Dec 2025 16:41:53 +0800 Subject: [PATCH 30/30] make sure all del queue refer to a new deepcopy object, the origin ptr refered object is not managed by kube-ovn-controller Signed-off-by: zbb88888 --- pkg/controller/admin_network_policy.go | 2 +- pkg/controller/baseline_admin_network_policy.go | 2 +- pkg/controller/cluster_network_policy.go | 2 +- pkg/controller/dns_name_resolver.go | 2 +- pkg/controller/ip.go | 2 +- pkg/controller/ippool.go | 2 +- pkg/controller/ovn_eip.go | 2 +- pkg/controller/service.go | 2 +- pkg/controller/subnet.go | 2 +- pkg/controller/vip.go | 2 +- pkg/controller/vpc.go | 2 +- pkg/controller/vpc_nat_gw_eip.go | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/controller/admin_network_policy.go b/pkg/controller/admin_network_policy.go index 43f141ec508..d4a5ead4f63 100644 --- a/pkg/controller/admin_network_policy.go +++ b/pkg/controller/admin_network_policy.go @@ -68,7 +68,7 @@ func (c *Controller) enqueueDeleteAnp(obj any) { } klog.V(3).Infof("enqueue delete anp %s", cache.MetaObjectToName(anp).String()) - c.deleteAnpQueue.Add(anp) + c.deleteAnpQueue.Add(anp.DeepCopy()) } func (c *Controller) enqueueUpdateAnp(oldObj, newObj any) { diff --git a/pkg/controller/baseline_admin_network_policy.go b/pkg/controller/baseline_admin_network_policy.go index ff4372fdd43..da0ee7db66c 100644 --- a/pkg/controller/baseline_admin_network_policy.go +++ b/pkg/controller/baseline_admin_network_policy.go @@ -40,7 +40,7 @@ func (c *Controller) enqueueDeleteBanp(obj any) { } klog.V(3).Infof("enqueue delete bnp %s", cache.MetaObjectToName(bnp).String()) - c.deleteBanpQueue.Add(bnp) + c.deleteBanpQueue.Add(bnp.DeepCopy()) } func (c *Controller) enqueueUpdateBanp(oldObj, newObj any) { diff --git a/pkg/controller/cluster_network_policy.go b/pkg/controller/cluster_network_policy.go index f9ef31a0fc6..c81073a6c39 100644 --- a/pkg/controller/cluster_network_policy.go +++ b/pkg/controller/cluster_network_policy.go @@ -99,7 +99,7 @@ func (c *Controller) enqueueDeleteCnp(obj any) { } klog.V(3).Infof("enqueue delete cnp %s", cache.MetaObjectToName(cnp).String()) - c.deleteCnpQueue.Add(cnp) + c.deleteCnpQueue.Add(cnp.DeepCopy()) } func (c *Controller) handleAddCnp(key string) (err error) { diff --git a/pkg/controller/dns_name_resolver.go b/pkg/controller/dns_name_resolver.go index e2231165521..b30cbf26142 100644 --- a/pkg/controller/dns_name_resolver.go +++ b/pkg/controller/dns_name_resolver.go @@ -51,7 +51,7 @@ func (c *Controller) enqueueDeleteDNSNameResolver(obj any) { } klog.V(3).Infof("enqueue delete dns name resolver %s", cache.MetaObjectToName(dnsNameResolver).String()) - c.deleteDNSNameResolverQueue.Add(dnsNameResolver) + c.deleteDNSNameResolverQueue.Add(dnsNameResolver.DeepCopy()) } func (c *Controller) handleAddOrUpdateDNSNameResolver(key string) error { diff --git a/pkg/controller/ip.go b/pkg/controller/ip.go index 49b5318cd7a..329c88f4d00 100644 --- a/pkg/controller/ip.go +++ b/pkg/controller/ip.go @@ -107,7 +107,7 @@ func (c *Controller) enqueueDelIP(obj any) { key := cache.MetaObjectToName(ipObj).String() klog.V(3).Infof("enqueue del ip %s", key) - c.delIPQueue.Add(ipObj) + c.delIPQueue.Add(ipObj.DeepCopy()) } func (c *Controller) handleAddReservedIP(key string) error { diff --git a/pkg/controller/ippool.go b/pkg/controller/ippool.go index 04c443f3fe4..829204b34b0 100644 --- a/pkg/controller/ippool.go +++ b/pkg/controller/ippool.go @@ -44,7 +44,7 @@ func (c *Controller) enqueueDeleteIPPool(obj any) { } klog.V(3).Infof("enqueue delete ippool %s", cache.MetaObjectToName(ippool).String()) - c.deleteIPPoolQueue.Add(ippool) + c.deleteIPPoolQueue.Add(ippool.DeepCopy()) } func (c *Controller) enqueueUpdateIPPool(oldObj, newObj any) { diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index e7a5c133cc9..1e23ca1def0 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -73,7 +73,7 @@ func (c *Controller) enqueueDelOvnEip(obj any) { key := cache.MetaObjectToName(eip).String() klog.Infof("enqueue del ovn eip %s", key) - c.delOvnEipQueue.Add(eip) + c.delOvnEipQueue.Add(eip.DeepCopy()) } func (c *Controller) handleAddOvnEip(key string) error { diff --git a/pkg/controller/service.go b/pkg/controller/service.go index b803df5bff8..948e16f89fd 100644 --- a/pkg/controller/service.go +++ b/pkg/controller/service.go @@ -107,7 +107,7 @@ func (c *Controller) enqueueDeleteService(obj any) { vpcSvc := &vpcService{ Protocol: port.Protocol, Vpc: svc.Annotations[util.VpcAnnotation], - Svc: svc, + Svc: svc.DeepCopy(), } for _, ip := range ips { vpcSvc.Vips = append(vpcSvc.Vips, util.JoinHostPort(ip, port.Port)) diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index 9ce55204d6b..e6a7d64fa2c 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -53,7 +53,7 @@ func (c *Controller) enqueueDeleteSubnet(obj any) { } klog.V(3).Infof("enqueue delete subnet %s", subnet.Name) - c.deleteSubnetQueue.Add(subnet) + c.deleteSubnetQueue.Add(subnet.DeepCopy()) } func readyToRemoveFinalizer(subnet *kubeovnv1.Subnet) bool { diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index a4f44d4ea83..901d830dac1 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -64,7 +64,7 @@ func (c *Controller) enqueueDelVirtualIP(obj any) { key := cache.MetaObjectToName(vip).String() klog.Infof("enqueue del vip %s", key) - c.delVirtualIPQueue.Add(vip) + c.delVirtualIPQueue.Add(vip.DeepCopy()) } func (c *Controller) handleAddVirtualIP(key string) error { diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index 8477ea41323..7b2667f4b2b 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -97,7 +97,7 @@ func (c *Controller) enqueueDelVpc(obj any) { if _, ok := vpc.Labels[util.VpcExternalLabel]; !vpc.Status.Default || !ok { klog.V(3).Infof("enqueue delete vpc %s", vpc.Name) - c.delVpcQueue.Add(vpc) + c.delVpcQueue.Add(vpc.DeepCopy()) } } diff --git a/pkg/controller/vpc_nat_gw_eip.go b/pkg/controller/vpc_nat_gw_eip.go index 52390e461c5..57458c28906 100644 --- a/pkg/controller/vpc_nat_gw_eip.go +++ b/pkg/controller/vpc_nat_gw_eip.go @@ -59,7 +59,7 @@ func (c *Controller) enqueueDelIptablesEip(obj any) { key := cache.MetaObjectToName(eip).String() klog.Infof("enqueue del iptables eip %s", key) - c.delIptablesEipQueue.Add(eip) + c.delIptablesEipQueue.Add(eip.DeepCopy()) } func (c *Controller) handleAddIptablesEip(key string) error {