diff --git a/makefiles/e2e.mk b/makefiles/e2e.mk index 92db5979fd4..8c623cd97d2 100644 --- a/makefiles/e2e.mk +++ b/makefiles/e2e.mk @@ -87,6 +87,7 @@ e2e-build: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/multus ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/non-primary-cni ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ip ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vip ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vpc-egress-gateway ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw @@ -188,6 +189,15 @@ kube-ovn-lb-svc-conformance-e2e: ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ --focus=CNI:Kube-OVN ./test/e2e/lb-svc/lb-svc.test -- $(TEST_BIN_ARGS) +.PHONY: ip-conformance-e2e +ip-conformance-e2e: + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ip + E2E_BRANCH=$(E2E_BRANCH) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ + ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ + --focus=CNI:Kube-OVN ./test/e2e/ip/ip.test -- $(TEST_BIN_ARGS) + .PHONY: vip-conformance-e2e vip-conformance-e2e: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vip @@ -205,8 +215,8 @@ vpc-egress-gateway-e2e: ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v --timeout=30m \ --focus=CNI:Kube-OVN ./test/e2e/vpc-egress-gateway/vpc-egress-gateway.test -- $(TEST_BIN_ARGS) -.PHONY: iptables-vpc-nat-gw-conformance-e2e -iptables-vpc-nat-gw-conformance-e2e: +.PHONY: iptables-eip-conformance-e2e +iptables-eip-conformance-e2e: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw E2E_BRANCH=$(E2E_BRANCH) \ E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ @@ -214,6 +224,22 @@ iptables-vpc-nat-gw-conformance-e2e: ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ --focus=CNI:Kube-OVN ./test/e2e/iptables-vpc-nat-gw/iptables-vpc-nat-gw.test -- $(TEST_BIN_ARGS) +.PHONY: iptables-eip-qos-conformance-e2e +iptables-eip-qos-conformance-e2e: + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-eip-qos + E2E_BRANCH=$(E2E_BRANCH) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ + ginkgo $(GINKGO_OUTPUT_OPT) --randomize-all -v \ + --focus=CNI:Kube-OVN ./test/e2e/iptables-eip-qos/iptables-eip-qos.test -- $(TEST_BIN_ARGS) + +.PHONY: iptables-vpc-nat-gw-conformance-e2e +iptables-vpc-nat-gw-conformance-e2e: + $(MAKE) iptables-eip-conformance-e2e + $(MAKE) iptables-eip-qos-conformance-e2e + + + .PHONY: ovn-vpc-nat-gw-conformance-e2e ovn-vpc-nat-gw-conformance-e2e: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ovn-vpc-nat-gw diff --git a/makefiles/ut.mk b/makefiles/ut.mk index 4ea06fa06cf..2ed2eccddbf 100644 --- a/makefiles/ut.mk +++ b/makefiles/ut.mk @@ -4,6 +4,8 @@ ut: ginkgo -mod=mod --show-node-events --poll-progress-after=60s $(GINKGO_OUTPUT_OPT) -v test/unittest go test -coverprofile=profile.cov $$(go list ./pkg/... | grep -vw '^github.com/kubeovn/kube-ovn/pkg/client') + @echo "Running e2e framework unit tests..." + go test -v ./test/e2e/framework/docker .PHONY: ovs-sandbox ovs-sandbox: clean-ovs-sandbox diff --git a/pkg/controller/admin_network_policy.go b/pkg/controller/admin_network_policy.go index 43f141ec508..d4a5ead4f63 100644 --- a/pkg/controller/admin_network_policy.go +++ b/pkg/controller/admin_network_policy.go @@ -68,7 +68,7 @@ func (c *Controller) enqueueDeleteAnp(obj any) { } klog.V(3).Infof("enqueue delete anp %s", cache.MetaObjectToName(anp).String()) - c.deleteAnpQueue.Add(anp) + c.deleteAnpQueue.Add(anp.DeepCopy()) } func (c *Controller) enqueueUpdateAnp(oldObj, newObj any) { diff --git a/pkg/controller/baseline_admin_network_policy.go b/pkg/controller/baseline_admin_network_policy.go index ff4372fdd43..da0ee7db66c 100644 --- a/pkg/controller/baseline_admin_network_policy.go +++ b/pkg/controller/baseline_admin_network_policy.go @@ -40,7 +40,7 @@ func (c *Controller) enqueueDeleteBanp(obj any) { } klog.V(3).Infof("enqueue delete bnp %s", cache.MetaObjectToName(bnp).String()) - c.deleteBanpQueue.Add(bnp) + c.deleteBanpQueue.Add(bnp.DeepCopy()) } func (c *Controller) enqueueUpdateBanp(oldObj, newObj any) { diff --git a/pkg/controller/cluster_network_policy.go b/pkg/controller/cluster_network_policy.go index f9ef31a0fc6..c81073a6c39 100644 --- a/pkg/controller/cluster_network_policy.go +++ b/pkg/controller/cluster_network_policy.go @@ -99,7 +99,7 @@ func (c *Controller) enqueueDeleteCnp(obj any) { } klog.V(3).Infof("enqueue delete cnp %s", cache.MetaObjectToName(cnp).String()) - c.deleteCnpQueue.Add(cnp) + c.deleteCnpQueue.Add(cnp.DeepCopy()) } func (c *Controller) handleAddCnp(key string) (err error) { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index d13e8a72a9c..984a659ea93 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -159,7 +159,7 @@ type Controller struct { addIptablesEipQueue workqueue.TypedRateLimitingInterface[string] updateIptablesEipQueue workqueue.TypedRateLimitingInterface[string] resetIptablesEipQueue workqueue.TypedRateLimitingInterface[string] - delIptablesEipQueue workqueue.TypedRateLimitingInterface[string] + delIptablesEipQueue workqueue.TypedRateLimitingInterface[*kubeovnv1.IptablesEIP] iptablesFipsLister kubeovnlister.IptablesFIPRuleLister iptablesFipSynced cache.InformerSynced @@ -184,7 +184,7 @@ type Controller struct { addOvnEipQueue workqueue.TypedRateLimitingInterface[string] updateOvnEipQueue workqueue.TypedRateLimitingInterface[string] resetOvnEipQueue workqueue.TypedRateLimitingInterface[string] - delOvnEipQueue workqueue.TypedRateLimitingInterface[string] + delOvnEipQueue workqueue.TypedRateLimitingInterface[*kubeovnv1.OvnEip] ovnFipsLister kubeovnlister.OvnFipLister ovnFipSynced cache.InformerSynced @@ -472,7 +472,7 @@ func Run(ctx context.Context, config *Configuration) { addIptablesEipQueue: newTypedRateLimitingQueue("AddIptablesEip", custCrdRateLimiter), updateIptablesEipQueue: newTypedRateLimitingQueue("UpdateIptablesEip", custCrdRateLimiter), resetIptablesEipQueue: newTypedRateLimitingQueue("ResetIptablesEip", custCrdRateLimiter), - delIptablesEipQueue: newTypedRateLimitingQueue("DeleteIptablesEip", custCrdRateLimiter), + delIptablesEipQueue: newTypedRateLimitingQueue[*kubeovnv1.IptablesEIP]("DeleteIptablesEip", nil), iptablesFipsLister: iptablesFipInformer.Lister(), iptablesFipSynced: iptablesFipInformer.Informer().HasSynced, @@ -563,7 +563,7 @@ func Run(ctx context.Context, config *Configuration) { addOvnEipQueue: newTypedRateLimitingQueue("AddOvnEip", custCrdRateLimiter), updateOvnEipQueue: newTypedRateLimitingQueue("UpdateOvnEip", custCrdRateLimiter), resetOvnEipQueue: newTypedRateLimitingQueue("ResetOvnEip", custCrdRateLimiter), - delOvnEipQueue: newTypedRateLimitingQueue("DeleteOvnEip", custCrdRateLimiter), + delOvnEipQueue: newTypedRateLimitingQueue[*kubeovnv1.OvnEip]("DeleteOvnEip", nil), ovnFipsLister: ovnFipInformer.Lister(), ovnFipSynced: ovnFipInformer.Informer().HasSynced, diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 7a6bcdc5057..2457fdbdca7 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -117,6 +117,7 @@ func newFakeControllerWithOptions(t *testing.T, opts *FakeControllerOptions) (*f serviceInformer := kubeInformerFactory.Core().V1().Services() namespaceInformer := kubeInformerFactory.Core().V1().Namespaces() podInformer := kubeInformerFactory.Core().V1().Pods() + configMapInformer := kubeInformerFactory.Core().V1().ConfigMaps() nadInformerFactory := nadinformers.NewSharedInformerFactory(nadClient, 0) nadInformer := nadInformerFactory.K8sCniCncfIo().V1().NetworkAttachmentDefinitions() @@ -149,6 +150,7 @@ func newFakeControllerWithOptions(t *testing.T, opts *FakeControllerOptions) (*f subnetSynced: alwaysReady, netAttachLister: nadInformer.Lister(), netAttachSynced: alwaysReady, + configMapsLister: configMapInformer.Lister(), OVNNbClient: mockOvnClient, syncVirtualPortsQueue: newTypedRateLimitingQueue[string]("SyncVirtualPort", nil), } diff --git a/pkg/controller/dns_name_resolver.go b/pkg/controller/dns_name_resolver.go index e2231165521..b30cbf26142 100644 --- a/pkg/controller/dns_name_resolver.go +++ b/pkg/controller/dns_name_resolver.go @@ -51,7 +51,7 @@ func (c *Controller) enqueueDeleteDNSNameResolver(obj any) { } klog.V(3).Infof("enqueue delete dns name resolver %s", cache.MetaObjectToName(dnsNameResolver).String()) - c.deleteDNSNameResolverQueue.Add(dnsNameResolver) + c.deleteDNSNameResolverQueue.Add(dnsNameResolver.DeepCopy()) } func (c *Controller) handleAddOrUpdateDNSNameResolver(key string) error { diff --git a/pkg/controller/external_gw.go b/pkg/controller/external_gw.go index 34c07f20658..687cf0cddbf 100644 --- a/pkg/controller/external_gw.go +++ b/pkg/controller/external_gw.go @@ -145,8 +145,16 @@ func (c *Controller) establishExternalGateway(config map[string]string) error { klog.Errorf("failed to get gateway chassis, %v", err) return err } + + // Get external gateway switch using centralized logic + externalGwSwitch, err := c.getExternalGatewaySwitchWithConfigMap(config) + if err != nil { + klog.Errorf("failed to get external gateway switch: %v", err) + return err + } + var lrpIP, lrpMac string - lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch) + lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, externalGwSwitch) lrp, err := c.OVNNbClient.GetLogicalRouterPort(lrpName, true) if err != nil { klog.Errorf("failed to get lrp %s, %v", lrpName, err) @@ -159,7 +167,7 @@ func (c *Controller) establishExternalGateway(config map[string]string) error { lrpMac = lrp.MAC lrpIP = lrp.Networks[0] case config["nic-ip"] == "": - if lrpIP, lrpMac, err = c.createDefaultVpcLrpEip(); err != nil { + if lrpIP, lrpMac, err = c.createDefaultVpcLrpEip(externalGwSwitch); err != nil { klog.Errorf("failed to create ovn eip for default vpc lrp: %v", err) return err } @@ -168,22 +176,22 @@ func (c *Controller) establishExternalGateway(config map[string]string) error { lrpMac = config["nic-mac"] } - if err := c.OVNNbClient.CreateGatewayLogicalSwitch(c.config.ExternalGatewaySwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIP, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil { - klog.Errorf("failed to create external gateway switch %s: %v", c.config.ExternalGatewaySwitch, err) + if err := c.OVNNbClient.CreateGatewayLogicalSwitch(externalGwSwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIP, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil { + klog.Errorf("failed to create external gateway switch %s: %v", externalGwSwitch, err) return err } return nil } -func (c *Controller) createDefaultVpcLrpEip() (string, string, error) { - cachedSubnet, err := c.subnetsLister.Get(c.config.ExternalGatewaySwitch) +func (c *Controller) createDefaultVpcLrpEip(externalGwSwitch string) (string, string, error) { + cachedSubnet, err := c.subnetsLister.Get(externalGwSwitch) if err != nil { - klog.Errorf("failed to get subnet %s, %v", c.config.ExternalGatewaySwitch, err) + klog.Errorf("failed to get subnet %s, %v", externalGwSwitch, err) return "", "", err } needCreateEip := false - lrpEipName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch) + lrpEipName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, externalGwSwitch) cachedEip, err := c.ovnEipsLister.Get(lrpEipName) if err != nil { if !k8serrors.IsNotFound(err) { @@ -203,12 +211,12 @@ func (c *Controller) createDefaultVpcLrpEip() (string, string, error) { } } else { var v6ip string - v4ip, v6ip, mac, err = c.acquireIPAddress(c.config.ExternalGatewaySwitch, lrpEipName, lrpEipName) + v4ip, v6ip, mac, err = c.acquireIPAddress(externalGwSwitch, lrpEipName, lrpEipName) if err != nil { klog.Errorf("failed to acquire ip address for default vpc lrp %s, %v", lrpEipName, err) return "", "", err } - if err := c.createOrUpdateOvnEipCR(lrpEipName, c.config.ExternalGatewaySwitch, v4ip, v6ip, mac, util.OvnEipTypeLRP); err != nil { + if err := c.createOrUpdateOvnEipCR(lrpEipName, externalGwSwitch, v4ip, v6ip, mac, util.OvnEipTypeLRP); err != nil { klog.Errorf("failed to create ovn lrp eip %s, %v", lrpEipName, err) return "", "", err } @@ -221,6 +229,83 @@ func (c *Controller) createDefaultVpcLrpEip() (string, string, error) { return v4ipCidr, mac, nil } +// getExternalGatewaySwitchWithConfigMap determines which external gateway switch to use. +// Two modes: +// - Traditional mode: c.config.ExternalGatewaySwitch (OVN LogicalSwitch, NOT managed by Subnet CRD) +// - ConfigMap mode: User-specified subnet in ConfigMap (managed by Subnet CRD) +// Logic: +// 1. ConfigMap not specified -> use default (traditional mode) +// 2. ConfigMap same as default -> use default (traditional mode) +// 3. ConfigMap different from default -> check conflict and verify ConfigMap subnet exists +func (c *Controller) getExternalGatewaySwitchWithConfigMap(configData map[string]string) (string, error) { + configMapSwitch := configData["external-gw-switch"] + defaultSwitch := c.config.ExternalGatewaySwitch + + // 1. ConfigMap not specified -> use default + if configMapSwitch == "" { + return defaultSwitch, nil + } + + // 2. ConfigMap specified same as default -> use default + if configMapSwitch == defaultSwitch { + return defaultSwitch, nil + } + + // 3. ConfigMap specified different from default + // Check if default logical switch exists in OVN (configuration conflict) + // Note: c.config.ExternalGatewaySwitch is OVN LogicalSwitch, NOT Subnet CRD + exists, err := c.OVNNbClient.LogicalSwitchExists(defaultSwitch) + if err != nil { + klog.Errorf("failed to check if default logical switch %s exists: %v", defaultSwitch, err) + return "", err + } + if exists { + // Default logical switch exists - conflict + err := fmt.Errorf("configuration conflict: default external logical switch %s exists, but ConfigMap specifies different subnet %s. Please use only one mode: either remove the default logical switch or remove the ConfigMap setting", defaultSwitch, configMapSwitch) + klog.Error(err) + return "", err + } + + // Default subnet does not exist, verify ConfigMap-specified subnet exists + _, err = c.subnetsLister.Get(configMapSwitch) + if err != nil { + if k8serrors.IsNotFound(err) { + err := fmt.Errorf("ConfigMap specifies external subnet %s, but it does not exist. Please create the subnet first or update the ConfigMap", configMapSwitch) + klog.Error(err) + return "", err + } + err := fmt.Errorf("failed to get subnet %s from lister: %w", configMapSwitch, err) + klog.Error(err) + return "", err + } + + return configMapSwitch, nil +} + +// getConfigDefaultExternalSwitch determines which (from config or configmap) external gateway switch to use +// ConfigMap not specified -> use default; +// default not exists + ConfigMap specified -> use ConfigMap +func (c *Controller) getConfigDefaultExternalSwitch() (string, error) { + cm, err := c.configMapsLister.ConfigMaps(c.config.ExternalGatewayConfigNS).Get(util.ExternalGatewayConfig) + if err != nil { + if k8serrors.IsNotFound(err) { + // ConfigMap doesn't exist, use default + return c.config.ExternalGatewaySwitch, nil + } + err = fmt.Errorf("failed to get ConfigMap %s: %w", util.ExternalGatewayConfig, err) + klog.Error(err) + return "", err + } + + // Check if ConfigMap is enabled + if cm.Data["enable-external-gw"] == "false" { + return c.config.ExternalGatewaySwitch, nil + } + + // Use the centralized logic + return c.getExternalGatewaySwitchWithConfigMap(cm.Data) +} + func (c *Controller) getGatewayChassis(config map[string]string) ([]string, error) { chassises := []string{} nodes, err := c.nodesLister.List(externalGatewayNodeSelector) diff --git a/pkg/controller/external_gw_test.go b/pkg/controller/external_gw_test.go new file mode 100644 index 00000000000..e696e60b859 --- /dev/null +++ b/pkg/controller/external_gw_test.go @@ -0,0 +1,402 @@ +package controller + +import ( + "context" + "strings" + "testing" + "time" + + "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + + mockovs "github.com/kubeovn/kube-ovn/mocks/pkg/ovs" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + kubeovnfake "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/fake" + kubeovninformers "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// Helper function to create a test controller with optional initial objects +// defaultLSExists controls whether the default logical switch "external" exists in OVN +func newTestController(t *testing.T, subnets []*kubeovnv1.Subnet, configMaps []*corev1.ConfigMap, defaultLSExists bool) *Controller { + // Create clientsets + var kubeObjects []runtime.Object + for _, cm := range configMaps { + kubeObjects = append(kubeObjects, cm) + } + kubeClient := fake.NewSimpleClientset(kubeObjects...) + + var kubeovnObjects []runtime.Object + for _, subnet := range subnets { + kubeovnObjects = append(kubeovnObjects, subnet) + } + kubeOvnClient := kubeovnfake.NewSimpleClientset(kubeovnObjects...) + + // Create informer factories + kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, 0) + kubeovnInformerFactory := kubeovninformers.NewSharedInformerFactory(kubeOvnClient, 0) + + config := &Configuration{ + KubeClient: kubeClient, + KubeOvnClient: kubeOvnClient, + ExternalGatewaySwitch: "external", // Default name + ExternalGatewayConfigNS: "kube-system", + } + + // Create mock OVN client + mockOvnClient := mockovs.NewMockNbClient(gomock.NewController(t)) + // Set expectation for LogicalSwitchExists based on defaultLSExists parameter + mockOvnClient.EXPECT().LogicalSwitchExists("external").Return(defaultLSExists, nil).AnyTimes() + + controller := &Controller{ + config: config, + configMapsLister: kubeInformerFactory.Core().V1().ConfigMaps().Lister(), + subnetsLister: kubeovnInformerFactory.Kubeovn().V1().Subnets().Lister(), + OVNNbClient: mockOvnClient, + } + + // Start informers and wait for cache sync + stopCh := make(chan struct{}) + defer close(stopCh) + + kubeInformerFactory.Start(stopCh) + kubeovnInformerFactory.Start(stopCh) + + kubeInformerFactory.WaitForCacheSync(stopCh) + kubeovnInformerFactory.WaitForCacheSync(stopCh) + + // Give informers time to sync + time.Sleep(100 * time.Millisecond) + + return controller +} + +// Test Scenario 1: Default "external" subnet does NOT exist, ConfigMap NOT specified +// Expected: Return default name "external" (will fail later with subnet not found) +func TestGetExternalGatewaySwitch_Scenario1_DefaultNotExist_ConfigMapNotSpecified(t *testing.T) { + c := newTestController(t, nil, nil, false) + + result, err := c.getConfigDefaultExternalSwitch() + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + + expected := "external" + if result != expected { + t.Errorf("expected %s, got %s", expected, result) + } +} + +// Test Scenario 2: Default "external" subnet EXISTS, ConfigMap NOT specified +// Expected: Use default "external" +func TestGetExternalGatewaySwitch_Scenario2_DefaultExists_ConfigMapNotSpecified(t *testing.T) { + defaultSubnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "external", + }, + } + + c := newTestController(t, []*kubeovnv1.Subnet{defaultSubnet}, nil, true) + + result, err := c.getConfigDefaultExternalSwitch() + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + + expected := "external" + if result != expected { + t.Errorf("expected %s, got %s", expected, result) + } +} + +// Test Scenario 3: Default "external" subnet does NOT exist, ConfigMap specifies "custom-ext" +// Expected: Use "custom-ext" +func TestGetExternalGatewaySwitch_Scenario3_DefaultNotExist_ConfigMapSpecifiedDifferent(t *testing.T) { + // Create the custom-ext subnet that ConfigMap references + customSubnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-ext", + }, + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.ExternalGatewayConfig, + Namespace: "kube-system", + }, + Data: map[string]string{ + "enable-external-gw": "true", + "external-gw-switch": "custom-ext", + }, + } + + c := newTestController(t, []*kubeovnv1.Subnet{customSubnet}, []*corev1.ConfigMap{configMap}, false) + + result, err := c.getConfigDefaultExternalSwitch() + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + + expected := "custom-ext" + if result != expected { + t.Errorf("expected %s, got %s", expected, result) + } +} + +// Test Scenario 4: Default "external" subnet EXISTS, ConfigMap specifies "custom-ext" (different) +// Expected: ERROR - configuration conflict +func TestGetExternalGatewaySwitch_Scenario4_DefaultExists_ConfigMapSpecifiedDifferent(t *testing.T) { + defaultSubnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "external", + }, + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.ExternalGatewayConfig, + Namespace: "kube-system", + }, + Data: map[string]string{ + "enable-external-gw": "true", + "external-gw-switch": "custom-ext", + }, + } + + c := newTestController(t, []*kubeovnv1.Subnet{defaultSubnet}, []*corev1.ConfigMap{configMap}, true) + + _, err := c.getConfigDefaultExternalSwitch() + if err == nil { + t.Error("expected error due to configuration conflict, but got nil") + return + } + + expectedErrMsg := "configuration conflict" + if !strings.Contains(err.Error(), expectedErrMsg) { + t.Errorf("expected error message to contain '%s', got: %v", expectedErrMsg, err) + } +} + +// Test Scenario 5: Default "external" subnet EXISTS, ConfigMap specifies "external" (same name) +// Expected: Use default "external" (no conflict) +func TestGetExternalGatewaySwitch_Scenario5_DefaultExists_ConfigMapSpecifiedSame(t *testing.T) { + defaultSubnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "external", + }, + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.ExternalGatewayConfig, + Namespace: "kube-system", + }, + Data: map[string]string{ + "enable-external-gw": "true", + "external-gw-switch": "external", + }, + } + + c := newTestController(t, []*kubeovnv1.Subnet{defaultSubnet}, []*corev1.ConfigMap{configMap}, true) + + result, err := c.getConfigDefaultExternalSwitch() + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + + expected := "external" + if result != expected { + t.Errorf("expected %s, got %s", expected, result) + } +} + +// Test ConfigMap disabled: enable-external-gw = false +// Expected: Return default regardless of ConfigMap content +func TestGetExternalGatewaySwitch_ConfigMapDisabled(t *testing.T) { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.ExternalGatewayConfig, + Namespace: "kube-system", + }, + Data: map[string]string{ + "enable-external-gw": "false", + "external-gw-switch": "custom-ext", // Should be ignored + }, + } + + c := newTestController(t, nil, []*corev1.ConfigMap{configMap}, false) + + result, err := c.getConfigDefaultExternalSwitch() + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + + expected := "external" + if result != expected { + t.Errorf("expected %s (should ignore ConfigMap when disabled), got %s", expected, result) + } +} + +// Test getExternalGatewaySwitchWithConfigMap directly +func TestGetExternalGatewaySwitchWithConfigMap_AllScenarios(t *testing.T) { + tests := []struct { + name string + defaultExists bool + configSwitch string + expectedResult string + expectError bool + errorMsgContains string + }{ + { + name: "Scenario 1: default not exist, config not specified", + defaultExists: false, + configSwitch: "", + expectedResult: "external", + expectError: false, + }, + { + name: "Scenario 2: default exists, config not specified", + defaultExists: true, + configSwitch: "", + expectedResult: "external", + expectError: false, + }, + { + name: "Scenario 3: default not exist, config specified", + defaultExists: false, + configSwitch: "custom-ext", + expectedResult: "custom-ext", + expectError: false, + }, + { + name: "Scenario 4: default exists, config specified different", + defaultExists: true, + configSwitch: "custom-ext", + expectedResult: "", + expectError: true, + errorMsgContains: "configuration conflict", + }, + { + name: "Scenario 5: default exists, config specified same", + defaultExists: true, + configSwitch: "external", + expectedResult: "external", + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var subnets []*kubeovnv1.Subnet + if tt.configSwitch != "" && tt.configSwitch != "external" { + // Add the custom subnet referenced by ConfigMap + subnets = append(subnets, &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: tt.configSwitch, + }, + }) + } + + c := newTestController(t, subnets, nil, tt.defaultExists) + + configData := map[string]string{} + if tt.configSwitch != "" { + configData["external-gw-switch"] = tt.configSwitch + } + + result, err := c.getExternalGatewaySwitchWithConfigMap(configData) + + if tt.expectError { + if err == nil { + t.Error("expected error but got nil") + return + } + if tt.errorMsgContains != "" && !strings.Contains(err.Error(), tt.errorMsgContains) { + t.Errorf("expected error containing '%s', got: %v", tt.errorMsgContains, err) + } + } else { + if err != nil { + t.Errorf("expected no error, got: %v", err) + } + if result != tt.expectedResult { + t.Errorf("expected result %s, got %s", tt.expectedResult, result) + } + } + }) + } +} + +// Test with actual Kubernetes client operations +func TestGetExternalGatewaySwitch_WithClientOperations(t *testing.T) { + kubeClient := fake.NewSimpleClientset() + kubeOvnClient := kubeovnfake.NewSimpleClientset() + + kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, 0) + kubeovnInformerFactory := kubeovninformers.NewSharedInformerFactory(kubeOvnClient, 0) + + config := &Configuration{ + KubeClient: kubeClient, + KubeOvnClient: kubeOvnClient, + ExternalGatewaySwitch: "external", + ExternalGatewayConfigNS: "kube-system", + } + + // Create mock OVN client - default logical switch exists (conflict case) + mockOvnClient := mockovs.NewMockNbClient(gomock.NewController(t)) + mockOvnClient.EXPECT().LogicalSwitchExists("external").Return(true, nil).AnyTimes() + + controller := &Controller{ + config: config, + configMapsLister: kubeInformerFactory.Core().V1().ConfigMaps().Lister(), + subnetsLister: kubeovnInformerFactory.Kubeovn().V1().Subnets().Lister(), + OVNNbClient: mockOvnClient, + } + + stopCh := make(chan struct{}) + defer close(stopCh) + + kubeInformerFactory.Start(stopCh) + kubeovnInformerFactory.Start(stopCh) + + // Create subnet after informer starts + subnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "external", + }, + } + _, err := kubeOvnClient.KubeovnV1().Subnets().Create(context.Background(), subnet, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create subnet: %v", err) + } + + // Create ConfigMap + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.ExternalGatewayConfig, + Namespace: "kube-system", + }, + Data: map[string]string{ + "enable-external-gw": "true", + "external-gw-switch": "custom-ext", + }, + } + _, err = kubeClient.CoreV1().ConfigMaps("kube-system").Create(context.Background(), configMap, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create configmap: %v", err) + } + + // Wait for informer cache sync + time.Sleep(200 * time.Millisecond) + + // This should return error due to conflict + _, err = controller.getConfigDefaultExternalSwitch() + if err == nil { + t.Error("expected configuration conflict error, got nil") + } else if !strings.Contains(err.Error(), "configuration conflict") { + t.Errorf("expected configuration conflict error, got: %v", err) + } +} diff --git a/pkg/controller/external_vpc.go b/pkg/controller/external_vpc.go index 877d3cf08bb..6992016bc92 100644 --- a/pkg/controller/external_vpc.go +++ b/pkg/controller/external_vpc.go @@ -27,24 +27,31 @@ func (c *Controller) syncExternalVpc() { return } for _, cachedVpc := range vpcs { - vpc := cachedVpc.DeepCopy() - if _, ok := logicalRouters[vpc.Name]; ok { + vpcName := cachedVpc.Name + if _, ok := logicalRouters[vpcName]; ok { + // Get the latest VPC object before updating status to avoid conflicts + vpc, err := c.config.KubeOvnClient.KubeovnV1().Vpcs().Get(context.Background(), vpcName, metav1.GetOptions{}) + if err != nil { + klog.Errorf("failed to get latest vpc %s: %v", vpcName, err) + continue + } + // update vpc status subnet list vpc.Status.Subnets = []string{} - for _, ls := range logicalRouters[vpc.Name].LogicalSwitches { + for _, ls := range logicalRouters[vpcName].LogicalSwitches { vpc.Status.Subnets = append(vpc.Status.Subnets, ls.Name) } _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().UpdateStatus(context.Background(), vpc, metav1.UpdateOptions{}) if err != nil { - klog.Errorf("failed to update vpc %s status: %v", vpc.Name, err) + klog.Errorf("failed to update vpc %s status: %v", vpcName, err) continue } - delete(logicalRouters, vpc.Name) + delete(logicalRouters, vpcName) } else { - klog.Infof("external vpc %s has no ovn logical router, delete it", vpc.Name) - err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Delete(context.Background(), vpc.Name, metav1.DeleteOptions{}) + klog.Infof("external vpc %s has no ovn logical router, delete it", vpcName) + err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Delete(context.Background(), vpcName, metav1.DeleteOptions{}) if err != nil { - klog.Errorf("failed to delete vpc %s: %v", vpc.Name, err) + klog.Errorf("failed to delete vpc %s: %v", vpcName, err) continue } } diff --git a/pkg/controller/ip.go b/pkg/controller/ip.go index f475cac705b..329c88f4d00 100644 --- a/pkg/controller/ip.go +++ b/pkg/controller/ip.go @@ -8,8 +8,8 @@ import ( "maps" "net" "reflect" - "slices" "strings" + "time" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,12 +30,6 @@ func (c *Controller) enqueueAddIP(obj any) { if strings.HasPrefix(ipObj.Name, util.U2OInterconnName[0:19]) { return } - klog.V(3).Infof("enqueue update status subnet %s", ipObj.Spec.Subnet) - c.updateSubnetStatusQueue.Add(ipObj.Spec.Subnet) - for _, as := range ipObj.Spec.AttachSubnets { - klog.V(3).Infof("enqueue update attach status for subnet %s", as) - c.updateSubnetStatusQueue.Add(as) - } key := cache.MetaObjectToName(ipObj).String() klog.V(3).Infof("enqueue add ip %s", key) @@ -88,13 +82,6 @@ func (c *Controller) enqueueUpdateIP(oldObj, newObj any) { c.updateIPQueue.Add(key) return } - if !slices.Equal(oldIP.Spec.AttachSubnets, newIP.Spec.AttachSubnets) { - klog.V(3).Infof("enqueue update status subnet %s", newIP.Spec.Subnet) - for _, as := range newIP.Spec.AttachSubnets { - klog.V(3).Infof("enqueue update status for attach subnet %s", as) - c.updateSubnetStatusQueue.Add(as) - } - } } func (c *Controller) enqueueDelIP(obj any) { @@ -120,7 +107,7 @@ func (c *Controller) enqueueDelIP(obj any) { key := cache.MetaObjectToName(ipObj).String() klog.V(3).Infof("enqueue del ip %s", key) - c.delIPQueue.Add(ipObj) + c.delIPQueue.Add(ipObj.DeepCopy()) } func (c *Controller) handleAddReservedIP(key string) error { @@ -176,8 +163,8 @@ func (c *Controller) handleAddReservedIP(key string) error { return err } if lsp != nil { - // port already exists means the ip already created - klog.V(3).Infof("ip %s is ready", portName) + // port already exists means the ip already created, finalizer already added above + klog.V(3).Infof("ip %s is ready, finalizer already added", portName) return nil } @@ -220,6 +207,10 @@ func (c *Controller) handleAddReservedIP(key string) error { return err } } + + // Trigger subnet status update after all operations complete + // At this point: IPAM allocated, IP CR created with labels+finalizer + c.updateSubnetStatusQueue.Add(subnet.Name) return nil } @@ -232,6 +223,8 @@ func (c *Controller) handleUpdateIP(key string) error { klog.Error(err) return err } + + // Handle deletion first if !cachedIP.DeletionTimestamp.IsZero() { klog.Infof("handle deleting ip %s", cachedIP.Name) subnet, err := c.subnetsLister.Get(cachedIP.Spec.Subnet) @@ -272,18 +265,32 @@ func (c *Controller) handleUpdateIP(key string) error { klog.Errorf("failed to handle del ip finalizer %v", err) return err } - c.updateSubnetStatusQueue.Add(cachedIP.Spec.Subnet) + return nil } + + // Non-deletion case: ensure finalizer is added + if err = c.handleAddOrUpdateIPFinalizer(cachedIP); err != nil { + klog.Errorf("failed to handle add or update finalizer for ip %s: %v", key, err) + return err + } + return nil } func (c *Controller) handleDelIP(ip *kubeovnv1.IP) error { - klog.Infof("deleting ip %s enqueue update status subnet %s", ip.Name, ip.Spec.Subnet) - c.updateSubnetStatusQueue.Add(ip.Spec.Subnet) + klog.Infof("deleting ip %s", ip.Name) + + // For IP CRs deleted without finalizer (race condition or direct deletion), + // we need to ensure subnet status is updated. + // Note: IPAM release should have been done before this (either in handleUpdateIP + // or in pod controller), but we trigger subnet status update here as a safety net. + if ip.Spec.Subnet != "" { + c.updateSubnetStatusQueue.Add(ip.Spec.Subnet) + } for _, as := range ip.Spec.AttachSubnets { - klog.V(3).Infof("enqueue update attach status for subnet %s", as) c.updateSubnetStatusQueue.Add(as) } + return nil } @@ -298,6 +305,39 @@ func (c *Controller) syncIPFinalizer(cl client.Client) error { }) } +func (c *Controller) handleAddOrUpdateIPFinalizer(cachedIP *kubeovnv1.IP) error { + if !cachedIP.DeletionTimestamp.IsZero() { + // IP is being deleted, don't handle finalizer add/update + return nil + } + + newIP := cachedIP.DeepCopy() + controllerutil.RemoveFinalizer(newIP, util.DepreciatedFinalizerName) + controllerutil.AddFinalizer(newIP, util.KubeOVNControllerFinalizer) + patch, err := util.GenerateMergePatchPayload(cachedIP, newIP) + if err != nil { + klog.Errorf("failed to generate patch payload for ip '%s', %v", cachedIP.Name, err) + return err + } + if _, err := c.config.KubeOvnClient.KubeovnV1().IPs().Patch(context.Background(), cachedIP.Name, + types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + klog.Errorf("failed to add finalizer for ip '%s', %v", cachedIP.Name, err) + return err + } + + // Trigger subnet status update after finalizer is processed as a fallback + // This handles cases where finalizer was not added during creation + // AddFinalizer is idempotent, so this is safe even if finalizer already exists + c.updateSubnetStatusQueue.Add(cachedIP.Spec.Subnet) + for _, as := range cachedIP.Spec.AttachSubnets { + c.updateSubnetStatusQueue.Add(as) + } + return nil +} + func (c *Controller) handleDelIPFinalizer(cachedIP *kubeovnv1.IP) error { if len(cachedIP.GetFinalizers()) == 0 { return nil @@ -318,6 +358,15 @@ func (c *Controller) handleDelIPFinalizer(cachedIP *kubeovnv1.IP) error { klog.Errorf("failed to remove finalizer from ip %s, %v", cachedIP.Name, err) return err } + + // Trigger subnet status update after finalizer is removed + // This ensures subnet status reflects the IP release + // Add delay to ensure API server completes the finalizer removal + time.Sleep(300 * time.Millisecond) + c.updateSubnetStatusQueue.Add(cachedIP.Spec.Subnet) + for _, as := range cachedIP.Spec.AttachSubnets { + c.updateSubnetStatusQueue.Add(as) + } return nil } @@ -411,16 +460,17 @@ func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, } v4IP, v6IP := util.SplitStringIP(ip) if ipCR == nil { + // Create CR with finalizer and labels all at once ipCR = &kubeovnv1.IP{ ObjectMeta: metav1.ObjectMeta{ - Name: ipName, + Name: ipName, + Finalizers: []string{util.KubeOVNControllerFinalizer}, Labels: map[string]string{ util.SubnetNameLabel: subnetName, util.NodeNameLabel: nodeName, subnetName: "", util.IPReservedLabel: "false", // ip create with pod or node, ip not reserved }, - Finalizers: []string{util.KubeOVNControllerFinalizer}, }, Spec: kubeovnv1.IPSpec{ PodName: key, @@ -437,7 +487,7 @@ func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, PodType: podType, }, } - if _, err = c.config.KubeOvnClient.KubeovnV1().IPs().Create(context.Background(), ipCR, metav1.CreateOptions{}); err != nil { + if ipCR, err = c.config.KubeOvnClient.KubeovnV1().IPs().Create(context.Background(), ipCR, metav1.CreateOptions{}); err != nil { errMsg := fmt.Errorf("failed to create ip CR %s: %w", ipName, err) klog.Error(errMsg) return errMsg @@ -469,7 +519,6 @@ func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, if maps.Equal(newIPCR.Labels, ipCR.Labels) && reflect.DeepEqual(newIPCR.Spec, ipCR.Spec) { return nil } - controllerutil.AddFinalizer(newIPCR, util.KubeOVNControllerFinalizer) if _, err = c.config.KubeOvnClient.KubeovnV1().IPs().Update(context.Background(), newIPCR, metav1.UpdateOptions{}); err != nil { err := fmt.Errorf("failed to update ip CR %s: %w", ipCRName, err) @@ -477,6 +526,12 @@ func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, return err } } + // Trigger subnet status update after CR creation with finalizer + time.Sleep(300 * time.Millisecond) + c.updateSubnetStatusQueue.Add(ipCR.Spec.Subnet) + for _, as := range ipCR.Spec.AttachSubnets { + c.updateSubnetStatusQueue.Add(as) + } return nil } diff --git a/pkg/controller/ippool.go b/pkg/controller/ippool.go index d91c028b7ae..829204b34b0 100644 --- a/pkg/controller/ippool.go +++ b/pkg/controller/ippool.go @@ -44,7 +44,7 @@ func (c *Controller) enqueueDeleteIPPool(obj any) { } klog.V(3).Infof("enqueue delete ippool %s", cache.MetaObjectToName(ippool).String()) - c.deleteIPPoolQueue.Add(ippool) + c.deleteIPPoolQueue.Add(ippool.DeepCopy()) } func (c *Controller) enqueueUpdateIPPool(oldObj, newObj any) { @@ -215,6 +215,7 @@ func (c *Controller) handleAddIPPoolFinalizer(ippool *kubeovnv1.IPPool) error { } newIPPool := ippool.DeepCopy() + controllerutil.RemoveFinalizer(newIPPool, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newIPPool, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(ippool, newIPPool) if err != nil { diff --git a/pkg/controller/ovn_dnat.go b/pkg/controller/ovn_dnat.go index 1647d98bc48..14bf14fb804 100644 --- a/pkg/controller/ovn_dnat.go +++ b/pkg/controller/ovn_dnat.go @@ -34,8 +34,9 @@ func (c *Controller) enqueueUpdateOvnDnatRule(oldObj, newObj any) { // avoid delete twice return } - klog.Infof("enqueue del ovn dnat %s", key) - c.delOvnDnatRuleQueue.Add(key) + // DNAT with finalizer should be handled in updateOvnDnatRuleQueue + klog.Infof("enqueue update (deleting) ovn dnat %s", key) + c.updateOvnDnatRuleQueue.Add(key) return } oldDnat := oldObj.(*kubeovnv1.OvnDnatRule) @@ -297,6 +298,48 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error { klog.Error(err) return err } + + // Handle deletion first (for DNATs with finalizers) + if !cachedDnat.DeletionTimestamp.IsZero() { + klog.Infof("handle deleting ovn dnat %s", key) + if cachedDnat.Status.Vpc == "" { + // Already cleaned, just remove finalizer + if err = c.handleDelOvnDnatFinalizer(cachedDnat); err != nil { + klog.Errorf("failed to remove finalizer for ovn dnat %s, %v", cachedDnat.Name, err) + return err + } + return nil + } + + // ovn delete dnat + if cachedDnat.Status.V4Eip != "" && cachedDnat.Status.ExternalPort != "" { + if err = c.DelDnatRule(cachedDnat.Status.Vpc, cachedDnat.Name, + cachedDnat.Status.V4Eip, cachedDnat.Status.ExternalPort); err != nil { + klog.Errorf("failed to delete v4 dnat %s, %v", key, err) + return err + } + } + if cachedDnat.Status.V6Eip != "" && cachedDnat.Status.ExternalPort != "" { + if err = c.DelDnatRule(cachedDnat.Status.Vpc, cachedDnat.Name, + cachedDnat.Status.V6Eip, cachedDnat.Status.ExternalPort); err != nil { + klog.Errorf("failed to delete v6 dnat %s, %v", key, err) + return err + } + } + + // Remove finalizer + if err = c.handleDelOvnDnatFinalizer(cachedDnat); err != nil { + klog.Errorf("failed to remove finalizer for ovn dnat %s, %v", cachedDnat.Name, err) + return err + } + + // Reset eip + if cachedDnat.Spec.OvnEip != "" { + c.resetOvnEipQueue.Add(cachedDnat.Spec.OvnEip) + } + return nil + } + if !cachedDnat.Status.Ready { // create dnat only in add process, just check to error out here klog.Infof("wait ovn dnat %s to be ready only in the handle add process", cachedDnat.Name) @@ -617,6 +660,7 @@ func (c *Controller) handleAddOvnDnatFinalizer(cachedDnat *kubeovnv1.OvnDnatRule err error ) + controllerutil.RemoveFinalizer(newDnat, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newDnat, util.KubeOVNControllerFinalizer) if patch, err = util.GenerateMergePatchPayload(cachedDnat, newDnat); err != nil { klog.Errorf("failed to generate patch payload for ovn dnat '%s', %v", cachedDnat.Name, err) @@ -655,5 +699,12 @@ func (c *Controller) handleDelOvnDnatFinalizer(cachedDnat *kubeovnv1.OvnDnatRule klog.Errorf("failed to remove finalizer from ovn dnat '%s', %v", cachedDnat.Name, err) return err } + + // Trigger associated EIP to recheck if it can be deleted now + if cachedDnat.Spec.OvnEip != "" { + klog.Infof("triggering eip %s update after dnat %s deletion", cachedDnat.Spec.OvnEip, cachedDnat.Name) + c.updateOvnEipQueue.Add(cachedDnat.Spec.OvnEip) + } + return nil } diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index 388ba3d5b77..1e23ca1def0 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -35,8 +35,9 @@ func (c *Controller) enqueueUpdateOvnEip(oldObj, newObj any) { // avoid delete eip twice return } - klog.Infof("enqueue del ovn eip %s", key) - c.delOvnEipQueue.Add(key) + // EIP with finalizer should be handled in updateOvnEipQueue + klog.Infof("enqueue update (deleting) ovn eip %s", key) + c.updateOvnEipQueue.Add(key) return } oldEip := oldObj.(*kubeovnv1.OvnEip) @@ -72,7 +73,7 @@ func (c *Controller) enqueueDelOvnEip(obj any) { key := cache.MetaObjectToName(eip).String() klog.Infof("enqueue del ovn eip %s", key) - c.delOvnEipQueue.Add(key) + c.delOvnEipQueue.Add(eip.DeepCopy()) } func (c *Controller) handleAddOvnEip(key string) error { @@ -89,6 +90,7 @@ func (c *Controller) handleAddOvnEip(key string) error { return nil } klog.Infof("handle add ovn eip %s", cachedEip.Name) + var v4ip, v6ip, mac, subnetName string subnetName = cachedEip.Spec.ExternalSubnet if subnetName == "" { @@ -143,11 +145,10 @@ func (c *Controller) handleAddOvnEip(key string) error { return err } } - if err = c.handleAddOvnEipFinalizer(cachedEip); err != nil { - klog.Errorf("failed to add finalizer for ovn eip, %v", err) - return err - } - c.updateSubnetStatusQueue.Add(subnetName) + + // Trigger subnet status update after all operations complete + // At this point: IPAM allocated, OvnEip CR created with labels+status+finalizer + c.updateSubnetStatusQueue.Add(subnet.Name) return nil } @@ -160,6 +161,55 @@ func (c *Controller) handleUpdateOvnEip(key string) error { klog.Error(err) return err } + + // Handle deletion first + if !cachedEip.DeletionTimestamp.IsZero() { + klog.Infof("handle deleting ovn eip %s", key) + + // Check if EIP is still being used by any NAT rules (FIP/DNAT/SNAT) BEFORE cleanup + // Only proceed with cleanup and finalizer removal when no NAT rules are using it + nat, err := c.getOvnEipNat(cachedEip.Spec.V4Ip) + if err != nil { + klog.Errorf("failed to get ovn eip %s nat rules, %v", key, err) + return err + } + if nat != "" { + err := fmt.Errorf("ovn eip %s is still being used by NAT rules: %s, waiting for them to be deleted", key, nat) + klog.Error(err) + return err + } + + // Clean up resources before removing finalizer + if cachedEip.Spec.Type == util.OvnEipTypeLSP { + if err := c.OVNNbClient.DeleteLogicalSwitchPort(cachedEip.Name); err != nil { + klog.Errorf("failed to delete lsp %s, %v", cachedEip.Name, err) + return err + } + } + if cachedEip.Spec.Type == util.OvnEipTypeLRP { + if err := c.OVNNbClient.DeleteLogicalRouterPort(cachedEip.Name); err != nil { + klog.Errorf("failed to delete lrp %s, %v", cachedEip.Name, err) + return err + } + } + + // Release IP from IPAM before removing finalizer + c.ipam.ReleaseAddressByPod(cachedEip.Name, cachedEip.Spec.ExternalSubnet) + + // Now remove finalizer, which will trigger subnet status update + if err = c.handleDelOvnEipFinalizer(cachedEip); err != nil { + klog.Errorf("failed to handle remove ovn eip finalizer , %v", err) + return err + } + return nil + } + + // Always ensure finalizer is added regardless of Status + if err = c.handleAddOrUpdateOvnEipFinalizer(cachedEip); err != nil { + klog.Errorf("failed to handle add or update finalizer for ovn eip %s: %v", key, err) + return err + } + if !cachedEip.Status.Ready { // create eip only in add process, just check to error out here klog.Infof("wait ovn eip %s to be ready only in the handle add process", cachedEip.Name) @@ -216,17 +266,12 @@ func (c *Controller) handleResetOvnEip(key string) error { return nil } -func (c *Controller) handleDelOvnEip(key string) error { - klog.Infof("handle del ovn eip %s", key) - eip, err := c.ovnEipsLister.Get(key) - if err != nil { - if k8serrors.IsNotFound(err) { - return nil - } - klog.Error(err) - return err - } +func (c *Controller) handleDelOvnEip(eip *kubeovnv1.OvnEip) error { + // This handles deletion of EIPs without finalizers (race condition or direct deletion) + // EIPs with finalizers are handled in handleUpdateOvnEip + klog.Infof("handle del ovn eip %s (without finalizer)", eip.Name) + // Clean up resources if they still exist if eip.Spec.Type == util.OvnEipTypeLSP { if err := c.OVNNbClient.DeleteLogicalSwitchPort(eip.Name); err != nil { klog.Errorf("failed to delete lsp %s, %v", eip.Name, err) @@ -240,12 +285,14 @@ func (c *Controller) handleDelOvnEip(key string) error { } } - if err = c.handleDelOvnEipFinalizer(eip); err != nil { - klog.Errorf("failed to handle remove ovn eip finalizer , %v", err) - return err - } + // Release IP from IPAM c.ipam.ReleaseAddressByPod(eip.Name, eip.Spec.ExternalSubnet) - c.updateSubnetStatusQueue.Add(eip.Spec.ExternalSubnet) + + // Ensure subnet status is updated + if eip.Spec.ExternalSubnet != "" { + c.updateSubnetStatusQueue.Add(eip.Spec.ExternalSubnet) + } + return nil } @@ -253,9 +300,11 @@ func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usageT cachedEip, err := c.ovnEipsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { + // Create CR with finalizer, labels and status all at once _, err := c.config.KubeOvnClient.KubeovnV1().OvnEips().Create(context.Background(), &kubeovnv1.OvnEip{ ObjectMeta: metav1.ObjectMeta{ - Name: key, + Name: key, + Finalizers: []string{util.KubeOVNControllerFinalizer}, Labels: map[string]string{ util.SubnetNameLabel: subnet, util.OvnEipTypeLabel: usageType, @@ -270,6 +319,14 @@ func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usageT MacAddress: mac, Type: usageType, }, + Status: kubeovnv1.OvnEipStatus{ + V4Ip: v4ip, + V6Ip: v6ip, + MacAddress: mac, + Type: usageType, + Nat: "", + Ready: false, + }, }, metav1.CreateOptions{}) if err != nil { err := fmt.Errorf("failed to create crd ovn eip '%s', %w", key, err) @@ -284,8 +341,17 @@ func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usageT } } else { ovnEip := cachedEip.DeepCopy() - needUpdate := false + // Ensure labels are set correctly before any update + if ovnEip.Labels == nil { + ovnEip.Labels = make(map[string]string) + } + ovnEip.Labels[util.SubnetNameLabel] = subnet + ovnEip.Labels[util.OvnEipTypeLabel] = usageType + ovnEip.Labels[util.EipV4IpLabel] = v4ip + ovnEip.Labels[util.EipV6IpLabel] = v6ip + + needUpdate := false if mac != "" && ovnEip.Spec.MacAddress != mac { ovnEip.Spec.MacAddress = mac needUpdate = true @@ -303,6 +369,7 @@ func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usageT needUpdate = true } if needUpdate { + // Update with labels and spec in one call if _, err := c.config.KubeOvnClient.KubeovnV1().OvnEips().Update(context.Background(), ovnEip, metav1.UpdateOptions{}); err != nil { errMsg := fmt.Errorf("failed to update ovn eip '%s', %w", key, err) klog.Error(errMsg) @@ -341,42 +408,10 @@ func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usageT return err } } - - var needUpdateLabel bool - var op string - if len(ovnEip.Labels) == 0 { - op = "add" - ovnEip.Labels = map[string]string{ - util.SubnetNameLabel: subnet, - util.OvnEipTypeLabel: usageType, - util.EipV4IpLabel: v4ip, - util.EipV6IpLabel: v6ip, - } - needUpdateLabel = true - } - if ovnEip.Labels[util.SubnetNameLabel] != subnet { - op = "replace" - ovnEip.Labels[util.SubnetNameLabel] = subnet - ovnEip.Labels[util.EipV4IpLabel] = v4ip - ovnEip.Labels[util.EipV6IpLabel] = v6ip - needUpdateLabel = true - } - if ovnEip.Labels[util.OvnEipTypeLabel] != usageType { - op = "replace" - ovnEip.Labels[util.OvnEipTypeLabel] = usageType - needUpdateLabel = true - } - if needUpdateLabel { - patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/labels", "value": %s }]` - raw, _ := json.Marshal(ovnEip.Labels) - patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - if _, err := c.config.KubeOvnClient.KubeovnV1().OvnEips().Patch(context.Background(), ovnEip.Name, types.JSONPatchType, - []byte(patchPayload), metav1.PatchOptions{}); err != nil { - klog.Errorf("failed to patch label for ovn eip '%s', %v", ovnEip.Name, err) - return err - } - } } + // Trigger subnet status update after CR creation or update + time.Sleep(300 * time.Millisecond) + c.updateSubnetStatusQueue.Add(subnet) return nil } @@ -504,11 +539,12 @@ func (c *Controller) syncOvnEipFinalizer(cl client.Client) error { }) } -func (c *Controller) handleAddOvnEipFinalizer(cachedEip *kubeovnv1.OvnEip) error { - if !cachedEip.DeletionTimestamp.IsZero() || len(cachedEip.GetFinalizers()) != 0 { +func (c *Controller) handleAddOrUpdateOvnEipFinalizer(cachedEip *kubeovnv1.OvnEip) error { + if !cachedEip.DeletionTimestamp.IsZero() { return nil } newEip := cachedEip.DeepCopy() + controllerutil.RemoveFinalizer(newEip, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newEip, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedEip, newEip) if err != nil { @@ -523,6 +559,11 @@ func (c *Controller) handleAddOvnEipFinalizer(cachedEip *kubeovnv1.OvnEip) error klog.Errorf("failed to add finalizer for ovn eip '%s', %v", cachedEip.Name, err) return err } + + // Trigger subnet status update after finalizer is processed as a fallback + // This handles cases where finalizer was not added during creation + // AddFinalizer is idempotent, so this is safe even if finalizer already exists + c.updateSubnetStatusQueue.Add(cachedEip.Spec.ExternalSubnet) return nil } @@ -558,6 +599,12 @@ func (c *Controller) handleDelOvnEipFinalizer(cachedEip *kubeovnv1.OvnEip) error klog.Errorf("failed to remove finalizer from ovn eip '%s', %v", cachedEip.Name, err) return err } + + // Trigger subnet status update after finalizer is removed + // This ensures subnet status reflects the IP release + // Add delay to ensure API server completes the finalizer removal + time.Sleep(300 * time.Millisecond) + c.updateSubnetStatusQueue.Add(cachedEip.Spec.ExternalSubnet) return nil } diff --git a/pkg/controller/ovn_fip.go b/pkg/controller/ovn_fip.go index 7c47cc2998e..724f3767993 100644 --- a/pkg/controller/ovn_fip.go +++ b/pkg/controller/ovn_fip.go @@ -35,8 +35,9 @@ func (c *Controller) enqueueUpdateOvnFip(oldObj, newObj any) { // avoid delete twice return } - klog.Infof("enqueue del ovn fip %s", key) - c.delOvnFipQueue.Add(key) + // FIP with finalizer should be handled in updateOvnFipQueue + klog.Infof("enqueue update (deleting) ovn fip %s", key) + c.updateOvnFipQueue.Add(key) return } oldFip := oldObj.(*kubeovnv1.OvnFip) @@ -267,6 +268,46 @@ func (c *Controller) handleUpdateOvnFip(key string) error { klog.Error(err) return err } + + // Handle deletion first (for FIPs with finalizers) + if !cachedFip.DeletionTimestamp.IsZero() { + klog.Infof("handle deleting ovn fip %s", key) + if cachedFip.Status.Vpc == "" { + // Already cleaned, just remove finalizer + if err = c.handleDelOvnFipFinalizer(cachedFip); err != nil { + klog.Errorf("failed to remove finalizer for ovn fip %s, %v", cachedFip.Name, err) + return err + } + return nil + } + + // ovn delete fip nat + if cachedFip.Status.V4Eip != "" && cachedFip.Status.V4Ip != "" { + if err = c.OVNNbClient.DeleteNat(cachedFip.Status.Vpc, ovnnb.NATTypeDNATAndSNAT, cachedFip.Status.V4Eip, cachedFip.Status.V4Ip); err != nil { + klog.Errorf("failed to delete v4 fip %s, %v", key, err) + return err + } + } + if cachedFip.Status.V6Eip != "" && cachedFip.Status.V6Ip != "" { + if err = c.OVNNbClient.DeleteNat(cachedFip.Status.Vpc, ovnnb.NATTypeDNATAndSNAT, cachedFip.Status.V6Eip, cachedFip.Status.V6Ip); err != nil { + klog.Errorf("failed to delete v6 fip %s, %v", key, err) + return err + } + } + + // Remove finalizer + if err = c.handleDelOvnFipFinalizer(cachedFip); err != nil { + klog.Errorf("failed to remove finalizer for ovn fip %s, %v", cachedFip.Name, err) + return err + } + + // Reset eip + if cachedFip.Spec.OvnEip != "" { + c.resetOvnEipQueue.Add(cachedFip.Spec.OvnEip) + } + return nil + } + if !cachedFip.Status.Ready { // create fip only in add process, just check to error out here klog.Infof("wait ovn fip %s to be ready only in the handle add process", cachedFip.Name) @@ -542,6 +583,7 @@ func (c *Controller) handleAddOvnFipFinalizer(cachedFip *kubeovnv1.OvnFip) error return nil } newFip := cachedFip.DeepCopy() + controllerutil.RemoveFinalizer(newFip, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newFip, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedFip, newFip) if err != nil { @@ -580,5 +622,12 @@ func (c *Controller) handleDelOvnFipFinalizer(cachedFip *kubeovnv1.OvnFip) error klog.Errorf("failed to remove finalizer from ovn fip '%s', %v", cachedFip.Name, err) return err } + + // Trigger associated EIP to recheck if it can be deleted now + if cachedFip.Spec.OvnEip != "" { + klog.Infof("triggering eip %s update after fip %s deletion", cachedFip.Spec.OvnEip, cachedFip.Name) + c.updateOvnEipQueue.Add(cachedFip.Spec.OvnEip) + } + return nil } diff --git a/pkg/controller/ovn_snat.go b/pkg/controller/ovn_snat.go index 11de6beed10..e2e5df5b1cb 100644 --- a/pkg/controller/ovn_snat.go +++ b/pkg/controller/ovn_snat.go @@ -33,8 +33,9 @@ func (c *Controller) enqueueUpdateOvnSnatRule(oldObj, newObj any) { // avoid delete twice return } - klog.Infof("enqueue del ovn snat %s", key) - c.delOvnSnatRuleQueue.Add(key) + // SNAT with finalizer should be handled in updateOvnSnatRuleQueue + klog.Infof("enqueue update (deleting) ovn snat %s", key) + c.updateOvnSnatRuleQueue.Add(key) return } oldSnat := oldObj.(*kubeovnv1.OvnSnatRule) @@ -197,6 +198,46 @@ func (c *Controller) handleUpdateOvnSnatRule(key string) error { klog.Error(err) return err } + + // Handle deletion first (for SNATs with finalizers) + if !cachedSnat.DeletionTimestamp.IsZero() { + klog.Infof("handle deleting ovn snat %s", key) + if cachedSnat.Status.Vpc == "" { + // Already cleaned, just remove finalizer + if err = c.handleDelOvnSnatFinalizer(cachedSnat); err != nil { + klog.Errorf("failed to remove finalizer for ovn snat %s, %v", cachedSnat.Name, err) + return err + } + return nil + } + + // ovn delete snat + if cachedSnat.Status.V4Eip != "" && cachedSnat.Status.V4IpCidr != "" { + if err = c.OVNNbClient.DeleteNat(cachedSnat.Status.Vpc, ovnnb.NATTypeSNAT, cachedSnat.Status.V4Eip, cachedSnat.Status.V4IpCidr); err != nil { + klog.Errorf("failed to delete v4 snat %s, %v", key, err) + return err + } + } + if cachedSnat.Status.V6Eip != "" && cachedSnat.Status.V6IpCidr != "" { + if err = c.OVNNbClient.DeleteNat(cachedSnat.Status.Vpc, ovnnb.NATTypeSNAT, cachedSnat.Status.V6Eip, cachedSnat.Status.V6IpCidr); err != nil { + klog.Errorf("failed to delete v6 snat %s, %v", key, err) + return err + } + } + + // Remove finalizer + if err = c.handleDelOvnSnatFinalizer(cachedSnat); err != nil { + klog.Errorf("failed to remove finalizer for ovn snat %s, %v", cachedSnat.Name, err) + return err + } + + // Reset eip + if cachedSnat.Spec.OvnEip != "" { + c.resetOvnEipQueue.Add(cachedSnat.Spec.OvnEip) + } + return nil + } + if !cachedSnat.Status.Ready { klog.Infof("wait ovn snat %s to be ready only in the handle add process", cachedSnat.Name) return nil @@ -458,6 +499,7 @@ func (c *Controller) handleAddOvnSnatFinalizer(cachedSnat *kubeovnv1.OvnSnatRule return nil } newSnat := cachedSnat.DeepCopy() + controllerutil.RemoveFinalizer(newSnat, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newSnat, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedSnat, newSnat) if err != nil { @@ -495,5 +537,12 @@ func (c *Controller) handleDelOvnSnatFinalizer(cachedSnat *kubeovnv1.OvnSnatRule klog.Errorf("failed to remove finalizer from ovn snat '%s', %v", cachedSnat.Name, err) return err } + + // Trigger associated EIP to recheck if it can be deleted now + if cachedSnat.Spec.OvnEip != "" { + klog.Infof("triggering eip %s update after snat %s deletion", cachedSnat.Spec.OvnEip, cachedSnat.Name) + c.updateOvnEipQueue.Add(cachedSnat.Spec.OvnEip) + } + return nil } diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 87ed37c1709..4e4452bc1fe 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -1204,6 +1204,9 @@ func (c *Controller) handleDeletePod(key string) (err error) { } // release ipam address after delete ip CR c.ipam.ReleaseAddressByNic(podKey, portName, podNet.Subnet.Name) + // Trigger subnet status update after IPAM release + // This is needed when IP CR is deleted without finalizer (race condition) + c.updateSubnetStatusQueue.Add(podNet.Subnet.Name) } } if pod.Annotations[util.VipAnnotation] != "" { diff --git a/pkg/controller/qos_policy.go b/pkg/controller/qos_policy.go index 3b0f890c302..a2dc1912e82 100644 --- a/pkg/controller/qos_policy.go +++ b/pkg/controller/qos_policy.go @@ -164,6 +164,7 @@ func (c *Controller) handleDelQoSPoliciesFinalizer(key string) error { return nil } newQoSPolicies := cachedQoSPolicies.DeepCopy() + controllerutil.RemoveFinalizer(newQoSPolicies, util.DepreciatedFinalizerName) controllerutil.RemoveFinalizer(newQoSPolicies, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedQoSPolicies, newQoSPolicies) if err != nil { @@ -435,6 +436,7 @@ func (c *Controller) handleAddQoSPolicyFinalizer(key string) error { return nil } newQoSPolicy := cachedQoSPolicy.DeepCopy() + controllerutil.RemoveFinalizer(newQoSPolicy, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newQoSPolicy, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedQoSPolicy, newQoSPolicy) if err != nil { diff --git a/pkg/controller/service.go b/pkg/controller/service.go index b803df5bff8..948e16f89fd 100644 --- a/pkg/controller/service.go +++ b/pkg/controller/service.go @@ -107,7 +107,7 @@ func (c *Controller) enqueueDeleteService(obj any) { vpcSvc := &vpcService{ Protocol: port.Protocol, Vpc: svc.Annotations[util.VpcAnnotation], - Svc: svc, + Svc: svc.DeepCopy(), } for _, ip := range ips { vpcSvc.Vips = append(vpcSvc.Vips, util.JoinHostPort(ip, port.Port)) diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index d6f17d8c10c..e6a7d64fa2c 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -53,7 +53,7 @@ func (c *Controller) enqueueDeleteSubnet(obj any) { } klog.V(3).Infof("enqueue delete subnet %s", subnet.Name) - c.deleteSubnetQueue.Add(subnet) + c.deleteSubnetQueue.Add(subnet.DeepCopy()) } func readyToRemoveFinalizer(subnet *kubeovnv1.Subnet) bool { @@ -275,6 +275,7 @@ func (c *Controller) syncSubnetFinalizer(cl client.Client) error { func (c *Controller) handleSubnetFinalizer(subnet *kubeovnv1.Subnet) (*kubeovnv1.Subnet, bool, error) { if subnet.DeletionTimestamp.IsZero() && !slices.Contains(subnet.GetFinalizers(), util.KubeOVNControllerFinalizer) { newSubnet := subnet.DeepCopy() + controllerutil.RemoveFinalizer(newSubnet, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newSubnet, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(subnet, newSubnet) if err != nil { @@ -298,6 +299,7 @@ func (c *Controller) handleSubnetFinalizer(subnet *kubeovnv1.Subnet) (*kubeovnv1 u2oInterconnIP := subnet.Status.U2OInterconnectionIP if !subnet.DeletionTimestamp.IsZero() && (usingIPs == 0 || (usingIPs == 1 && u2oInterconnIP != "")) { newSubnet := subnet.DeepCopy() + controllerutil.RemoveFinalizer(newSubnet, util.DepreciatedFinalizerName) controllerutil.RemoveFinalizer(newSubnet, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(subnet, newSubnet) if err != nil { diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index 722114975d1..901d830dac1 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -7,6 +7,7 @@ import ( "fmt" "slices" "strings" + "time" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -63,7 +64,7 @@ func (c *Controller) enqueueDelVirtualIP(obj any) { key := cache.MetaObjectToName(vip).String() klog.Infof("enqueue del vip %s", key) - c.delVirtualIPQueue.Add(vip) + c.delVirtualIPQueue.Add(vip.DeepCopy()) } func (c *Controller) handleAddVirtualIP(key string) error { @@ -80,6 +81,7 @@ func (c *Controller) handleAddVirtualIP(key string) error { return nil } klog.V(3).Infof("handle add vip %s", key) + vip := cachedVip.DeepCopy() var sourceV4Ip, sourceV6Ip, v4ip, v6ip, mac, subnetName string subnetName = vip.Spec.Subnet @@ -167,6 +169,10 @@ func (c *Controller) handleAddVirtualIP(key string) error { klog.Error(err) return err } + + // Trigger subnet status update after all operations complete + // At this point: IPAM allocated, VIP CR created with labels+status+finalizer + c.updateSubnetStatusQueue.Add(subnetName) return nil } @@ -182,6 +188,31 @@ func (c *Controller) handleUpdateVirtualIP(key string) error { vip := cachedVip.DeepCopy() // should delete if !vip.DeletionTimestamp.IsZero() { + klog.Infof("handle deleting vip %s", vip.Name) + // Clean up resources before removing finalizer + if vip.Spec.Type != "" { + subnet, err := c.subnetsLister.Get(vip.Spec.Subnet) + if err != nil { + klog.Errorf("failed to get subnet %s: %v", vip.Spec.Subnet, err) + return err + } + portName := ovs.PodNameToPortName(vip.Name, vip.Spec.Namespace, subnet.Spec.Provider) + klog.Infof("delete vip lsp %s", portName) + if err := c.OVNNbClient.DeleteLogicalSwitchPort(portName); err != nil { + err = fmt.Errorf("failed to delete lsp %s: %w", vip.Name, err) + klog.Error(err) + return err + } + } + // delete virtual ports + if err := c.OVNNbClient.DeleteLogicalSwitchPort(vip.Name); err != nil { + klog.Errorf("delete virtual logical switch port %s from logical switch %s: %v", vip.Name, vip.Spec.Subnet, err) + return err + } + // Release IP from IPAM before removing finalizer + c.ipam.ReleaseAddressByPod(vip.Name, vip.Spec.Subnet) + + // Now remove finalizer, which will trigger subnet status update if err = c.handleDelVipFinalizer(key); err != nil { klog.Errorf("failed to handle vip finalizer %v", err) return err @@ -217,38 +248,26 @@ func (c *Controller) handleUpdateVirtualIP(key string) error { klog.Error(err) return err } - if err = c.handleAddVipFinalizer(key); err != nil { - klog.Errorf("failed to handle vip finalizer %v", err) - return err - } + } + // Always ensure finalizer is added regardless of Status + if err = c.handleAddOrUpdateVipFinalizer(key); err != nil { + klog.Errorf("failed to handle vip finalizer %v", err) + return err } return nil } func (c *Controller) handleDelVirtualIP(vip *kubeovnv1.Vip) error { - klog.Infof("handle delete vip %s", vip.Name) - // TODO:// clean vip in its parent port aap list - if vip.Spec.Type != "" { - subnet, err := c.subnetsLister.Get(vip.Spec.Subnet) - if err != nil { - klog.Errorf("failed to get subnet %s: %v", vip.Spec.Subnet, err) - return err - } - portName := ovs.PodNameToPortName(vip.Name, vip.Spec.Namespace, subnet.Spec.Provider) - klog.Infof("delete vip lsp %s", portName) - if err := c.OVNNbClient.DeleteLogicalSwitchPort(portName); err != nil { - err = fmt.Errorf("failed to delete lsp %s: %w", vip.Name, err) - klog.Error(err) - return err - } - } - // delete virtual ports - if err := c.OVNNbClient.DeleteLogicalSwitchPort(vip.Name); err != nil { - klog.Errorf("delete virtual logical switch port %s from logical switch %s: %v", vip.Name, vip.Spec.Subnet, err) - return err + // Cleanup is now handled in handleUpdateVirtualIP before finalizer removal + // This function is kept for compatibility with the delete queue + klog.V(3).Infof("vip %s cleanup already done in update handler", vip.Name) + + // For VIPs deleted without finalizer (race condition or direct deletion), + // we need to ensure subnet status is updated as a safety net. + if vip.Spec.Subnet != "" { + c.updateSubnetStatusQueue.Add(vip.Spec.Subnet) } - c.ipam.ReleaseAddressByPod(vip.Name, vip.Spec.Subnet) - c.updateSubnetStatusQueue.Add(vip.Spec.Subnet) + return nil } @@ -348,14 +367,16 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string vipCR, err := c.virtualIpsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { + // Create CR with finalizer, labels and status all at once if _, err := c.config.KubeOvnClient.KubeovnV1().Vips().Create(context.Background(), &kubeovnv1.Vip{ ObjectMeta: metav1.ObjectMeta{ - Name: key, + Name: key, + Namespace: ns, + Finalizers: []string{util.KubeOVNControllerFinalizer}, Labels: map[string]string{ util.SubnetNameLabel: subnet, util.IPReservedLabel: "", }, - Namespace: ns, }, Spec: kubeovnv1.VipSpec{ Namespace: ns, @@ -364,6 +385,11 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string V6ip: v6ip, MacAddress: mac, }, + Status: kubeovnv1.VipStatus{ + V4ip: v4ip, + V6ip: v6ip, + Mac: mac, + }, }, metav1.CreateOptions{}); err != nil { err := fmt.Errorf("failed to create crd vip '%s', %w", key, err) klog.Error(err) @@ -376,6 +402,14 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string } } else { vip := vipCR.DeepCopy() + + // Ensure labels are set correctly + if vip.Labels == nil { + vip.Labels = make(map[string]string) + } + vip.Labels[util.SubnetNameLabel] = subnet + vip.Labels[util.IPReservedLabel] = "" + if vip.Status.Mac == "" && mac != "" || vip.Status.V4ip == "" && v4ip != "" || vip.Status.V6ip == "" && v6ip != "" { @@ -391,39 +425,16 @@ func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac string vip.Status.Mac = mac vip.Status.Type = vip.Spec.Type // TODO:// Ready = true as subnet.Status.Ready + // Update with labels, spec and status in one call if _, err := c.config.KubeOvnClient.KubeovnV1().Vips().Update(context.Background(), vip, metav1.UpdateOptions{}); err != nil { err := fmt.Errorf("failed to update vip '%s', %w", key, err) klog.Error(err) return err } } - var needUpdateLabel bool - var op string - if len(vip.Labels) == 0 { - op = "add" - vip.Labels = map[string]string{ - util.SubnetNameLabel: subnet, - util.IPReservedLabel: "", - } - needUpdateLabel = true - } - if _, ok := vip.Labels[util.SubnetNameLabel]; !ok { - op = "add" - vip.Labels[util.SubnetNameLabel] = subnet - vip.Labels[util.IPReservedLabel] = "" - needUpdateLabel = true - } - if needUpdateLabel { - patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/labels", "value": %s }]` - raw, _ := json.Marshal(vip.Labels) - patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - if _, err := c.config.KubeOvnClient.KubeovnV1().Vips().Patch(context.Background(), vip.Name, types.JSONPatchType, - []byte(patchPayload), metav1.PatchOptions{}); err != nil { - klog.Errorf("failed to patch label for vip '%s', %v", vip.Name, err) - return err - } - } } + // Trigger subnet status update after CR creation or update + time.Sleep(300 * time.Millisecond) c.updateSubnetStatusQueue.Add(subnet) return nil } @@ -460,7 +471,6 @@ func (c *Controller) podReuseVip(vipName, portName string, keepVIP bool) error { return err } c.ipam.ReleaseAddressByPod(vipName, vip.Spec.Subnet) - c.updateSubnetStatusQueue.Add(vip.Spec.Subnet) return nil } @@ -500,12 +510,11 @@ func (c *Controller) releaseVip(key string) error { if _, _, _, err = c.ipam.GetStaticAddress(key, vip.Name, vip.Status.V4ip, mac, vip.Spec.Subnet, false); err != nil { klog.Errorf("failed to recover IPAM from vip CR %s: %v", vip.Name, err) } - c.updateSubnetStatusQueue.Add(vip.Spec.Subnet) } return nil } -func (c *Controller) handleAddVipFinalizer(key string) error { +func (c *Controller) handleAddOrUpdateVipFinalizer(key string) error { cachedVip, err := c.virtualIpsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { @@ -514,10 +523,11 @@ func (c *Controller) handleAddVipFinalizer(key string) error { klog.Error(err) return err } - if !cachedVip.DeletionTimestamp.IsZero() || len(cachedVip.GetFinalizers()) != 0 { + if !cachedVip.DeletionTimestamp.IsZero() { return nil } newVip := cachedVip.DeepCopy() + controllerutil.RemoveFinalizer(newVip, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newVip, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedVip, newVip) if err != nil { @@ -532,6 +542,11 @@ func (c *Controller) handleAddVipFinalizer(key string) error { klog.Errorf("failed to add finalizer for vip '%s', %v", cachedVip.Name, err) return err } + + // Trigger subnet status update after finalizer is processed as a fallback + // This handles cases where finalizer was not added during creation + // AddFinalizer is idempotent, so this is safe even if finalizer already exists + c.updateSubnetStatusQueue.Add(cachedVip.Spec.Subnet) return nil } @@ -548,6 +563,7 @@ func (c *Controller) handleDelVipFinalizer(key string) error { return nil } newVip := cachedVip.DeepCopy() + controllerutil.RemoveFinalizer(newVip, util.DepreciatedFinalizerName) controllerutil.RemoveFinalizer(newVip, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedVip, newVip) if err != nil { @@ -562,6 +578,12 @@ func (c *Controller) handleDelVipFinalizer(key string) error { klog.Errorf("failed to remove finalizer from vip '%s', %v", cachedVip.Name, err) return err } + + // Trigger subnet status update after finalizer is removed + // This ensures subnet status reflects the IP release + // Add delay to ensure API server completes the finalizer removal + time.Sleep(300 * time.Millisecond) + c.updateSubnetStatusQueue.Add(cachedVip.Spec.Subnet) return nil } diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index 6d712ee28ae..7b2667f4b2b 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -97,7 +97,7 @@ func (c *Controller) enqueueDelVpc(obj any) { if _, ok := vpc.Labels[util.VpcExternalLabel]; !vpc.Status.Default || !ok { klog.V(3).Infof("enqueue delete vpc %s", vpc.Name) - c.delVpcQueue.Add(vpc) + c.delVpcQueue.Add(vpc.DeepCopy()) } } @@ -129,14 +129,15 @@ func (c *Controller) handleDelVpc(vpc *kubeovnv1.Vpc) error { return err } - if err := c.handleDelVpcExternalSubnet(vpc.Name, c.config.ExternalGatewaySwitch); err != nil { - klog.Errorf("failed to delete external connection for vpc %s, error %v", vpc.Name, err) + // Delete connection to default external network + if err := c.deleteVpc2ExternalConnection(vpc.Name); err != nil { + klog.Errorf("failed to delete default external connection for vpc %s: %v", vpc.Name, err) return err } for _, subnet := range vpc.Status.ExtraExternalSubnets { klog.Infof("disconnect external network %s to vpc %s", subnet, vpc.Name) - if err := c.handleDelVpcExternalSubnet(vpc.Name, subnet); err != nil { + if err := c.handleDelVpcRes2ExternalSubnet(vpc.Name, subnet); err != nil { klog.Error(err) return err } @@ -349,20 +350,28 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { return err } + // Determine which external gateway switch to use + // Logic: default subnet exists -> use default; default not exists + ConfigMap specified -> use ConfigMap + externalGwSwitch, err := c.getConfigDefaultExternalSwitch() + if err != nil { + klog.Warningf("failed to get external gateway switch: %v", err) + externalGwSwitch = c.config.ExternalGatewaySwitch // fallback to default + } + var externalSubnet *kubeovnv1.Subnet externalSubnetExist := false externalSubnetGW := "" if c.config.EnableEipSnat { - externalSubnet, err = c.subnetsLister.Get(c.config.ExternalGatewaySwitch) + externalSubnet, err = c.subnetsLister.Get(externalGwSwitch) if err != nil { - klog.Warningf("enable-eip-snat need external subnet %s to be exist: %v", c.config.ExternalGatewaySwitch, err) + klog.Warningf("enable-eip-snat need external subnet %s to exist: %v", externalGwSwitch, err) } else { if !externalSubnet.Spec.LogicalGateway { // logical gw external subnet can not access external externalSubnetExist = true externalSubnetGW = externalSubnet.Spec.Gateway } else { - klog.Infof("default external subnet %s using logical gw", c.config.ExternalGatewaySwitch) + klog.Infof("external subnet %s using logical gw", externalGwSwitch) } } } @@ -416,7 +425,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { nextHop := cm.Data["external-gw-addr"] if nextHop == "" { if !externalSubnetExist { - err = fmt.Errorf("failed to get external subnet %s", c.config.ExternalGatewaySwitch) + err = fmt.Errorf("failed to get external subnet %s", externalGwSwitch) klog.Error(err) return err } @@ -602,7 +611,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { } if vpc.Spec.EnableExternal || vpc.Status.EnableExternal { - if err = c.handleUpdateVpcExternal(cachedVpc, custVpcEnableExternalEcmp, externalSubnetExist, externalSubnetGW); err != nil { + if err = c.handleUpdateVpcExternal(cachedVpc, externalGwSwitch, custVpcEnableExternalEcmp, externalSubnetExist, externalSubnetGW); err != nil { klog.Errorf("failed to handle update external subnet for vpc %s, %v", key, err) return err } @@ -613,17 +622,25 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { klog.Error(err) return err } + + // Get the latest VPC object before updating status to avoid conflicts + latestVpc, err := c.config.KubeOvnClient.KubeovnV1().Vpcs().Get(context.Background(), key, metav1.GetOptions{}) + if err != nil { + klog.Errorf("failed to get latest vpc %s: %v", key, err) + return err + } + if vpc.Spec.BFDPort == nil || !vpc.Spec.BFDPort.Enabled { - vpc.Status.BFDPort = kubeovnv1.BFDPortStatus{} + latestVpc.Status.BFDPort = kubeovnv1.BFDPortStatus{} } else { - vpc.Status.BFDPort = kubeovnv1.BFDPortStatus{ + latestVpc.Status.BFDPort = kubeovnv1.BFDPortStatus{ Name: bfdPortName, IP: vpc.Spec.BFDPort.IP, Nodes: bfdPortNodes, } } if _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs(). - UpdateStatus(context.Background(), vpc, metav1.UpdateOptions{}); err != nil { + UpdateStatus(context.Background(), latestVpc, metav1.UpdateOptions{}); err != nil { klog.Error(err) return err } @@ -631,7 +648,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { return nil } -func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, custVpcEnableExternalEcmp, defaultExternalSubnetExist bool, externalSubnetGW string) error { +func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, externalGwSwitch string, custVpcEnableExternalEcmp, defaultExternalSubnetExist bool, externalSubnetGW string) error { if c.config.EnableEipSnat && vpc.Name == util.DefaultVpc { klog.Infof("external_gw handle ovn default external gw %s", vpc.Name) return nil @@ -652,14 +669,14 @@ func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, custVpcEnableEx if !vpc.Spec.EnableExternal && vpc.Status.EnableExternal { // just del all external subnets connection - klog.Infof("disconnect default external subnet %s to vpc %s", c.config.ExternalGatewaySwitch, vpc.Name) - if err := c.handleDelVpcExternalSubnet(vpc.Name, c.config.ExternalGatewaySwitch); err != nil { - klog.Errorf("failed to delete external subnet %s connection for vpc %s, error %v", c.config.ExternalGatewaySwitch, vpc.Name, err) + klog.Infof("disconnect default external subnet %s to vpc %s", externalGwSwitch, vpc.Name) + if err := c.handleDelVpcRes2ExternalSubnet(vpc.Name, externalGwSwitch); err != nil { + klog.Errorf("failed to delete external subnet %s connection for vpc %s, error %v", externalGwSwitch, vpc.Name, err) return err } for _, subnet := range vpc.Status.ExtraExternalSubnets { klog.Infof("disconnect external subnet %s to vpc %s", subnet, vpc.Name) - if err := c.handleDelVpcExternalSubnet(vpc.Name, subnet); err != nil { + if err := c.handleDelVpcRes2ExternalSubnet(vpc.Name, subnet); err != nil { klog.Errorf("failed to delete external subnet %s connection for vpc %s, error %v", subnet, vpc.Name, err) return err } @@ -671,9 +688,9 @@ func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, custVpcEnableEx // just add external connection if vpc.Spec.ExtraExternalSubnets == nil && defaultExternalSubnetExist { // only connect default external subnet - klog.Infof("connect default external subnet %s with vpc %s", c.config.ExternalGatewaySwitch, vpc.Name) - if err := c.handleAddVpcExternalSubnet(vpc.Name, c.config.ExternalGatewaySwitch); err != nil { - klog.Errorf("failed to add external subnet %s connection for vpc %s, error %v", c.config.ExternalGatewaySwitch, vpc.Name, err) + klog.Infof("connect default external subnet %s with vpc %s", externalGwSwitch, vpc.Name) + if err := c.handleAddVpcExternalSubnet(vpc.Name, externalGwSwitch); err != nil { + klog.Errorf("failed to add external subnet %s connection for vpc %s, error %v", externalGwSwitch, vpc.Name, err) return err } } @@ -703,7 +720,7 @@ func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, custVpcEnableEx for _, subnet := range vpc.Status.ExtraExternalSubnets { if !slices.Contains(vpc.Spec.ExtraExternalSubnets, subnet) { klog.Infof("disconnect external subnet %s to vpc %s", subnet, vpc.Name) - if err := c.handleDelVpcExternalSubnet(vpc.Name, subnet); err != nil { + if err := c.handleDelVpcRes2ExternalSubnet(vpc.Name, subnet); err != nil { klog.Errorf("failed to delete external subnet %s connection for vpc %s, error %v", subnet, vpc.Name, err) return err } @@ -716,7 +733,7 @@ func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, custVpcEnableEx // create bfd between lrp and physical switch gw // bfd status down means current lrp binding chassis node external nic lost external network connectivity // should switch lrp to another node - lrpEipName := fmt.Sprintf("%s-%s", vpc.Name, c.config.ExternalGatewaySwitch) + lrpEipName := fmt.Sprintf("%s-%s", vpc.Name, externalGwSwitch) v4ExtGw, _ := util.SplitStringIP(externalSubnetGW) // TODO: dualstack if _, err := c.OVNNbClient.CreateBFD(lrpEipName, v4ExtGw, c.config.BfdMinRx, c.config.BfdMinTx, c.config.BfdDetectMult, nil); err != nil { @@ -735,7 +752,7 @@ func (c *Controller) handleUpdateVpcExternal(vpc *kubeovnv1.Vpc, custVpcEnableEx } if !vpc.Spec.EnableBfd && vpc.Status.EnableBfd { - lrpEipName := fmt.Sprintf("%s-%s", vpc.Name, c.config.ExternalGatewaySwitch) + lrpEipName := fmt.Sprintf("%s-%s", vpc.Name, externalGwSwitch) if err := c.OVNNbClient.DeleteBFDByDstIP(lrpEipName, ""); err != nil { klog.Error(err) return err @@ -1182,11 +1199,13 @@ func (c *Controller) formatVpc(vpc *kubeovnv1.Vpc) (*kubeovnv1.Vpc, error) { } if vpc.DeletionTimestamp.IsZero() && !slices.Contains(vpc.GetFinalizers(), util.KubeOVNControllerFinalizer) { + controllerutil.RemoveFinalizer(vpc, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(vpc, util.KubeOVNControllerFinalizer) changed = true } if !vpc.DeletionTimestamp.IsZero() && len(vpc.Status.Subnets) == 0 { + controllerutil.RemoveFinalizer(vpc, util.DepreciatedFinalizerName) controllerutil.RemoveFinalizer(vpc, util.KubeOVNControllerFinalizer) changed = true } @@ -1425,7 +1444,7 @@ func (c *Controller) handleDeleteVpcStaticRoute(key string) error { return nil } -func (c *Controller) handleDelVpcExternalSubnet(key, subnet string) error { +func (c *Controller) handleDelVpcRes2ExternalSubnet(key, subnet string) error { lspName := fmt.Sprintf("%s-%s", subnet, key) lrpName := fmt.Sprintf("%s-%s", key, subnet) klog.Infof("delete vpc lrp %s", lrpName) @@ -1512,3 +1531,30 @@ func (c *Controller) updateVpcExternalStatus(key string, enableExternal bool) er return nil } + +// deleteVpc2ExternalConnection deletes VPC connections to external networks +// Deletes both ConfigMap-specified and default connections to ensure complete cleanup +// even if configuration changed during VPC lifecycle +func (c *Controller) deleteVpc2ExternalConnection(vpcName string) error { + var anyErr error + + // Try to delete ConfigMap-specified connection if exists + cm, err := c.configMapsLister.ConfigMaps(c.config.ExternalGatewayConfigNS).Get(util.ExternalGatewayConfig) + if err == nil && cm.Data["external-gw-switch"] != "" { + configSwitch := cm.Data["external-gw-switch"] + if err := c.handleDelVpcRes2ExternalSubnet(vpcName, configSwitch); err != nil { + klog.Errorf("failed to delete ConfigMap-specified connection %s for vpc %s: %v", configSwitch, vpcName, err) + anyErr = err + } + } + + // Always try to delete default connection + if err := c.handleDelVpcRes2ExternalSubnet(vpcName, c.config.ExternalGatewaySwitch); err != nil { + klog.Errorf("failed to delete default connection %s for vpc %s: %v", c.config.ExternalGatewaySwitch, vpcName, err) + if anyErr == nil { + anyErr = err + } + } + + return anyErr +} diff --git a/pkg/controller/vpc_egress_gateway.go b/pkg/controller/vpc_egress_gateway.go index 558e6a95b82..09bc36da434 100644 --- a/pkg/controller/vpc_egress_gateway.go +++ b/pkg/controller/vpc_egress_gateway.go @@ -114,6 +114,7 @@ func (c *Controller) handleAddOrUpdateVpcEgressGateway(key string) error { return err } + controllerutil.RemoveFinalizer(gw, util.DepreciatedFinalizerName) if controllerutil.AddFinalizer(gw, util.KubeOVNControllerFinalizer) { updatedGateway, err := c.config.KubeOvnClient.KubeovnV1().VpcEgressGateways(gw.Namespace). Update(context.Background(), gw, metav1.UpdateOptions{}) @@ -981,6 +982,7 @@ func (c *Controller) handleDelVpcEgressGateway(key string) error { } gw := cachedGateway.DeepCopy() + controllerutil.RemoveFinalizer(gw, util.DepreciatedFinalizerName) if controllerutil.RemoveFinalizer(gw, util.KubeOVNControllerFinalizer) { if _, err = c.config.KubeOvnClient.KubeovnV1().VpcEgressGateways(gw.Namespace). Update(context.Background(), gw, metav1.UpdateOptions{}); err != nil { diff --git a/pkg/controller/vpc_nat_gw_eip.go b/pkg/controller/vpc_nat_gw_eip.go index fb72e4a4a7c..57458c28906 100644 --- a/pkg/controller/vpc_nat_gw_eip.go +++ b/pkg/controller/vpc_nat_gw_eip.go @@ -38,13 +38,6 @@ func (c *Controller) enqueueUpdateIptablesEip(oldObj, newObj any) { klog.Infof("enqueue update iptables eip %s", key) c.updateIptablesEipQueue.Add(key) } - - // Trigger subnet status update when EIP CR is updated - // This ensures both CR labels and IPAM state are synced before status calculation - if oldEip.Status.IP != newEip.Status.IP || oldEip.Status.Ready != newEip.Status.Ready { - externalNetwork := util.GetExternalNetwork(newEip.Spec.ExternalSubnet) - c.updateSubnetStatusQueue.Add(externalNetwork) - } } func (c *Controller) enqueueDelIptablesEip(obj any) { @@ -66,12 +59,7 @@ func (c *Controller) enqueueDelIptablesEip(obj any) { key := cache.MetaObjectToName(eip).String() klog.Infof("enqueue del iptables eip %s", key) - c.delIptablesEipQueue.Add(key) - - // Trigger subnet status update when EIP is deleted - // This ensures subnet status reflects the IP release - externalNetwork := util.GetExternalNetwork(eip.Spec.ExternalSubnet) - c.updateSubnetStatusQueue.Add(externalNetwork) + c.delIptablesEipQueue.Add(eip.DeepCopy()) } func (c *Controller) handleAddIptablesEip(key string) error { @@ -166,6 +154,9 @@ func (c *Controller) handleAddIptablesEip(key string) error { return err } + // Trigger subnet status update after all operations complete + // At this point: IPAM allocated, IptablesEIP CR created with labels+status+finalizer + c.updateSubnetStatusQueue.Add(subnet.Name) return nil } @@ -225,6 +216,21 @@ func (c *Controller) handleUpdateIptablesEip(key string) error { if !cachedEip.DeletionTimestamp.IsZero() { klog.Infof("clean eip %q in pod", key) + + // Check if EIP is still being used by any NAT rules (FIP/DNAT/SNAT) + // Only remove finalizer when no NAT rules are using it + // Note: We query NAT rules directly instead of relying on cachedEip.Status.Nat + // to avoid cache staleness issues + nat, err := c.getIptablesEipNat(cachedEip.Spec.V4ip) + if err != nil { + klog.Errorf("failed to get eip %s nat rules, %v", key, err) + return err + } + if nat != "" { + klog.Infof("eip %s is still being used by NAT rules: %s, waiting for them to be deleted", key, nat) + return nil + } + if vpcNatEnabled == "true" { v4ipCidr, err := util.GetIPAddrWithMask(cachedEip.Status.IP, v4Cidr) if err != nil { @@ -243,11 +249,14 @@ func (c *Controller) handleUpdateIptablesEip(key string) error { return err } } + // Release IP from IPAM before removing finalizer + c.ipam.ReleaseAddressByPod(key, cachedEip.Spec.ExternalSubnet) + + // Now remove finalizer, which will trigger subnet status update if err = c.handleDelIptablesEipFinalizer(key); err != nil { klog.Errorf("failed to handle del finalizer for eip %s, %v", key, err) return err } - c.ipam.ReleaseAddressByPod(key, cachedEip.Spec.ExternalSubnet) return nil } @@ -337,15 +346,23 @@ func (c *Controller) handleUpdateIptablesEip(key string) error { return err } } - if err = c.handleAddIptablesEipFinalizer(key); err != nil { + if err = c.handleAddOrUpdateIptablesEipFinalizer(key); err != nil { klog.Errorf("failed to handle add finalizer for eip, %v", err) return err } return nil } -func (c *Controller) handleDelIptablesEip(key string) error { - klog.Infof("handle delete iptables eip %s", key) +func (c *Controller) handleDelIptablesEip(eip *kubeovnv1.IptablesEIP) error { + klog.Infof("handle delete iptables eip %s", eip.Name) + + // For IptablesEIPs deleted without finalizer (race condition or direct deletion), + // we need to ensure subnet status is updated as a safety net. + externalNetwork := util.GetExternalNetwork(eip.Spec.ExternalSubnet) + if externalNetwork != "" { + c.updateSubnetStatusQueue.Add(externalNetwork) + } + return nil } @@ -581,12 +598,13 @@ func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, ext return err } } - externalNetwork := util.GetExternalNetwork(cachedEip.Spec.ExternalSubnet) if needCreate { klog.V(3).Infof("create eip cr %s", key) + // Create CR with finalizer, labels and status all at once _, err := c.config.KubeOvnClient.KubeovnV1().IptablesEIPs().Create(context.Background(), &kubeovnv1.IptablesEIP{ ObjectMeta: metav1.ObjectMeta{ - Name: key, + Name: key, + Finalizers: []string{util.KubeOVNControllerFinalizer}, Labels: map[string]string{ util.SubnetNameLabel: externalNet, util.EipV4IpLabel: v4ip, @@ -598,6 +616,14 @@ func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, ext V6ip: v6ip, MacAddress: mac, NatGwDp: natGwDp, + QoSPolicy: qos, + }, + Status: kubeovnv1.IptablesEIPStatus{ + IP: v4ip, + Ready: true, + QoSPolicy: qos, + Nat: "", + Redo: "", }, }, metav1.CreateOptions{}) if err != nil { @@ -607,12 +633,25 @@ func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, ext } } else { eip := cachedEip.DeepCopy() + + // Ensure labels are set correctly before any update + if eip.Labels == nil { + eip.Labels = make(map[string]string) + } + eip.Labels[util.SubnetNameLabel] = externalNet + eip.Labels[util.VpcNatGatewayNameLabel] = natGwDp + eip.Labels[util.EipV4IpLabel] = v4ip + if eip.Spec.QoSPolicy != "" { + eip.Labels[util.QoSLabel] = eip.Spec.QoSPolicy + } + if v4ip != "" { klog.V(3).Infof("update eip cr %s", key) eip.Spec.V4ip = v4ip eip.Spec.V6ip = v6ip eip.Spec.NatGwDp = natGwDp eip.Spec.MacAddress = mac + // Update with labels and spec in one call if _, err := c.config.KubeOvnClient.KubeovnV1().IptablesEIPs().Update(context.Background(), eip, metav1.UpdateOptions{}); err != nil { errMsg := fmt.Errorf("failed to update eip crd %s, %w", key, err) klog.Error(errMsg) @@ -639,38 +678,15 @@ func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, ext return err } } - var needUpdateLabel bool - var op string - if len(eip.Labels) == 0 { - op = "add" - eip.Labels = map[string]string{ - util.SubnetNameLabel: externalNetwork, - util.VpcNatGatewayNameLabel: natGwDp, - util.EipV4IpLabel: v4ip, - } - needUpdateLabel = true - } else if eip.Labels[util.SubnetNameLabel] != externalNetwork { - op = "replace" - eip.Labels[util.SubnetNameLabel] = externalNetwork - eip.Labels[util.VpcNatGatewayNameLabel] = natGwDp - needUpdateLabel = true - } - if eip.Spec.QoSPolicy != "" && eip.Labels[util.QoSLabel] != eip.Spec.QoSPolicy { - eip.Labels[util.QoSLabel] = eip.Spec.QoSPolicy - needUpdateLabel = true - } - if needUpdateLabel { - if err := c.updateIptableLabels(eip.Name, op, "eip", eip.Labels); err != nil { - klog.Errorf("failed to update eip %s labels, %v", eip.Name, err) - return err - } - } - if err = c.handleAddIptablesEipFinalizer(key); err != nil { - klog.Errorf("failed to handle add finalizer for eip, %v", err) + if err = c.handleAddOrUpdateIptablesEipFinalizer(key); err != nil { + klog.Errorf("failed to handle add or update finalizer for eip, %v", err) return err } } + // Trigger subnet status update after all operations complete + time.Sleep(300 * time.Millisecond) + c.updateSubnetStatusQueue.Add(externalNet) return nil } @@ -685,7 +701,7 @@ func (c *Controller) syncIptablesEipFinalizer(cl client.Client) error { }) } -func (c *Controller) handleAddIptablesEipFinalizer(key string) error { +func (c *Controller) handleAddOrUpdateIptablesEipFinalizer(key string) error { cachedIptablesEip, err := c.iptablesEipsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { @@ -694,10 +710,11 @@ func (c *Controller) handleAddIptablesEipFinalizer(key string) error { klog.Error(err) return err } - if !cachedIptablesEip.DeletionTimestamp.IsZero() || len(cachedIptablesEip.GetFinalizers()) != 0 { + if !cachedIptablesEip.DeletionTimestamp.IsZero() { return nil } newIptablesEip := cachedIptablesEip.DeepCopy() + controllerutil.RemoveFinalizer(newIptablesEip, util.DepreciatedFinalizerName) controllerutil.AddFinalizer(newIptablesEip, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedIptablesEip, newIptablesEip) if err != nil { @@ -712,6 +729,12 @@ func (c *Controller) handleAddIptablesEipFinalizer(key string) error { klog.Errorf("failed to add finalizer for iptables eip '%s', %v", cachedIptablesEip.Name, err) return err } + + // Trigger subnet status update after finalizer is processed as a fallback + // This handles cases where finalizer was not added during creation + // AddFinalizer is idempotent, so this is safe even if finalizer already exists + externalNetwork := util.GetExternalNetwork(cachedIptablesEip.Spec.ExternalSubnet) + c.updateSubnetStatusQueue.Add(externalNetwork) return nil } @@ -728,6 +751,7 @@ func (c *Controller) handleDelIptablesEipFinalizer(key string) error { return nil } newIptablesEip := cachedIptablesEip.DeepCopy() + controllerutil.RemoveFinalizer(newIptablesEip, util.DepreciatedFinalizerName) controllerutil.RemoveFinalizer(newIptablesEip, util.KubeOVNControllerFinalizer) patch, err := util.GenerateMergePatchPayload(cachedIptablesEip, newIptablesEip) if err != nil { @@ -742,6 +766,13 @@ func (c *Controller) handleDelIptablesEipFinalizer(key string) error { klog.Errorf("failed to remove finalizer from iptables eip '%s', %v", cachedIptablesEip.Name, err) return err } + + // Trigger subnet status update after finalizer is removed + // This ensures subnet status reflects the IP release + // Add delay to ensure API server completes the finalizer removal + time.Sleep(300 * time.Millisecond) + externalNetwork := util.GetExternalNetwork(cachedIptablesEip.Spec.ExternalSubnet) + c.updateSubnetStatusQueue.Add(externalNetwork) return nil } diff --git a/pkg/daemon/ovs_linux.go b/pkg/daemon/ovs_linux.go index 1410ed5cfee..463d5faaaef 100644 --- a/pkg/daemon/ovs_linux.go +++ b/pkg/daemon/ovs_linux.go @@ -1491,6 +1491,18 @@ func (c *Controller) configProviderNic(nicName, brName string, trunks []string) var mtu int if !isUserspaceDP { + // Check if bridge exists before attempting to add port + output, err := ovs.Exec("list-br") + if err != nil { + klog.Errorf("failed to list OVS bridges: %v, %q", err, output) + return 0, err + } + if !slices.Contains(strings.Split(output, "\n"), brName) { + err := fmt.Errorf("bridge %s does not exist", brName) + klog.Error(err) + return 0, err + } + mtu, err = c.transferAddrsAndRoutes(nicName, brName, false) if err != nil { klog.Errorf("failed to transfer addresses and routes from %s to %s: %v", nicName, brName, err) diff --git a/pkg/ovs/ovn-nb-acl_test.go b/pkg/ovs/ovn-nb-acl_test.go index fa4f2bd87c9..46451f890e4 100644 --- a/pkg/ovs/ovn-nb-acl_test.go +++ b/pkg/ovs/ovn-nb-acl_test.go @@ -2596,8 +2596,15 @@ func (suite *OvnClientTestSuite) testUpdateAnpRuleACLOps() { isIngress := true isBanp := false + // Clean up any existing port group from previous test runs + _ = nbClient.DeletePortGroup(pgName) + err := nbClient.CreatePortGroup(pgName, nil) require.NoError(t, err) + defer func() { + err := nbClient.DeletePortGroup(pgName) + require.NoError(t, err) + }() ops, err := nbClient.UpdateAnpRuleACLOps(pgName, asName, protocol, aclName, priority, aclAction, logACLActions, rulePorts, isIngress, isBanp) require.NoError(t, err) require.NotEmpty(t, ops) @@ -2616,8 +2623,15 @@ func (suite *OvnClientTestSuite) testUpdateAnpRuleACLOps() { isIngress := false isBanp := true + // Clean up any existing port group from previous test runs + _ = nbClient.DeletePortGroup(pgName) + err := nbClient.CreatePortGroup(pgName, nil) require.NoError(t, err) + defer func() { + err := nbClient.DeletePortGroup(pgName) + require.NoError(t, err) + }() ops, err := nbClient.UpdateAnpRuleACLOps(pgName, asName, protocol, aclName, priority, aclAction, logACLActions, rulePorts, isIngress, isBanp) require.NoError(t, err) require.NotEmpty(t, ops) diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index 8e003344aaa..75bab91f307 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -63,7 +63,7 @@ func NewLegacyClient(timeout int) *LegacyClient { func NewDynamicOvnNbClient( ovnNbAddr string, - ovnNbTimeout, ovsDbConTimeout, ovsDbInactivityTimeout int, + ovnNbTimeout, ovsDbConTimeout, ovsDbInactivityTimeout, maxRetry int, tables ...string, ) (*OVNNbClient, map[string]model.Model, error) { dbModel, err := model.NewClientDBModel(ovnnb.DatabaseName, nil) @@ -71,16 +71,28 @@ func NewDynamicOvnNbClient( return nil, nil, fmt.Errorf("failed to create client db model: %w", err) } - nbClient, err := ovsclient.NewOvsDbClient( - ovsclient.NBDB, - ovnNbAddr, - dbModel, - nil, - ovsDbConTimeout, - ovsDbInactivityTimeout, - ) - if err != nil { - return nil, nil, fmt.Errorf("failed to create initial ovsdb client to fetch schema: %w", err) + // First connection to fetch schema with retry + var nbClient client.Client + try := 0 + for { + nbClient, err = ovsclient.NewOvsDbClient( + ovsclient.NBDB, + ovnNbAddr, + dbModel, + nil, + ovsDbConTimeout, + ovsDbInactivityTimeout, + ) + if err != nil { + klog.Errorf("failed to create initial OVN NB client to fetch schema: %v", err) + } else { + break + } + if try >= maxRetry { + return nil, nil, fmt.Errorf("failed to create initial ovsdb client to fetch schema after %d retries: %w", maxRetry, err) + } + time.Sleep(2 * time.Second) + try++ } schemaTables := nbClient.Schema().Tables @@ -117,15 +129,27 @@ func NewDynamicOvnNbClient( return nil, nil, fmt.Errorf("failed to create dynamic client db model: %w", err) } - if nbClient, err = ovsclient.NewOvsDbClient( - ovsclient.NBDB, - ovnNbAddr, - dbModel, - monitors, - ovsDbConTimeout, - ovsDbInactivityTimeout, - ); err != nil { - return nil, nil, fmt.Errorf("failed to create dynamic ovsdb client: %w", err) + // Second connection with dynamic model and retry + try = 0 + for { + nbClient, err = ovsclient.NewOvsDbClient( + ovsclient.NBDB, + ovnNbAddr, + dbModel, + monitors, + ovsDbConTimeout, + ovsDbInactivityTimeout, + ) + if err != nil { + klog.Errorf("failed to create dynamic OVN NB client: %v", err) + } else { + break + } + if try >= maxRetry { + return nil, nil, fmt.Errorf("failed to create dynamic ovsdb client after %d retries: %w", maxRetry, err) + } + time.Sleep(2 * time.Second) + try++ } c := &OVNNbClient{ diff --git a/test/e2e/framework/docker/network.go b/test/e2e/framework/docker/network.go index 5eefb54d353..9f33a38378b 100644 --- a/test/e2e/framework/docker/network.go +++ b/test/e2e/framework/docker/network.go @@ -5,9 +5,12 @@ import ( "crypto/sha1" "encoding/binary" "fmt" + "math/rand" "net" "net/netip" "strconv" + "strings" + "time" "github.com/moby/moby/api/types/network" "github.com/moby/moby/client" @@ -18,6 +21,41 @@ import ( const MTU = 1500 +// Base network for test networks: 172.28.0.0/16 +// This allows 256 /24 subnets (172.28.0.0/24 to 172.28.255.0/24) +const testNetworkBase = "172.28" + +// GenerateRandomSubnets generates N random /24 subnets within 172.28.0.0/16 +// Returns a slice of subnets in CIDR notation (e.g., ["172.28.123.0/24", "172.28.124.0/24"]) +// Subnets are guaranteed to be unique within the returned slice +func GenerateRandomSubnets(count int) []string { + if count <= 0 || count > 256 { + panic(fmt.Sprintf("invalid subnet count: %d (must be 1-256)", count)) + } + + // Use current nanosecond timestamp as seed for better randomness + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + + // Generate a random permutation of subnet indices + subnets := make([]string, count) + usedOctets := make(map[int]bool) + + for i := 0; i < count; i++ { + var octet int + // Find an unused octet + for { + octet = rng.Intn(256) + if !usedOctets[octet] { + usedOctets[octet] = true + break + } + } + subnets[i] = fmt.Sprintf("%s.%d.0/24", testNetworkBase, octet) + } + + return subnets +} + // https://github.com/kubernetes-sigs/kind/tree/main/pkg/cluster/internal/providers/docker/network.go#L313 // generateULASubnetFromName generate an IPv6 subnet based on the // name and Nth probing attempt @@ -71,7 +109,8 @@ func NetworkInspect(name string) (*network.Inspect, error) { return getNetwork(name, false) } -func NetworkCreate(name string, ipv6, skipIfExists bool) (*network.Inspect, error) { +// NetworkCreateWithSubnet creates a docker network with specified IPv4 subnet +func NetworkCreateWithSubnet(name, ipv4Subnet string, ipv6, skipIfExists bool) (*network.Inspect, error) { if skipIfExists { network, err := getNetwork(name, true) if err != nil { @@ -93,6 +132,21 @@ func NetworkCreate(name string, ipv6, skipIfExists bool) (*network.Inspect, erro "com.docker.network.driver.mtu": strconv.Itoa(MTU), }, } + + // Add IPv4 subnet if specified + if ipv4Subnet != "" { + gateway, err := util.FirstIP(ipv4Subnet) + if err != nil { + return nil, fmt.Errorf("failed to get gateway for subnet %s: %w", ipv4Subnet, err) + } + config := network.IPAMConfig{ + Subnet: netip.MustParsePrefix(ipv4Subnet), + Gateway: netip.MustParseAddr(gateway), + } + options.IPAM.Config = append(options.IPAM.Config, config) + } + + // Add IPv6 subnet if enabled if ipv6 { options.EnableIPv6 = ptr.To(true) subnet := generateULASubnetFromName(name, 0) @@ -114,12 +168,27 @@ func NetworkCreate(name string, ipv6, skipIfExists bool) (*network.Inspect, erro defer cli.Close() if _, err = cli.NetworkCreate(context.Background(), name, options); err != nil { + // Handle race condition: if network was created between our check and create attempt + // Docker returns error like "network with name xxx already exists" + if skipIfExists && strings.Contains(err.Error(), "already exists") { + // Network already exists, retrieve and return it + network, getErr := getNetwork(name, false) + if getErr != nil { + return nil, getErr + } + return network, nil + } return nil, err } return getNetwork(name, false) } +// NetworkCreate creates a docker network (backward compatible wrapper) +func NetworkCreate(name string, ipv6, skipIfExists bool) (*network.Inspect, error) { + return NetworkCreateWithSubnet(name, "", ipv6, skipIfExists) +} + func NetworkConnect(networkID, containerID string) error { cli, err := client.New(client.FromEnv) if err != nil { diff --git a/test/e2e/framework/docker/network_test.go b/test/e2e/framework/docker/network_test.go new file mode 100644 index 00000000000..6c6094b27a6 --- /dev/null +++ b/test/e2e/framework/docker/network_test.go @@ -0,0 +1,274 @@ +package docker + +import ( + "fmt" + "regexp" + "strings" + "testing" +) + +func TestGenerateRandomSubnets(t *testing.T) { + tests := []struct { + name string + count int + expectPanic bool + validateFunc func([]string) error + }{ + { + name: "generate 1 subnet", + count: 1, + validateFunc: func(subnets []string) error { + if len(subnets) != 1 { + return fmt.Errorf("expected 1 subnet, got %d", len(subnets)) + } + return nil + }, + }, + { + name: "generate 2 subnets", + count: 2, + validateFunc: func(subnets []string) error { + if len(subnets) != 2 { + return fmt.Errorf("expected 2 subnets, got %d", len(subnets)) + } + return nil + }, + }, + { + name: "generate 5 subnets", + count: 5, + validateFunc: func(subnets []string) error { + if len(subnets) != 5 { + return fmt.Errorf("expected 5 subnets, got %d", len(subnets)) + } + return nil + }, + }, + { + name: "generate maximum (256) subnets", + count: 256, + validateFunc: func(subnets []string) error { + if len(subnets) != 256 { + return fmt.Errorf("expected 256 subnets, got %d", len(subnets)) + } + return nil + }, + }, + { + name: "invalid count: 0", + count: 0, + expectPanic: true, + }, + { + name: "invalid count: negative", + count: -1, + expectPanic: true, + }, + { + name: "invalid count: > 256", + count: 257, + expectPanic: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.expectPanic { + defer func() { + if r := recover(); r == nil { + t.Errorf("expected panic but didn't get one") + } + }() + GenerateRandomSubnets(tt.count) + return + } + + subnets := GenerateRandomSubnets(tt.count) + + if tt.validateFunc != nil { + if err := tt.validateFunc(subnets); err != nil { + t.Errorf("validation failed: %v", err) + } + } + + // Common validations for all non-panic cases + if err := validateSubnetFormat(subnets); err != nil { + t.Errorf("subnet format validation failed: %v", err) + } + + if err := validateSubnetUniqueness(subnets); err != nil { + t.Errorf("subnet uniqueness validation failed: %v", err) + } + + if err := validateSubnetRange(subnets); err != nil { + t.Errorf("subnet range validation failed: %v", err) + } + }) + } +} + +func TestGenerateRandomSubnetsUniqueness(t *testing.T) { + // Test that multiple invocations produce different results + iterations := 10 + allSubnets := make(map[string]int) + + for i := 0; i < iterations; i++ { + subnets := GenerateRandomSubnets(2) + for _, subnet := range subnets { + allSubnets[subnet]++ + } + } + + // We expect to see some variety (not all the same) + if len(allSubnets) < 3 { + t.Errorf("expected at least 3 different subnets across %d iterations, got %d: %v", + iterations, len(allSubnets), allSubnets) + } + + t.Logf("Generated %d unique subnets across %d iterations", len(allSubnets), iterations) +} + +func TestGenerateRandomSubnetsDistribution(t *testing.T) { + // Test randomness distribution + count := 100 + subnets := make([]string, count) + for i := 0; i < count; i++ { + result := GenerateRandomSubnets(1) + subnets[i] = result[0] + } + + uniqueSubnets := make(map[string]bool) + for _, subnet := range subnets { + uniqueSubnets[subnet] = true + } + + uniqueCount := len(uniqueSubnets) + // Expect at least 50% uniqueness in 100 iterations + minExpected := count / 2 + if uniqueCount < minExpected { + t.Errorf("poor randomness: expected at least %d unique subnets in %d iterations, got %d", + minExpected, count, uniqueCount) + } + + t.Logf("Randomness check: %d unique subnets out of %d iterations (%.1f%%)", + uniqueCount, count, float64(uniqueCount)/float64(count)*100) +} + +func TestGenerateRandomSubnetsNoCollision(t *testing.T) { + // Test that generating 2 subnets never produces collision within a single call + iterations := 100 + for i := 0; i < iterations; i++ { + subnets := GenerateRandomSubnets(2) + if subnets[0] == subnets[1] { + t.Errorf("iteration %d: collision detected: %s == %s", i, subnets[0], subnets[1]) + } + } + t.Logf("No collisions detected in %d iterations", iterations) +} + +func TestSubnetFormatConsistency(t *testing.T) { + // Test that all generated subnets follow the expected format + subnets := GenerateRandomSubnets(10) + expectedPattern := regexp.MustCompile(`^172\.28\.\d{1,3}\.0/24$`) + + for i, subnet := range subnets { + if !expectedPattern.MatchString(subnet) { + t.Errorf("subnet %d has invalid format: %s", i, subnet) + } + + // Verify the third octet is in valid range (0-255) + parts := strings.Split(strings.TrimSuffix(subnet, "/24"), ".") + if len(parts) != 4 { + t.Errorf("subnet %d has invalid structure: %s", i, subnet) + continue + } + + var thirdOctet int + if _, err := fmt.Sscanf(parts[2], "%d", &thirdOctet); err != nil { + t.Errorf("subnet %d has invalid third octet: %s", i, parts[2]) + continue + } + + if thirdOctet < 0 || thirdOctet > 255 { + t.Errorf("subnet %d has third octet out of range: %d", i, thirdOctet) + } + } +} + +// Helper functions for validation + +func validateSubnetFormat(subnets []string) error { + pattern := regexp.MustCompile(`^172\.28\.\d{1,3}\.0/24$`) + for i, subnet := range subnets { + if !pattern.MatchString(subnet) { + return fmt.Errorf("subnet %d has invalid format: %s (expected 172.28.X.0/24)", i, subnet) + } + } + return nil +} + +func validateSubnetUniqueness(subnets []string) error { + seen := make(map[string]bool) + for i, subnet := range subnets { + if seen[subnet] { + return fmt.Errorf("duplicate subnet found at index %d: %s", i, subnet) + } + seen[subnet] = true + } + return nil +} + +func validateSubnetRange(subnets []string) error { + for i, subnet := range subnets { + parts := strings.Split(strings.TrimSuffix(subnet, "/24"), ".") + if len(parts) != 4 { + return fmt.Errorf("subnet %d has invalid structure: %s", i, subnet) + } + + // Validate base (172.28) + if parts[0] != "172" || parts[1] != "28" { + return fmt.Errorf("subnet %d has invalid base: %s.%s (expected 172.28)", i, parts[0], parts[1]) + } + + // Validate third octet (0-255) + var thirdOctet int + if _, err := fmt.Sscanf(parts[2], "%d", &thirdOctet); err != nil { + return fmt.Errorf("subnet %d has invalid third octet: %s", i, parts[2]) + } + if thirdOctet < 0 || thirdOctet > 255 { + return fmt.Errorf("subnet %d has third octet out of range: %d (must be 0-255)", i, thirdOctet) + } + + // Validate fourth octet (must be 0) + if parts[3] != "0" { + return fmt.Errorf("subnet %d has invalid fourth octet: %s (expected 0)", i, parts[3]) + } + } + return nil +} + +// Benchmark tests + +func BenchmarkGenerateRandomSubnets1(b *testing.B) { + for i := 0; i < b.N; i++ { + GenerateRandomSubnets(1) + } +} + +func BenchmarkGenerateRandomSubnets2(b *testing.B) { + for i := 0; i < b.N; i++ { + GenerateRandomSubnets(2) + } +} + +func BenchmarkGenerateRandomSubnets10(b *testing.B) { + for i := 0; i < b.N; i++ { + GenerateRandomSubnets(10) + } +} + +func BenchmarkGenerateRandomSubnets100(b *testing.B) { + for i := 0; i < b.N; i++ { + GenerateRandomSubnets(100) + } +} diff --git a/test/e2e/framework/ovn_address_set.go b/test/e2e/framework/ovn_address_set.go index 1211e51ad73..18453c3fd93 100644 --- a/test/e2e/framework/ovn_address_set.go +++ b/test/e2e/framework/ovn_address_set.go @@ -176,15 +176,48 @@ func getOVNNbClient(tables ...string) (*ovs.OVNNbClient, map[string]model.Model) }) ExpectNoError(ovnnbAddrErr) - client, models, err := ovs.NewDynamicOvnNbClient( - ovnnbAddr, - ovnNbTimeoutSeconds, - ovsdbConnTimeout, - ovsdbInactivityTimeout, - tables..., - ) - ExpectNoError(err) + var client *ovs.OVNNbClient + var models map[string]model.Model + var err error + + // Retry the entire client creation and connection verification process + for try := 0; try <= ovnClientMaxRetry; try++ { + client, models, err = ovs.NewDynamicOvnNbClient( + ovnnbAddr, + ovnNbTimeoutSeconds, + ovsdbConnTimeout, + ovsdbInactivityTimeout, + ovnClientMaxRetry, + tables..., + ) + if err != nil { + if try < ovnClientMaxRetry { + Logf("Failed to create OVN NB client (attempt %d/%d): %v, retrying...", try+1, ovnClientMaxRetry+1, err) + time.Sleep(2 * time.Second) + continue + } + break + } + + // Verify the connection is actually working by checking if we're connected + connected := client.Connected() + if !connected { + client.Close() + err = errors.New("client created but not connected") + if try < ovnClientMaxRetry { + Logf("OVN NB client not connected (attempt %d/%d), retrying...", try+1, ovnClientMaxRetry+1) + time.Sleep(2 * time.Second) + continue + } + break + } + + // Connection verified, return the client + return client, models + } + + ExpectNoError(err) return client, models } diff --git a/test/e2e/framework/vpc-nat-gateway.go b/test/e2e/framework/vpc-nat-gateway.go index 0b2a547e1d6..1607df81003 100644 --- a/test/e2e/framework/vpc-nat-gateway.go +++ b/test/e2e/framework/vpc-nat-gateway.go @@ -222,3 +222,9 @@ func MakeVpcNatGateway(name, vpc, subnet, lanIP, externalSubnet, qosPolicyName s vpcNatGw.Spec.QoSPolicy = qosPolicyName return vpcNatGw } + +func MakeVpcNatGatewayWithNoDefaultEIP(name, vpc, subnet, lanIP, externalSubnet, qosPolicyName string, noDefaultEIP bool) *apiv1.VpcNatGateway { + vpcNatGw := MakeVpcNatGateway(name, vpc, subnet, lanIP, externalSubnet, qosPolicyName) + vpcNatGw.Spec.NoDefaultEIP = noDefaultEIP + return vpcNatGw +} diff --git a/test/e2e/ip/e2e_test.go b/test/e2e/ip/e2e_test.go new file mode 100644 index 00000000000..9178565d438 --- /dev/null +++ b/test/e2e/ip/e2e_test.go @@ -0,0 +1,449 @@ +package ip + +import ( + "context" + "flag" + "fmt" + "strings" + "testing" + "time" + + "github.com/onsi/ginkgo/v2" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "k8s.io/kubernetes/test/e2e" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/test/e2e/framework" +) + +var _ = framework.Describe("[group:ip]", func() { + f := framework.NewDefaultFramework("ip") + + var vpcClient *framework.VpcClient + var subnetClient *framework.SubnetClient + var podClient *framework.PodClient + var ipClient *framework.IPClient + var namespaceName, vpcName, subnetName, cidr string + + ginkgo.BeforeEach(func() { + vpcClient = f.VpcClient() + subnetClient = f.SubnetClient() + podClient = f.PodClient() + ipClient = f.IPClient() + namespaceName = f.Namespace.Name + cidr = framework.RandomCIDR(f.ClusterIPFamily) + + randomSuffix := framework.RandomSuffix() + vpcName = "vpc-" + randomSuffix + subnetName = "subnet-" + randomSuffix + + ginkgo.By("Creating vpc " + vpcName) + vpc := framework.MakeVpc(vpcName, "", false, false, []string{namespaceName}) + _ = vpcClient.CreateSync(vpc) + + ginkgo.By("Creating subnet " + subnetName) + subnet := framework.MakeSubnet(subnetName, "", cidr, "", vpcName, "", nil, nil, []string{namespaceName}) + _ = subnetClient.CreateSync(subnet) + }) + + ginkgo.AfterEach(func() { + ginkgo.By("Deleting subnet " + subnetName) + subnetClient.DeleteSync(subnetName) + ginkgo.By("Deleting vpc " + vpcName) + vpcClient.DeleteSync(vpcName) + }) + + framework.ConformanceIt("Test IP CR creation and deletion with finalizer and subnet status update", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + ginkgo.By("1. Get initial subnet status") + initialSubnet := subnetClient.Get(subnetName) + initialV4AvailableIPs := initialSubnet.Status.V4AvailableIPs + initialV4UsingIPs := initialSubnet.Status.V4UsingIPs + initialV6AvailableIPs := initialSubnet.Status.V6AvailableIPs + initialV6UsingIPs := initialSubnet.Status.V6UsingIPs + initialV4AvailableIPRange := initialSubnet.Status.V4AvailableIPRange + initialV4UsingIPRange := initialSubnet.Status.V4UsingIPRange + initialV6AvailableIPRange := initialSubnet.Status.V6AvailableIPRange + initialV6UsingIPRange := initialSubnet.Status.V6UsingIPRange + + ginkgo.By("2. Create a pod to trigger IP CR creation") + podName := "test-ip-pod-" + framework.RandomSuffix() + cmd := []string{"sleep", "infinity"} + pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) + _ = podClient.CreateSync(pod) + + ginkgo.By("3. Wait for IP CR to be created and get IP CR name") + var ipCR *apiv1.IP + var ipName string + for range 30 { + // IP CR name format: podName.namespaceName + ipName = fmt.Sprintf("%s.%s", podName, namespaceName) + ipCR = ipClient.Get(ipName) + if ipCR != nil && ipCR.Name != "" { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectNotNil(ipCR, "IP CR should be created for pod %s", podName) + framework.ExpectEqual(ipCR.Spec.Subnet, subnetName, "IP CR should be in the correct subnet") + + ginkgo.By("4. Wait for IP CR finalizer to be added") + for range 60 { + ipCR = ipClient.Get(ipName) + if ipCR != nil && len(ipCR.Finalizers) > 0 { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectNotNil(ipCR, "IP CR should exist") + framework.ExpectContainElement(ipCR.Finalizers, util.KubeOVNControllerFinalizer, + "IP CR should have finalizer after creation") + + ginkgo.By("5. Wait for subnet status to be updated after IP creation") + time.Sleep(5 * time.Second) + + ginkgo.By("6. Verify subnet status after IP CR creation") + afterCreateSubnet := subnetClient.Get(subnetName) + switch afterCreateSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: + // Verify IP count changed + framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1 after IP creation") + framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1 after IP creation") + + // Verify IP range changed + framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IP creation") + framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IP creation") + + // Verify the IP's address is in the using range + podIP := ipCR.Spec.V4IPAddress + framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V4UsingIPRange, podIP), + "Pod IP %s should be in V4UsingIPRange %s", podIP, afterCreateSubnet.Status.V4UsingIPRange) + case apiv1.ProtocolIPv6: + // Verify IP count changed + framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1 after IP creation") + framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1 after IP creation") + + // Verify IP range changed + framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IP creation") + framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IP creation") + + // Verify the IP's address is in the using range + podIP := ipCR.Spec.V6IPAddress + framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V6UsingIPRange, podIP), + "Pod IP %s should be in V6UsingIPRange %s", podIP, afterCreateSubnet.Status.V6UsingIPRange) + default: + // Dual stack + framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1 after IP creation") + framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1 after IP creation") + framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1 after IP creation") + framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1 after IP creation") + + framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IP creation") + framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IP creation") + framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IP creation") + framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IP creation") + } + + // Store the status after creation for later comparison + afterCreateV4AvailableIPs := afterCreateSubnet.Status.V4AvailableIPs + afterCreateV4UsingIPs := afterCreateSubnet.Status.V4UsingIPs + afterCreateV6AvailableIPs := afterCreateSubnet.Status.V6AvailableIPs + afterCreateV6UsingIPs := afterCreateSubnet.Status.V6UsingIPs + afterCreateV4AvailableIPRange := afterCreateSubnet.Status.V4AvailableIPRange + afterCreateV4UsingIPRange := afterCreateSubnet.Status.V4UsingIPRange + afterCreateV6AvailableIPRange := afterCreateSubnet.Status.V6AvailableIPRange + afterCreateV6UsingIPRange := afterCreateSubnet.Status.V6UsingIPRange + + ginkgo.By("7. Delete the pod to trigger IP CR deletion") + podClient.DeleteSync(podName) + + ginkgo.By("8. Wait for IP CR to be deleted") + deleted := false + for range 30 { + _, err := f.KubeOVNClientSet.KubeovnV1().IPs().Get(context.Background(), ipName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + deleted = true + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectTrue(deleted, "IP CR should be deleted") + + ginkgo.By("9. Wait for subnet status to be updated after IP deletion") + time.Sleep(5 * time.Second) + + ginkgo.By("10. Verify subnet status after IP CR deletion") + afterDeleteSubnet := subnetClient.Get(subnetName) + switch afterDeleteSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: + // Verify IP count is restored + framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1 after IP deletion") + framework.ExpectEqual(afterCreateV4UsingIPs-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1 after IP deletion") + + // Verify IP range changed + framework.ExpectNotEqual(afterCreateV4AvailableIPRange, afterDeleteSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IP deletion") + framework.ExpectNotEqual(afterCreateV4UsingIPRange, afterDeleteSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IP deletion") + + // Verify counts match initial state + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after IP deletion") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after IP deletion") + case apiv1.ProtocolIPv6: + // Verify IP count is restored + framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1 after IP deletion") + framework.ExpectEqual(afterCreateV6UsingIPs-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1 after IP deletion") + + // Verify IP range changed + framework.ExpectNotEqual(afterCreateV6AvailableIPRange, afterDeleteSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IP deletion") + framework.ExpectNotEqual(afterCreateV6UsingIPRange, afterDeleteSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IP deletion") + + // Verify counts match initial state + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after IP deletion") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after IP deletion") + default: + // Dual stack + framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1 after IP deletion") + framework.ExpectEqual(afterCreateV4UsingIPs-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1 after IP deletion") + framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1 after IP deletion") + framework.ExpectEqual(afterCreateV6UsingIPs-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1 after IP deletion") + + framework.ExpectNotEqual(afterCreateV4AvailableIPRange, afterDeleteSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IP deletion") + framework.ExpectNotEqual(afterCreateV4UsingIPRange, afterDeleteSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IP deletion") + framework.ExpectNotEqual(afterCreateV6AvailableIPRange, afterDeleteSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IP deletion") + framework.ExpectNotEqual(afterCreateV6UsingIPRange, afterDeleteSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IP deletion") + + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after IP deletion") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after IP deletion") + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after IP deletion") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after IP deletion") + } + + ginkgo.By("11. Test completed: IP CR creation and deletion properly updates subnet status via finalizer handlers") + }) + + framework.ConformanceIt("Test multiple IPs with pod lifecycle", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + ginkgo.By("1. Get initial subnet status") + initialSubnet := subnetClient.Get(subnetName) + initialV4AvailableIPs := initialSubnet.Status.V4AvailableIPs + initialV4UsingIPs := initialSubnet.Status.V4UsingIPs + initialV6AvailableIPs := initialSubnet.Status.V6AvailableIPs + initialV6UsingIPs := initialSubnet.Status.V6UsingIPs + + ginkgo.By("2. Create multiple pods to trigger multiple IP CR creations") + numPods := 3 + podNames := make([]string, numPods) + ipNames := make([]string, numPods) + cmd := []string{"sleep", "infinity"} + + for i := range numPods { + podName := fmt.Sprintf("test-multi-ip-pod-%d-%s", i, framework.RandomSuffix()) + podNames[i] = podName + ipNames[i] = fmt.Sprintf("%s.%s", podName, namespaceName) + + pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) + _ = podClient.CreateSync(pod) + } + + ginkgo.By("3. Wait for all IP CRs to be created and get finalizers") + for i := range numPods { + var ipCR *apiv1.IP + for range 60 { + ipCR = ipClient.Get(ipNames[i]) + if ipCR != nil && ipCR.Name != "" && len(ipCR.Finalizers) > 0 { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectNotNil(ipCR, "IP CR should be created for pod %s", podNames[i]) + framework.ExpectContainElement(ipCR.Finalizers, util.KubeOVNControllerFinalizer, + "IP CR should have finalizer for pod %s", podNames[i]) + } + + ginkgo.By("4. Wait for subnet status to be updated after all IPs created") + time.Sleep(5 * time.Second) + + ginkgo.By("5. Verify subnet status after multiple IP CR creations") + afterCreateSubnet := subnetClient.Get(subnetName) + switch afterCreateSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: + framework.ExpectEqual(initialV4AvailableIPs-float64(numPods), afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by %d after creating %d IPs", numPods, numPods) + framework.ExpectEqual(initialV4UsingIPs+float64(numPods), afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by %d after creating %d IPs", numPods, numPods) + case apiv1.ProtocolIPv6: + framework.ExpectEqual(initialV6AvailableIPs-float64(numPods), afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by %d after creating %d IPs", numPods, numPods) + framework.ExpectEqual(initialV6UsingIPs+float64(numPods), afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by %d after creating %d IPs", numPods, numPods) + default: + // Dual stack + framework.ExpectEqual(initialV4AvailableIPs-float64(numPods), afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by %d after creating %d IPs", numPods, numPods) + framework.ExpectEqual(initialV4UsingIPs+float64(numPods), afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by %d after creating %d IPs", numPods, numPods) + framework.ExpectEqual(initialV6AvailableIPs-float64(numPods), afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by %d after creating %d IPs", numPods, numPods) + framework.ExpectEqual(initialV6UsingIPs+float64(numPods), afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by %d after creating %d IPs", numPods, numPods) + } + + ginkgo.By("6. Delete all pods to trigger IP CR deletions") + for i := range numPods { + podClient.DeleteSync(podNames[i]) + } + + ginkgo.By("7. Wait for all IP CRs to be deleted") + for i := range numPods { + deleted := false + for range 30 { + _, err := f.KubeOVNClientSet.KubeovnV1().IPs().Get(context.Background(), ipNames[i], metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + deleted = true + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectTrue(deleted, "IP CR %s should be deleted", ipNames[i]) + } + + ginkgo.By("8. Wait for subnet status to be updated after all IPs deleted") + time.Sleep(5 * time.Second) + + ginkgo.By("9. Verify subnet status after multiple IP CR deletions") + afterDeleteSubnet := subnetClient.Get(subnetName) + switch afterDeleteSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after all IPs deleted") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after all IPs deleted") + case apiv1.ProtocolIPv6: + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after all IPs deleted") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after all IPs deleted") + default: + // Dual stack + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after all IPs deleted") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after all IPs deleted") + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after all IPs deleted") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after all IPs deleted") + } + + ginkgo.By("10. Test completed: Multiple IP CRs lifecycle properly updates subnet status") + }) + + framework.ConformanceIt("Test IP finalizer prevents premature deletion", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + ginkgo.By("1. Create a pod to trigger IP CR creation") + podName := "test-ip-finalizer-pod-" + framework.RandomSuffix() + cmd := []string{"sleep", "infinity"} + pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) + _ = podClient.CreateSync(pod) + + ginkgo.By("2. Wait for IP CR to be created with finalizer") + ipName := fmt.Sprintf("%s.%s", podName, namespaceName) + var ipCR *apiv1.IP + for range 60 { + ipCR = ipClient.Get(ipName) + if ipCR != nil && ipCR.Name != "" && len(ipCR.Finalizers) > 0 { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectNotNil(ipCR, "IP CR should be created for pod %s", podName) + framework.ExpectContainElement(ipCR.Finalizers, util.KubeOVNControllerFinalizer, + "IP CR should have finalizer") + + ginkgo.By("3. Verify IP CR details") + framework.ExpectEqual(ipCR.Spec.PodName, podName, + "IP CR should have correct pod name") + framework.ExpectEqual(ipCR.Spec.Namespace, namespaceName, + "IP CR should have correct namespace") + framework.ExpectEqual(ipCR.Spec.Subnet, subnetName, + "IP CR should reference correct subnet") + framework.ExpectNotEmpty(ipCR.Spec.V4IPAddress, "IP CR should have V4 or V6 address assigned") + + ginkgo.By("4. Delete the pod") + podClient.DeleteSync(podName) + + ginkgo.By("5. Verify IP CR is eventually deleted") + deleted := false + for range 30 { + _, err := f.KubeOVNClientSet.KubeovnV1().IPs().Get(context.Background(), ipName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + deleted = true + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectTrue(deleted, "IP CR should be deleted after pod deletion") + + ginkgo.By("6. Test completed: IP CR finalizer works correctly") + }) +}) + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + e2e.RunE2ETests(t) +} diff --git a/test/e2e/iptables-eip-qos/e2e_test.go b/test/e2e/iptables-eip-qos/e2e_test.go new file mode 100644 index 00000000000..b1cb5341106 --- /dev/null +++ b/test/e2e/iptables-eip-qos/e2e_test.go @@ -0,0 +1,989 @@ +package ovn_eip + +import ( + "context" + "errors" + "flag" + "fmt" + "strconv" + "strings" + "testing" + "time" + + dockernetwork "github.com/moby/moby/api/types/network" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + "k8s.io/kubernetes/test/e2e" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + + "github.com/onsi/ginkgo/v2" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/test/e2e/framework" + "github.com/kubeovn/kube-ovn/test/e2e/framework/docker" + "github.com/kubeovn/kube-ovn/test/e2e/framework/kind" +) + +const ( + dockerExtNet1Name = "kube-ovn-ext-net1" + dockerExtNet2Name = "kube-ovn-ext-net2" + vpcNatGWConfigMapName = "ovn-vpc-nat-gw-config" + vpcNatConfigName = "ovn-vpc-nat-config" + networkAttachDefName = "ovn-vpc-external-network" + externalSubnetProvider = "ovn-vpc-external-network.kube-system" +) + +const ( + iperf2Port = "20288" + skipIperf = false +) + +const ( + eipLimit = iota*5 + 10 + updatedEIPLimit + newEIPLimit + specificIPLimit + defaultNicLimit +) + +func setupNetworkAttachmentDefinition( + f *framework.Framework, + dockerExtNetNetwork *dockernetwork.Inspect, + attachNetClient *framework.NetworkAttachmentDefinitionClient, + subnetClient *framework.SubnetClient, + externalNetworkName string, + nicName string, + provider string, + dockerExtNetName string, +) { + ginkgo.GinkgoHelper() + + ginkgo.By("Getting docker network " + dockerExtNetName) + network, err := docker.NetworkInspect(dockerExtNetName) + framework.ExpectNoError(err, "getting docker network "+dockerExtNetName) + ginkgo.By("Getting or creating network attachment definition " + externalNetworkName) + attachConf := fmt.Sprintf(`{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "%s", + "mode": "bridge", + "ipam": { + "type": "kube-ovn", + "server_socket": "/run/openvswitch/kube-ovn-daemon.sock", + "provider": "%s" + } + }`, nicName, provider) + + // Try to get existing NAD first using raw Kubernetes API to avoid ExpectNoError + nad, err := attachNetClient.NetworkAttachmentDefinitionInterface.Get(context.TODO(), externalNetworkName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // NAD doesn't exist, create it + attachNet := framework.MakeNetworkAttachmentDefinition(externalNetworkName, framework.KubeOvnNamespace, attachConf) + nad = attachNetClient.Create(attachNet) + } else { + framework.ExpectNoError(err, "getting network attachment definition "+externalNetworkName) + } + + ginkgo.By("Got network attachment definition " + nad.Name) + + ginkgo.By("Creating underlay macvlan subnet " + externalNetworkName) + var cidrV4, cidrV6, gatewayV4, gatewayV6 string + for _, config := range dockerExtNetNetwork.IPAM.Config { + switch util.CheckProtocol(config.Subnet.Addr().String()) { + case apiv1.ProtocolIPv4: + if f.HasIPv4() { + cidrV4 = config.Subnet.String() + gatewayV4 = config.Gateway.String() + } + case apiv1.ProtocolIPv6: + if f.HasIPv6() { + cidrV6 = config.Subnet.String() + gatewayV6 = config.Gateway.String() + } + } + } + cidr := make([]string, 0, 2) + gateway := make([]string, 0, 2) + if f.HasIPv4() { + cidr = append(cidr, cidrV4) + gateway = append(gateway, gatewayV4) + } + if f.HasIPv6() { + cidr = append(cidr, cidrV6) + gateway = append(gateway, gatewayV6) + } + excludeIPs := make([]string, 0, len(network.Containers)*2) + for _, container := range network.Containers { + if container.IPv4Address.IsValid() && f.HasIPv4() { + excludeIPs = append(excludeIPs, container.IPv4Address.Addr().String()) + } + if container.IPv6Address.IsValid() && f.HasIPv6() { + excludeIPs = append(excludeIPs, container.IPv6Address.Addr().String()) + } + } + + // Check if subnet already exists + _, err = subnetClient.SubnetInterface.Get(context.TODO(), externalNetworkName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // Subnet doesn't exist, create it + macvlanSubnet := framework.MakeSubnet(externalNetworkName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", provider, excludeIPs, nil, nil) + _ = subnetClient.CreateSync(macvlanSubnet) + } else { + framework.ExpectNoError(err, "getting subnet "+externalNetworkName) + } +} + +func setupVpcNatGwTestEnvironment( + f *framework.Framework, + dockerExtNetNetwork *dockernetwork.Inspect, + attachNetClient *framework.NetworkAttachmentDefinitionClient, + subnetClient *framework.SubnetClient, + vpcClient *framework.VpcClient, + vpcNatGwClient *framework.VpcNatGatewayClient, + vpcName string, + overlaySubnetName string, + vpcNatGwName string, + natGwQosPolicy string, + overlaySubnetV4Cidr string, + overlaySubnetV4Gw string, + lanIP string, + dockerExtNetName string, + externalNetworkName string, + nicName string, + provider string, + skipNADSetup bool, +) { + ginkgo.GinkgoHelper() + + if !skipNADSetup { + setupNetworkAttachmentDefinition( + f, dockerExtNetNetwork, attachNetClient, + subnetClient, externalNetworkName, nicName, provider, dockerExtNetName) + } + + ginkgo.By("Getting config map " + vpcNatGWConfigMapName) + _, err := f.ClientSet.CoreV1().ConfigMaps(framework.KubeOvnNamespace).Get(context.Background(), vpcNatGWConfigMapName, metav1.GetOptions{}) + framework.ExpectNoError(err, "failed to get ConfigMap") + + ginkgo.By("Creating custom vpc " + vpcName) + vpc := framework.MakeVpc(vpcName, lanIP, false, false, nil) + _ = vpcClient.CreateSync(vpc) + + ginkgo.By("Creating custom overlay subnet " + overlaySubnetName) + overlaySubnet := framework.MakeSubnet(overlaySubnetName, "", overlaySubnetV4Cidr, overlaySubnetV4Gw, vpcName, "", nil, nil, nil) + _ = subnetClient.CreateSync(overlaySubnet) + + ginkgo.By("Creating custom vpc nat gw " + vpcNatGwName) + vpcNatGw := framework.MakeVpcNatGateway(vpcNatGwName, vpcName, overlaySubnetName, lanIP, externalNetworkName, natGwQosPolicy) + _ = vpcNatGwClient.CreateSync(vpcNatGw, f.ClientSet) +} + +type qosParams struct { + vpc1Name string + vpc2Name string + vpc1SubnetName string + vpc2SubnetName string + vpcNat1GwName string + vpcNat2GwName string + vpc1EIPName string + vpc2EIPName string + vpc1FIPName string + vpc2FIPName string + vpc1PodName string + vpc2PodName string + attachDefName string + subnetProvider string +} + +// waitForIptablesEIPReady waits for an IptablesEIP to be ready +func waitForIptablesEIPReady(eipClient *framework.IptablesEIPClient, eipName string, timeout time.Duration) *apiv1.IptablesEIP { + ginkgo.GinkgoHelper() + var eip *apiv1.IptablesEIP + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + eip = eipClient.Get(eipName) + if eip != nil && eip.Status.IP != "" && eip.Status.Ready { + framework.Logf("IptablesEIP %s is ready with IP: %s", eipName, eip.Status.IP) + return eip + } + time.Sleep(2 * time.Second) + } + framework.Failf("Timeout waiting for IptablesEIP %s to be ready", eipName) + return nil +} + +func iperf(f *framework.Framework, iperfClientPod *corev1.Pod, iperfServerEIP *apiv1.IptablesEIP) string { + ginkgo.GinkgoHelper() + + for i := range 20 { + command := fmt.Sprintf("iperf -e -p %s --reportstyle C -i 1 -c %s -t 10", iperf2Port, iperfServerEIP.Status.IP) + stdOutput, errOutput, err := framework.ExecShellInPod(context.Background(), f, iperfClientPod.Namespace, iperfClientPod.Name, command) + framework.Logf("output from exec on client pod %s (eip %s)\n", iperfClientPod.Name, iperfServerEIP.Name) + if stdOutput != "" && err == nil { + framework.Logf("output:\n%s", stdOutput) + return stdOutput + } + framework.Logf("exec %s failed err: %v, errOutput: %s, stdOutput: %s, retried %d times.", command, err, errOutput, stdOutput, i) + time.Sleep(6 * time.Second) + } + framework.ExpectNoError(errors.New("iperf failed")) + return "" +} + +func checkQos(f *framework.Framework, + vpc1Pod, vpc2Pod *corev1.Pod, vpc1EIP, vpc2EIP *apiv1.IptablesEIP, + limit int, expect bool, +) { + ginkgo.GinkgoHelper() + + if !skipIperf { + if expect { + output := iperf(f, vpc1Pod, vpc2EIP) + framework.ExpectTrue(validRateLimit(output, limit)) + output = iperf(f, vpc2Pod, vpc1EIP) + framework.ExpectTrue(validRateLimit(output, limit)) + } else { + output := iperf(f, vpc1Pod, vpc2EIP) + framework.ExpectFalse(validRateLimit(output, limit)) + output = iperf(f, vpc2Pod, vpc1EIP) + framework.ExpectFalse(validRateLimit(output, limit)) + } + } +} + +func getNicDefaultQoSPolicy(limit int) apiv1.QoSPolicyBandwidthLimitRules { + return apiv1.QoSPolicyBandwidthLimitRules{ + apiv1.QoSPolicyBandwidthLimitRule{ + Name: "net1-ingress", + Interface: "net1", + RateMax: strconv.Itoa(limit), + BurstMax: strconv.Itoa(limit), + Priority: 3, + Direction: apiv1.QoSDirectionIngress, + }, + apiv1.QoSPolicyBandwidthLimitRule{ + Name: "net1-egress", + Interface: "net1", + RateMax: strconv.Itoa(limit), + BurstMax: strconv.Itoa(limit), + Priority: 3, + Direction: apiv1.QoSDirectionEgress, + }, + } +} + +func getEIPQoSRule(limit int) apiv1.QoSPolicyBandwidthLimitRules { + return apiv1.QoSPolicyBandwidthLimitRules{ + apiv1.QoSPolicyBandwidthLimitRule{ + Name: "eip-ingress", + RateMax: strconv.Itoa(limit), + BurstMax: strconv.Itoa(limit), + Priority: 1, + Direction: apiv1.QoSDirectionIngress, + }, + apiv1.QoSPolicyBandwidthLimitRule{ + Name: "eip-egress", + RateMax: strconv.Itoa(limit), + BurstMax: strconv.Itoa(limit), + Priority: 1, + Direction: apiv1.QoSDirectionEgress, + }, + } +} + +func getSpecialQoSRule(limit int, ip string) apiv1.QoSPolicyBandwidthLimitRules { + return apiv1.QoSPolicyBandwidthLimitRules{ + apiv1.QoSPolicyBandwidthLimitRule{ + Name: "net1-extip-ingress", + Interface: "net1", + RateMax: strconv.Itoa(limit), + BurstMax: strconv.Itoa(limit), + Priority: 2, + Direction: apiv1.QoSDirectionIngress, + MatchType: apiv1.QoSMatchTypeIP, + MatchValue: "src " + ip + "/32", + }, + apiv1.QoSPolicyBandwidthLimitRule{ + Name: "net1-extip-egress", + Interface: "net1", + RateMax: strconv.Itoa(limit), + BurstMax: strconv.Itoa(limit), + Priority: 2, + Direction: apiv1.QoSDirectionEgress, + MatchType: apiv1.QoSMatchTypeIP, + MatchValue: "dst " + ip + "/32", + }, + } +} + +// defaultQoSCases test default qos policy= +func defaultQoSCases(f *framework.Framework, + vpcNatGwClient *framework.VpcNatGatewayClient, + podClient *framework.PodClient, + qosPolicyClient *framework.QoSPolicyClient, + vpc1Pod *corev1.Pod, + vpc2Pod *corev1.Pod, + vpc1EIP *apiv1.IptablesEIP, + vpc2EIP *apiv1.IptablesEIP, + natgwName string, +) { + ginkgo.GinkgoHelper() + + // create nic qos policy + qosPolicyName := "default-nic-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + qosPolicyName) + rules := getNicDefaultQoSPolicy(defaultNicLimit) + + qosPolicy := framework.MakeQoSPolicy(qosPolicyName, true, apiv1.QoSBindingTypeNatGw, rules) + _ = qosPolicyClient.CreateSync(qosPolicy) + + ginkgo.By("Patch natgw " + natgwName + " with qos policy " + qosPolicyName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, qosPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) + + ginkgo.By("Delete natgw pod " + natgwName + "-0") + natGwPodName := util.GenNatGwPodName(natgwName) + podClient.DeleteSync(natGwPodName) + + ginkgo.By("Wait for natgw " + natgwName + "qos rebuild") + time.Sleep(5 * time.Second) + + ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) + + ginkgo.By("Remove qos policy " + qosPolicyName + " from natgw " + natgwName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") + + ginkgo.By("Deleting qos policy " + qosPolicyName) + qosPolicyClient.DeleteSync(qosPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is not limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, false) +} + +// eipQoSCases test default qos policy +func eipQoSCases(f *framework.Framework, + eipClient *framework.IptablesEIPClient, + podClient *framework.PodClient, + qosPolicyClient *framework.QoSPolicyClient, + vpc1Pod *corev1.Pod, + vpc2Pod *corev1.Pod, + vpc1EIP *apiv1.IptablesEIP, + vpc2EIP *apiv1.IptablesEIP, + eipName string, + natgwName string, +) { + ginkgo.GinkgoHelper() + + // create eip qos policy + qosPolicyName := "eip-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + qosPolicyName) + rules := getEIPQoSRule(eipLimit) + + qosPolicy := framework.MakeQoSPolicy(qosPolicyName, false, apiv1.QoSBindingTypeEIP, rules) + qosPolicy = qosPolicyClient.CreateSync(qosPolicy) + + ginkgo.By("Patch eip " + eipName + " with qos policy " + qosPolicyName) + _ = eipClient.PatchQoSPolicySync(eipName, qosPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(eipLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, eipLimit, true) + + ginkgo.By("Update qos policy " + qosPolicyName + " with new rate limit") + + rules = getEIPQoSRule(updatedEIPLimit) + modifiedqosPolicy := qosPolicy.DeepCopy() + modifiedqosPolicy.Spec.BandwidthLimitRules = rules + qosPolicyClient.Patch(qosPolicy, modifiedqosPolicy) + qosPolicyClient.WaitToQoSReady(qosPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is changed to " + strconv.Itoa(updatedEIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, updatedEIPLimit, true) + + ginkgo.By("Delete natgw pod " + natgwName + "-0") + natGwPodName := util.GenNatGwPodName(natgwName) + podClient.DeleteSync(natGwPodName) + + ginkgo.By("Wait for natgw " + natgwName + "qos rebuid") + time.Sleep(5 * time.Second) + + ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(updatedEIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, updatedEIPLimit, true) + + newQoSPolicyName := "new-eip-qos-policy-" + framework.RandomSuffix() + newRules := getEIPQoSRule(newEIPLimit) + newQoSPolicy := framework.MakeQoSPolicy(newQoSPolicyName, false, apiv1.QoSBindingTypeEIP, newRules) + _ = qosPolicyClient.CreateSync(newQoSPolicy) + + ginkgo.By("Change qos policy of eip " + eipName + " to " + newQoSPolicyName) + _ = eipClient.PatchQoSPolicySync(eipName, newQoSPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(newEIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, newEIPLimit, true) + + ginkgo.By("Remove qos policy " + qosPolicyName + " from natgw " + eipName) + _ = eipClient.PatchQoSPolicySync(eipName, "") + + ginkgo.By("Deleting qos policy " + qosPolicyName) + qosPolicyClient.DeleteSync(qosPolicyName) + + ginkgo.By("Deleting qos policy " + newQoSPolicyName) + qosPolicyClient.DeleteSync(newQoSPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is not limited to " + strconv.Itoa(newEIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, newEIPLimit, false) +} + +// specifyingIPQoSCases test default qos policy +func specifyingIPQoSCases(f *framework.Framework, + vpcNatGwClient *framework.VpcNatGatewayClient, + qosPolicyClient *framework.QoSPolicyClient, + vpc1Pod *corev1.Pod, + vpc2Pod *corev1.Pod, + vpc1EIP *apiv1.IptablesEIP, + vpc2EIP *apiv1.IptablesEIP, + natgwName string, +) { + ginkgo.GinkgoHelper() + + // create nic qos policy + qosPolicyName := "specifying-ip-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + qosPolicyName) + + rules := getSpecialQoSRule(specificIPLimit, vpc2EIP.Status.IP) + + qosPolicy := framework.MakeQoSPolicy(qosPolicyName, true, apiv1.QoSBindingTypeNatGw, rules) + _ = qosPolicyClient.CreateSync(qosPolicy) + + ginkgo.By("Patch natgw " + natgwName + " with qos policy " + qosPolicyName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, qosPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(specificIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, specificIPLimit, true) + + ginkgo.By("Remove qos policy " + qosPolicyName + " from natgw " + natgwName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") + + ginkgo.By("Deleting qos policy " + qosPolicyName) + qosPolicyClient.DeleteSync(qosPolicyName) + + ginkgo.By("Check qos " + qosPolicyName + " is not limited to " + strconv.Itoa(specificIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, specificIPLimit, false) +} + +// priorityQoSCases test qos match priority +func priorityQoSCases(f *framework.Framework, + vpcNatGwClient *framework.VpcNatGatewayClient, + eipClient *framework.IptablesEIPClient, + qosPolicyClient *framework.QoSPolicyClient, + vpc1Pod *corev1.Pod, + vpc2Pod *corev1.Pod, + vpc1EIP *apiv1.IptablesEIP, + vpc2EIP *apiv1.IptablesEIP, + natgwName string, + eipName string, +) { + ginkgo.GinkgoHelper() + + // create nic qos policy + natGwQoSPolicyName := "priority-nic-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + natGwQoSPolicyName) + // default qos policy + special qos policy + natgwRules := getNicDefaultQoSPolicy(defaultNicLimit) + natgwRules = append(natgwRules, getSpecialQoSRule(specificIPLimit, vpc2EIP.Status.IP)...) + + natgwQoSPolicy := framework.MakeQoSPolicy(natGwQoSPolicyName, true, apiv1.QoSBindingTypeNatGw, natgwRules) + _ = qosPolicyClient.CreateSync(natgwQoSPolicy) + + ginkgo.By("Patch natgw " + natgwName + " with qos policy " + natGwQoSPolicyName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, natGwQoSPolicyName) + + eipQoSPolicyName := "eip-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + eipQoSPolicyName) + eipRules := getEIPQoSRule(eipLimit) + + eipQoSPolicy := framework.MakeQoSPolicy(eipQoSPolicyName, false, apiv1.QoSBindingTypeEIP, eipRules) + _ = qosPolicyClient.CreateSync(eipQoSPolicy) + + ginkgo.By("Patch eip " + eipName + " with qos policy " + eipQoSPolicyName) + _ = eipClient.PatchQoSPolicySync(eipName, eipQoSPolicyName) + + // match qos of priority 1 + ginkgo.By("Check qos to match priority 1 is limited to " + strconv.Itoa(eipLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, eipLimit, true) + + ginkgo.By("Remove qos policy " + eipQoSPolicyName + " from natgw " + eipName) + _ = eipClient.PatchQoSPolicySync(eipName, "") + + ginkgo.By("Deleting qos policy " + eipQoSPolicyName) + qosPolicyClient.DeleteSync(eipQoSPolicyName) + + // match qos of priority 2 + ginkgo.By("Check qos to match priority 2 is limited to " + strconv.Itoa(specificIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, specificIPLimit, true) + + // change qos policy of natgw + newNatGwQoSPolicyName := "new-priority-nic-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + newNatGwQoSPolicyName) + newNatgwRules := getNicDefaultQoSPolicy(defaultNicLimit) + + newNatgwQoSPolicy := framework.MakeQoSPolicy(newNatGwQoSPolicyName, true, apiv1.QoSBindingTypeNatGw, newNatgwRules) + _ = qosPolicyClient.CreateSync(newNatgwQoSPolicy) + + ginkgo.By("Change qos policy of natgw " + natgwName + " to " + newNatGwQoSPolicyName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, newNatGwQoSPolicyName) + + // match qos of priority 3 + ginkgo.By("Check qos to match priority 3 is limited to " + strconv.Itoa(specificIPLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) + + ginkgo.By("Remove qos policy " + natGwQoSPolicyName + " from natgw " + natgwName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") + + ginkgo.By("Deleting qos policy " + natGwQoSPolicyName) + qosPolicyClient.DeleteSync(natGwQoSPolicyName) + + ginkgo.By("Deleting qos policy " + newNatGwQoSPolicyName) + qosPolicyClient.DeleteSync(newNatGwQoSPolicyName) + + ginkgo.By("Check qos " + natGwQoSPolicyName + " is not limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, false) +} + +func createNatGwAndSetQosCases(f *framework.Framework, + vpcNatGwClient *framework.VpcNatGatewayClient, + ipClient *framework.IPClient, + eipClient *framework.IptablesEIPClient, + fipClient *framework.IptablesFIPClient, + subnetClient *framework.SubnetClient, + qosPolicyClient *framework.QoSPolicyClient, + vpc1Pod *corev1.Pod, + vpc2Pod *corev1.Pod, + vpc2EIP *apiv1.IptablesEIP, + natgwName string, + eipName string, + fipName string, + vpcName string, + overlaySubnetName string, + lanIP string, + attachDefName string, +) { + ginkgo.GinkgoHelper() + + // delete fip + ginkgo.By("Deleting fip " + fipName) + fipClient.DeleteSync(fipName) + + ginkgo.By("Deleting eip " + eipName) + eipClient.DeleteSync(eipName) + + // the only pod for vpc nat gateway + vpcNatGw1PodName := util.GenNatGwPodName(natgwName) + + // delete vpc nat gw statefulset remaining ip for eth0 and net2 + ginkgo.By("Deleting custom vpc nat gw " + natgwName) + vpcNatGwClient.DeleteSync(natgwName) + + overlaySubnet1 := subnetClient.Get(overlaySubnetName) + macvlanSubnet := subnetClient.Get(attachDefName) + eth0IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, overlaySubnet1.Spec.Provider) + net1IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, macvlanSubnet.Spec.Provider) + ginkgo.By("Deleting vpc nat gw eth0 ip " + eth0IpName) + ipClient.DeleteSync(eth0IpName) + ginkgo.By("Deleting vpc nat gw net1 ip " + net1IpName) + ipClient.DeleteSync(net1IpName) + + natgwQoSPolicyName := "default-nic-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + natgwQoSPolicyName) + rules := getNicDefaultQoSPolicy(defaultNicLimit) + + qosPolicy := framework.MakeQoSPolicy(natgwQoSPolicyName, true, apiv1.QoSBindingTypeNatGw, rules) + _ = qosPolicyClient.CreateSync(qosPolicy) + + ginkgo.By("Creating custom vpc nat gw") + vpcNatGw := framework.MakeVpcNatGateway(natgwName, vpcName, overlaySubnetName, lanIP, attachDefName, natgwQoSPolicyName) + _ = vpcNatGwClient.CreateSync(vpcNatGw, f.ClientSet) + + eipQoSPolicyName := "eip-qos-policy-" + framework.RandomSuffix() + ginkgo.By("Creating qos policy " + eipQoSPolicyName) + rules = getEIPQoSRule(eipLimit) + + eipQoSPolicy := framework.MakeQoSPolicy(eipQoSPolicyName, false, apiv1.QoSBindingTypeEIP, rules) + _ = qosPolicyClient.CreateSync(eipQoSPolicy) + + ginkgo.By("Creating eip " + eipName) + vpc1EIP := framework.MakeIptablesEIP(eipName, "", "", "", natgwName, attachDefName, eipQoSPolicyName) + _ = eipClient.CreateSync(vpc1EIP) + vpc1EIP = waitForIptablesEIPReady(eipClient, eipName, 60*time.Second) + + ginkgo.By("Creating fip " + fipName) + fip := framework.MakeIptablesFIPRule(fipName, eipName, vpc1Pod.Status.PodIP) + _ = fipClient.CreateSync(fip) + + ginkgo.By("Check qos " + eipQoSPolicyName + " is limited to " + strconv.Itoa(eipLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, eipLimit, true) + + ginkgo.By("Remove qos policy " + eipQoSPolicyName + " from natgw " + natgwName) + _ = eipClient.PatchQoSPolicySync(eipName, "") + + ginkgo.By("Check qos " + natgwQoSPolicyName + " is limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) + + ginkgo.By("Remove qos policy " + natgwQoSPolicyName + " from natgw " + natgwName) + _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") + + ginkgo.By("Check qos " + natgwQoSPolicyName + " is not limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") + checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, false) + + ginkgo.By("Deleting qos policy " + natgwQoSPolicyName) + qosPolicyClient.DeleteSync(natgwQoSPolicyName) + + ginkgo.By("Deleting qos policy " + eipQoSPolicyName) + qosPolicyClient.DeleteSync(eipQoSPolicyName) +} + +func validRateLimit(text string, limit int) bool { + maxValue := float64(limit) * 1024 * 1024 * 1.2 + minValue := float64(limit) * 1024 * 1024 * 0.8 + lines := strings.SplitSeq(text, "\n") + for line := range lines { + if line == "" { + continue + } + fields := strings.Split(line, ",") + number, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + continue + } + if v := float64(number); v >= minValue && v <= maxValue { + return true + } + } + return false +} + +var _ = framework.Describe("[group:qos-policy]", func() { + f := framework.NewDefaultFramework("qos-policy") + + var skip bool + var cs clientset.Interface + var attachNetClient *framework.NetworkAttachmentDefinitionClient + var clusterName string + var vpcClient *framework.VpcClient + var vpcNatGwClient *framework.VpcNatGatewayClient + var subnetClient *framework.SubnetClient + var podClient *framework.PodClient + var ipClient *framework.IPClient + var iptablesEIPClient *framework.IptablesEIPClient + var iptablesFIPClient *framework.IptablesFIPClient + var qosPolicyClient *framework.QoSPolicyClient + + var net1NicName string + var dockerExtNetName string + + // docker network + var dockerExtNetNetwork *dockernetwork.Inspect + + var vpcQosParams *qosParams + var vpc1Pod *corev1.Pod + var vpc2Pod *corev1.Pod + var vpc1EIP *apiv1.IptablesEIP + var vpc2EIP *apiv1.IptablesEIP + var vpc1FIP *apiv1.IptablesFIPRule + var vpc2FIP *apiv1.IptablesFIPRule + + var lanIP string + var overlaySubnetV4Cidr string + var overlaySubnetV4Gw string + var eth0Exist, net1Exist bool + var annotations1 map[string]string + var annotations2 map[string]string + var iperfServerCmd []string + + ginkgo.BeforeEach(func() { + randomSuffix := framework.RandomSuffix() + vpcQosParams = &qosParams{ + vpc1Name: "qos-vpc1-" + randomSuffix, + vpc2Name: "qos-vpc2-" + randomSuffix, + vpc1SubnetName: "qos-vpc1-subnet-" + randomSuffix, + vpc2SubnetName: "qos-vpc2-subnet-" + randomSuffix, + vpcNat1GwName: "qos-gw1-" + randomSuffix, + vpcNat2GwName: "qos-gw2-" + randomSuffix, + vpc1EIPName: "qos-vpc1-eip-" + randomSuffix, + vpc2EIPName: "qos-vpc2-eip-" + randomSuffix, + vpc1FIPName: "qos-vpc1-fip-" + randomSuffix, + vpc2FIPName: "qos-vpc2-fip-" + randomSuffix, + vpc1PodName: "qos-vpc1-pod-" + randomSuffix, + vpc2PodName: "qos-vpc2-pod-" + randomSuffix, + attachDefName: "qos-ovn-vpc-external-network-" + randomSuffix, + } + vpcQosParams.subnetProvider = fmt.Sprintf("%s.%s", vpcQosParams.attachDefName, framework.KubeOvnNamespace) + + dockerExtNetName = "kube-ovn-qos-" + randomSuffix + + cs = f.ClientSet + podClient = f.PodClient() + attachNetClient = f.NetworkAttachmentDefinitionClientNS(framework.KubeOvnNamespace) + subnetClient = f.SubnetClient() + vpcClient = f.VpcClient() + vpcNatGwClient = f.VpcNatGatewayClient() + iptablesEIPClient = f.IptablesEIPClient() + ipClient = f.IPClient() + iptablesFIPClient = f.IptablesFIPClient() + qosPolicyClient = f.QoSPolicyClient() + + if skip { + ginkgo.Skip("underlay spec only runs on kind clusters") + } + + if clusterName == "" { + ginkgo.By("Getting k8s nodes") + k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) + framework.ExpectNoError(err) + + cluster, ok := kind.IsKindProvided(k8sNodes.Items[0].Spec.ProviderID) + if !ok { + skip = true + ginkgo.Skip("underlay spec only runs on kind clusters") + } + clusterName = cluster + } + + ginkgo.By("Ensuring docker network " + dockerExtNetName + " exists") + network, err := docker.NetworkCreate(dockerExtNetName, true, true) + framework.ExpectNoError(err, "creating docker network "+dockerExtNetName) + dockerExtNetNetwork = network + + ginkgo.By("Getting kind nodes") + nodes, err := kind.ListNodes(clusterName, "") + framework.ExpectNoError(err, "getting nodes in kind cluster") + framework.ExpectNotEmpty(nodes) + + ginkgo.By("Connecting nodes to the docker network") + err = kind.NetworkConnect(dockerExtNetNetwork.ID, nodes) + framework.ExpectNoError(err, "connecting nodes to network "+dockerExtNetName) + + ginkgo.By("Getting node links that belong to the docker network") + nodes, err = kind.ListNodes(clusterName, "") + framework.ExpectNoError(err, "getting nodes in kind cluster") + + ginkgo.By("Validating node links") + network1, err := docker.NetworkInspect(dockerExtNetName) + framework.ExpectNoError(err) + for _, node := range nodes { + links, err := node.ListLinks() + framework.ExpectNoError(err, "failed to list links on node %s: %v", node.Name(), err) + net1Mac := network1.Containers[node.ID].MacAddress + for _, link := range links { + ginkgo.By("exist node nic " + link.IfName) + if link.IfName == "eth0" { + eth0Exist = true + } + if link.Address == net1Mac.String() { + net1NicName = link.IfName + net1Exist = true + } + } + framework.ExpectTrue(eth0Exist) + framework.ExpectTrue(net1Exist) + } + setupNetworkAttachmentDefinition( + f, dockerExtNetNetwork, attachNetClient, + subnetClient, vpcQosParams.attachDefName, net1NicName, vpcQosParams.subnetProvider, dockerExtNetName) + }) + + ginkgo.AfterEach(func() { + ginkgo.By("Deleting macvlan underlay subnet " + vpcQosParams.attachDefName) + subnetClient.DeleteSync(vpcQosParams.attachDefName) + + // delete net1 attachment definition + ginkgo.By("Deleting nad " + vpcQosParams.attachDefName) + attachNetClient.Delete(vpcQosParams.attachDefName) + + ginkgo.By("Getting nodes") + nodes, err := kind.ListNodes(clusterName, "") + framework.ExpectNoError(err, "getting nodes in cluster") + + if dockerExtNetNetwork != nil { + ginkgo.By("Disconnecting nodes from the docker network") + err = kind.NetworkDisconnect(dockerExtNetNetwork.ID, nodes) + framework.ExpectNoError(err, "disconnecting nodes from network "+dockerExtNetName) + ginkgo.By("Deleting docker network " + dockerExtNetName + " exists") + err := docker.NetworkRemove(dockerExtNetNetwork.ID) + framework.ExpectNoError(err, "deleting docker network "+dockerExtNetName) + } + }) + + _ = framework.Describe("vpc qos", func() { + ginkgo.BeforeEach(func() { + iperfServerCmd = []string{"iperf", "-s", "-i", "1", "-p", iperf2Port} + overlaySubnetV4Cidr = "10.0.0.0/24" + overlaySubnetV4Gw = "10.0.0.1" + lanIP = "10.0.0.254" + natgwQoS := "" + setupVpcNatGwTestEnvironment( + f, dockerExtNetNetwork, attachNetClient, + subnetClient, vpcClient, vpcNatGwClient, + vpcQosParams.vpc1Name, vpcQosParams.vpc1SubnetName, vpcQosParams.vpcNat1GwName, + natgwQoS, overlaySubnetV4Cidr, overlaySubnetV4Gw, lanIP, + dockerExtNetName, vpcQosParams.attachDefName, net1NicName, + vpcQosParams.subnetProvider, + true, + ) + annotations1 = map[string]string{ + util.LogicalSwitchAnnotation: vpcQosParams.vpc1SubnetName, + } + ginkgo.By("Creating pod " + vpcQosParams.vpc1PodName) + vpc1Pod = framework.MakePod(f.Namespace.Name, vpcQosParams.vpc1PodName, nil, annotations1, framework.AgnhostImage, iperfServerCmd, nil) + vpc1Pod = podClient.CreateSync(vpc1Pod) + + ginkgo.By("Creating eip " + vpcQosParams.vpc1EIPName) + vpc1EIP = framework.MakeIptablesEIP(vpcQosParams.vpc1EIPName, "", "", "", vpcQosParams.vpcNat1GwName, vpcQosParams.attachDefName, "") + _ = iptablesEIPClient.CreateSync(vpc1EIP) + vpc1EIP = waitForIptablesEIPReady(iptablesEIPClient, vpcQosParams.vpc1EIPName, 60*time.Second) + + ginkgo.By("Creating fip " + vpcQosParams.vpc1FIPName) + vpc1FIP = framework.MakeIptablesFIPRule(vpcQosParams.vpc1FIPName, vpcQosParams.vpc1EIPName, vpc1Pod.Status.PodIP) + _ = iptablesFIPClient.CreateSync(vpc1FIP) + + setupVpcNatGwTestEnvironment( + f, dockerExtNetNetwork, attachNetClient, + subnetClient, vpcClient, vpcNatGwClient, + vpcQosParams.vpc2Name, vpcQosParams.vpc2SubnetName, vpcQosParams.vpcNat2GwName, + natgwQoS, overlaySubnetV4Cidr, overlaySubnetV4Gw, lanIP, + dockerExtNetName, vpcQosParams.attachDefName, net1NicName, + vpcQosParams.subnetProvider, + true, + ) + + annotations2 = map[string]string{ + util.LogicalSwitchAnnotation: vpcQosParams.vpc2SubnetName, + } + + ginkgo.By("Creating pod " + vpcQosParams.vpc2PodName) + vpc2Pod = framework.MakePod(f.Namespace.Name, vpcQosParams.vpc2PodName, nil, annotations2, framework.AgnhostImage, iperfServerCmd, nil) + vpc2Pod = podClient.CreateSync(vpc2Pod) + + ginkgo.By("Creating eip " + vpcQosParams.vpc2EIPName) + vpc2EIP = framework.MakeIptablesEIP(vpcQosParams.vpc2EIPName, "", "", "", vpcQosParams.vpcNat2GwName, vpcQosParams.attachDefName, "") + _ = iptablesEIPClient.CreateSync(vpc2EIP) + vpc2EIP = waitForIptablesEIPReady(iptablesEIPClient, vpcQosParams.vpc2EIPName, 60*time.Second) + + ginkgo.By("Creating fip " + vpcQosParams.vpc2FIPName) + vpc2FIP = framework.MakeIptablesFIPRule(vpcQosParams.vpc2FIPName, vpcQosParams.vpc2EIPName, vpc2Pod.Status.PodIP) + _ = iptablesFIPClient.CreateSync(vpc2FIP) + }) + ginkgo.AfterEach(func() { + ginkgo.By("Deleting fip " + vpcQosParams.vpc1FIPName) + iptablesFIPClient.DeleteSync(vpcQosParams.vpc1FIPName) + + ginkgo.By("Deleting fip " + vpcQosParams.vpc2FIPName) + iptablesFIPClient.DeleteSync(vpcQosParams.vpc2FIPName) + + ginkgo.By("Deleting eip " + vpcQosParams.vpc1EIPName) + iptablesEIPClient.DeleteSync(vpcQosParams.vpc1EIPName) + + ginkgo.By("Deleting eip " + vpcQosParams.vpc2EIPName) + iptablesEIPClient.DeleteSync(vpcQosParams.vpc2EIPName) + + ginkgo.By("Deleting pod " + vpcQosParams.vpc1PodName) + podClient.DeleteSync(vpcQosParams.vpc1PodName) + + ginkgo.By("Deleting pod " + vpcQosParams.vpc2PodName) + podClient.DeleteSync(vpcQosParams.vpc2PodName) + + ginkgo.By("Deleting custom vpc nat gw " + vpcQosParams.vpcNat1GwName) + vpcNatGwClient.DeleteSync(vpcQosParams.vpcNat1GwName) + + ginkgo.By("Deleting custom vpc nat gw " + vpcQosParams.vpcNat2GwName) + vpcNatGwClient.DeleteSync(vpcQosParams.vpcNat2GwName) + + // the only pod for vpc nat gateway + vpcNatGw1PodName := util.GenNatGwPodName(vpcQosParams.vpcNat1GwName) + + // delete vpc nat gw statefulset remaining ip for eth0 and net2 + overlaySubnet1 := subnetClient.Get(vpcQosParams.vpc1SubnetName) + macvlanSubnet := subnetClient.Get(vpcQosParams.attachDefName) + eth0IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, overlaySubnet1.Spec.Provider) + net1IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, macvlanSubnet.Spec.Provider) + ginkgo.By("Deleting vpc nat gw eth0 ip " + eth0IpName) + ipClient.DeleteSync(eth0IpName) + ginkgo.By("Deleting vpc nat gw net1 ip " + net1IpName) + ipClient.DeleteSync(net1IpName) + ginkgo.By("Deleting overlay subnet " + vpcQosParams.vpc1SubnetName) + subnetClient.DeleteSync(vpcQosParams.vpc1SubnetName) + + ginkgo.By("Getting overlay subnet " + vpcQosParams.vpc2SubnetName) + overlaySubnet2 := subnetClient.Get(vpcQosParams.vpc2SubnetName) + + vpcNatGw2PodName := util.GenNatGwPodName(vpcQosParams.vpcNat2GwName) + eth0IpName = ovs.PodNameToPortName(vpcNatGw2PodName, framework.KubeOvnNamespace, overlaySubnet2.Spec.Provider) + net1IpName = ovs.PodNameToPortName(vpcNatGw2PodName, framework.KubeOvnNamespace, macvlanSubnet.Spec.Provider) + ginkgo.By("Deleting vpc nat gw eth0 ip " + eth0IpName) + ipClient.DeleteSync(eth0IpName) + ginkgo.By("Deleting vpc nat gw net1 ip " + net1IpName) + ipClient.DeleteSync(net1IpName) + ginkgo.By("Deleting overlay subnet " + vpcQosParams.vpc2SubnetName) + subnetClient.DeleteSync(vpcQosParams.vpc2SubnetName) + + ginkgo.By("Deleting custom vpc " + vpcQosParams.vpc1Name) + vpcClient.DeleteSync(vpcQosParams.vpc1Name) + + ginkgo.By("Deleting custom vpc " + vpcQosParams.vpc2Name) + vpcClient.DeleteSync(vpcQosParams.vpc2Name) + }) + framework.ConformanceIt("default nic qos", func() { + // case 1: set qos policy for natgw + // case 2: rebuild qos when natgw pod restart + defaultQoSCases(f, vpcNatGwClient, podClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpcNat1GwName) + }) + framework.ConformanceIt("eip qos", func() { + // case 1: set qos policy for eip + // case 2: update qos policy for eip + // case 3: change qos policy of eip + // case 4: rebuild qos when natgw pod restart + eipQoSCases(f, iptablesEIPClient, podClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpc1EIPName, vpcQosParams.vpcNat1GwName) + }) + framework.ConformanceIt("specifying ip qos", func() { + // case 1: set specific ip qos policy for natgw + specifyingIPQoSCases(f, vpcNatGwClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpcNat1GwName) + }) + framework.ConformanceIt("qos priority matching", func() { + // case 1: test qos match priority + // case 2: change qos policy of natgw + priorityQoSCases(f, vpcNatGwClient, iptablesEIPClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpcNat1GwName, vpcQosParams.vpc1EIPName) + }) + framework.ConformanceIt("create resource with qos policy", func() { + // case 1: test qos when create natgw with qos policy + // case 2: test qos when create eip with qos policy + createNatGwAndSetQosCases(f, + vpcNatGwClient, ipClient, iptablesEIPClient, iptablesFIPClient, + subnetClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc2EIP, vpcQosParams.vpcNat1GwName, + vpcQosParams.vpc1EIPName, vpcQosParams.vpc1FIPName, vpcQosParams.vpc1Name, + vpcQosParams.vpc1SubnetName, lanIP, vpcQosParams.attachDefName) + }) + }) +}) + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + e2e.RunE2ETests(t) +} diff --git a/test/e2e/iptables-vpc-nat-gw/e2e_test.go b/test/e2e/iptables-vpc-nat-gw/e2e_test.go index 33c2f336613..2668da96fef 100644 --- a/test/e2e/iptables-vpc-nat-gw/e2e_test.go +++ b/test/e2e/iptables-vpc-nat-gw/e2e_test.go @@ -2,16 +2,14 @@ package ovn_eip import ( "context" - "errors" "flag" "fmt" - "strconv" "strings" "testing" "time" dockernetwork "github.com/moby/moby/api/types/network" - corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -39,36 +37,6 @@ const ( externalSubnetProvider = "ovn-vpc-external-network.kube-system" ) -const ( - iperf2Port = "20288" - skipIperf = false -) - -const ( - eipLimit = iota*5 + 10 - updatedEIPLimit - newEIPLimit - specificIPLimit - defaultNicLimit -) - -type qosParams struct { - vpc1Name string - vpc2Name string - vpc1SubnetName string - vpc2SubnetName string - vpcNat1GwName string - vpcNat2GwName string - vpc1EIPName string - vpc2EIPName string - vpc1FIPName string - vpc2FIPName string - vpc1PodName string - vpc2PodName string - attachDefName string - subnetProvider string -} - func setupNetworkAttachmentDefinition( f *framework.Framework, dockerExtNetNetwork *dockernetwork.Inspect, @@ -84,7 +52,7 @@ func setupNetworkAttachmentDefinition( ginkgo.By("Getting docker network " + dockerExtNetName) network, err := docker.NetworkInspect(dockerExtNetName) framework.ExpectNoError(err, "getting docker network "+dockerExtNetName) - ginkgo.By("Getting network attachment definition " + externalNetworkName) + ginkgo.By("Getting or creating network attachment definition " + externalNetworkName) attachConf := fmt.Sprintf(`{ "cniVersion": "0.3.0", "type": "macvlan", @@ -96,9 +64,16 @@ func setupNetworkAttachmentDefinition( "provider": "%s" } }`, nicName, provider) - attachNet := framework.MakeNetworkAttachmentDefinition(externalNetworkName, framework.KubeOvnNamespace, attachConf) - attachNetClient.Create(attachNet) - nad := attachNetClient.Get(externalNetworkName) + + // Try to get existing NAD first + nad, err := attachNetClient.NetworkAttachmentDefinitionInterface.Get(context.TODO(), externalNetworkName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // NAD doesn't exist, create it + attachNet := framework.MakeNetworkAttachmentDefinition(externalNetworkName, framework.KubeOvnNamespace, attachConf) + nad = attachNetClient.Create(attachNet) + } else { + framework.ExpectNoError(err, "getting network attachment definition "+externalNetworkName) + } ginkgo.By("Got network attachment definition " + nad.Name) @@ -137,8 +112,16 @@ func setupNetworkAttachmentDefinition( excludeIPs = append(excludeIPs, container.IPv6Address.Addr().String()) } } - macvlanSubnet := framework.MakeSubnet(externalNetworkName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", provider, excludeIPs, nil, nil) - _ = subnetClient.CreateSync(macvlanSubnet) + + // Check if subnet already exists + _, err = subnetClient.SubnetInterface.Get(context.TODO(), externalNetworkName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // Subnet doesn't exist, create it + macvlanSubnet := framework.MakeSubnet(externalNetworkName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", provider, excludeIPs, nil, nil) + _ = subnetClient.CreateSync(macvlanSubnet) + } else { + framework.ExpectNoError(err, "getting subnet "+externalNetworkName) + } } func setupVpcNatGwTestEnvironment( @@ -207,6 +190,54 @@ func cleanVpcNatGwTestEnvironment( vpcClient.DeleteSync(vpcName) } +// waitForIptablesEIPReady waits for an IptablesEIP to be ready +func waitForIptablesEIPReady(eipClient *framework.IptablesEIPClient, eipName string, timeout time.Duration) *apiv1.IptablesEIP { + ginkgo.GinkgoHelper() + var eip *apiv1.IptablesEIP + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + eip = eipClient.Get(eipName) + if eip != nil && eip.Status.IP != "" && eip.Status.Ready { + framework.Logf("IptablesEIP %s is ready with IP: %s", eipName, eip.Status.IP) + return eip + } + time.Sleep(2 * time.Second) + } + framework.Failf("Timeout waiting for IptablesEIP %s to be ready", eipName) + return nil +} + +// verifySubnetStatusAfterEIPOperation verifies subnet status after EIP operation +func verifySubnetStatusAfterEIPOperation(subnetClient *framework.SubnetClient, subnetName string, + protocol, operation, shouldContainIP string, +) { + ginkgo.GinkgoHelper() + + subnet := subnetClient.Get(subnetName) + framework.Logf("Verifying subnet %s status after %s: Protocol=%s", subnetName, operation, protocol) + + switch protocol { + case apiv1.ProtocolIPv4: + framework.Logf("V4 Status: Available=%.0f, Using=%.0f", + subnet.Status.V4AvailableIPs, subnet.Status.V4UsingIPs) + if shouldContainIP != "" { + framework.ExpectTrue(strings.Contains(subnet.Status.V4UsingIPRange, shouldContainIP), + "IP %s should be in V4UsingIPRange after %s", shouldContainIP, operation) + } + case apiv1.ProtocolIPv6: + framework.Logf("V6 Status: Available=%.0f, Using=%.0f", + subnet.Status.V6AvailableIPs, subnet.Status.V6UsingIPs) + if shouldContainIP != "" { + framework.ExpectTrue(strings.Contains(subnet.Status.V6UsingIPRange, shouldContainIP), + "IP %s should be in V6UsingIPRange after %s", shouldContainIP, operation) + } + case apiv1.ProtocolDual: + framework.Logf("Dual Stack Status: V4Available=%.0f, V4Using=%.0f, V6Available=%.0f, V6Using=%.0f", + subnet.Status.V4AvailableIPs, subnet.Status.V4UsingIPs, + subnet.Status.V6AvailableIPs, subnet.Status.V6UsingIPs) + } +} + var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { f := framework.NewDefaultFramework("iptables-vpc-nat-gw") @@ -240,37 +271,38 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { var net2VpcName string var net2EipName string - vpcName = "vpc-" + framework.RandomSuffix() - vpcNatGwName = "gw-" + framework.RandomSuffix() - - fipVipName = "fip-vip-" + framework.RandomSuffix() - fipEipName = "fip-eip-" + framework.RandomSuffix() - fipName = "fip-" + framework.RandomSuffix() - - dnatVipName = "dnat-vip-" + framework.RandomSuffix() - dnatEipName = "dnat-eip-" + framework.RandomSuffix() - dnatName = "dnat-" + framework.RandomSuffix() - - // sharing case - sharedVipName = "shared-vip-" + framework.RandomSuffix() - sharedEipName = "shared-eip-" + framework.RandomSuffix() - sharedEipDnatName = "shared-eip-dnat-" + framework.RandomSuffix() - sharedEipSnatName = "shared-eip-snat-" + framework.RandomSuffix() - sharedEipFipShouldOkName = "shared-eip-fip-should-ok-" + framework.RandomSuffix() - sharedEipFipShouldFailName = "shared-eip-fip-should-fail-" + framework.RandomSuffix() - - snatEipName = "snat-eip-" + framework.RandomSuffix() - snatName = "snat-" + framework.RandomSuffix() - overlaySubnetName = "overlay-subnet-" + framework.RandomSuffix() - - net2AttachDefName = "net2-ovn-vpc-external-network-" + framework.RandomSuffix() - net2SubnetProvider = fmt.Sprintf("%s.%s", net2AttachDefName, framework.KubeOvnNamespace) - net2OverlaySubnetName = "net2-overlay-subnet-" + framework.RandomSuffix() - net2VpcNatGwName = "net2-gw-" + framework.RandomSuffix() - net2VpcName = "net2-vpc-" + framework.RandomSuffix() - net2EipName = "net2-eip-" + framework.RandomSuffix() - ginkgo.BeforeEach(func() { + randomSuffix := framework.RandomSuffix() + vpcName = "vpc-" + randomSuffix + vpcNatGwName = "gw-" + randomSuffix + + fipVipName = "fip-vip-" + randomSuffix + fipEipName = "fip-eip-" + randomSuffix + fipName = "fip-" + randomSuffix + + dnatVipName = "dnat-vip-" + randomSuffix + dnatEipName = "dnat-eip-" + randomSuffix + dnatName = "dnat-" + randomSuffix + + // sharing case + sharedVipName = "shared-vip-" + randomSuffix + sharedEipName = "shared-eip-" + randomSuffix + sharedEipDnatName = "shared-eip-dnat-" + randomSuffix + sharedEipSnatName = "shared-eip-snat-" + randomSuffix + sharedEipFipShouldOkName = "shared-eip-fip-should-ok-" + randomSuffix + sharedEipFipShouldFailName = "shared-eip-fip-should-fail-" + randomSuffix + + snatEipName = "snat-eip-" + randomSuffix + snatName = "snat-" + randomSuffix + overlaySubnetName = "overlay-subnet-" + randomSuffix + + net2AttachDefName = "net2-ovn-vpc-external-network-" + randomSuffix + net2SubnetProvider = fmt.Sprintf("%s.%s", net2AttachDefName, framework.KubeOvnNamespace) + net2OverlaySubnetName = "net2-overlay-subnet-" + randomSuffix + net2VpcNatGwName = "net2-gw-" + randomSuffix + net2VpcName = "net2-vpc-" + randomSuffix + net2EipName = "net2-eip-" + randomSuffix + cs = f.ClientSet attachNetClient = f.NetworkAttachmentDefinitionClientNS(framework.KubeOvnNamespace) subnetClient = f.SubnetClient() @@ -399,7 +431,7 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { cm, err := f.ClientSet.CoreV1().ConfigMaps(framework.KubeOvnNamespace).Get(context.Background(), vpcNatConfigName, metav1.GetOptions{}) framework.ExpectNoError(err) oldImage := cm.Data["image"] - cm.Data["image"] = "docker.io/kubeovn/vpc-nat-gateway:v1.12.18" + cm.Data["image"] = "docker.io/kubeovn/vpc-nat-gateway:v1.14.19" cm, err = f.ClientSet.CoreV1().ConfigMaps(framework.KubeOvnNamespace).Update(context.Background(), cm, metav1.UpdateOptions{}) framework.ExpectNoError(err) time.Sleep(3 * time.Second) @@ -520,10 +552,10 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { iptablesFIPClient.DeleteSync(sharedEipFipShouldOkName) ginkgo.By("Deleting share iptables fip " + sharedEipFipShouldFailName) iptablesFIPClient.DeleteSync(sharedEipFipShouldFailName) - ginkgo.By("Deleting share iptables dnat " + dnatName) - iptablesDnatRuleClient.DeleteSync(dnatName) - ginkgo.By("Deleting share iptables snat " + snatName) - iptablesSnatRuleClient.DeleteSync(snatName) + ginkgo.By("Deleting share iptables dnat " + sharedEipDnatName) + iptablesDnatRuleClient.DeleteSync(sharedEipDnatName) + ginkgo.By("Deleting share iptables snat " + sharedEipSnatName) + iptablesSnatRuleClient.DeleteSync(sharedEipSnatName) ginkgo.By("Deleting iptables fip " + fipName) iptablesFIPClient.DeleteSync(fipName) @@ -616,779 +648,451 @@ var _ = framework.SerialDescribe("[group:iptables-vpc-nat-gw]", func() { ginkgo.By("Deleting custom vpc " + net2VpcName) vpcClient.DeleteSync(net2VpcName) }) -}) -func iperf(f *framework.Framework, iperfClientPod *corev1.Pod, iperfServerEIP *apiv1.IptablesEIP) string { - ginkgo.GinkgoHelper() - - for i := range 20 { - command := fmt.Sprintf("iperf -e -p %s --reportstyle C -i 1 -c %s -t 10", iperf2Port, iperfServerEIP.Status.IP) - stdOutput, errOutput, err := framework.ExecShellInPod(context.Background(), f, iperfClientPod.Namespace, iperfClientPod.Name, command) - framework.Logf("output from exec on client pod %s (eip %s)\n", iperfClientPod.Name, iperfServerEIP.Name) - if stdOutput != "" && err == nil { - framework.Logf("output:\n%s", stdOutput) - return stdOutput - } - framework.Logf("exec %s failed err: %v, errOutput: %s, stdOutput: %s, retried %d times.", command, err, errOutput, stdOutput, i) - time.Sleep(6 * time.Second) - } - framework.ExpectNoError(errors.New("iperf failed")) - return "" -} + framework.ConformanceIt("should properly manage IptablesEIP lifecycle with finalizer and update subnet status", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") -func checkQos(f *framework.Framework, - vpc1Pod, vpc2Pod *corev1.Pod, vpc1EIP, vpc2EIP *apiv1.IptablesEIP, - limit int, expect bool, -) { - ginkgo.GinkgoHelper() + overlaySubnetV4Cidr := "10.0.3.0/24" + overlaySubnetV4Gw := "10.0.3.1" + lanIP := "10.0.3.254" + natgwQoS := "" + setupVpcNatGwTestEnvironment( + f, dockerExtNet1Network, attachNetClient, + subnetClient, vpcClient, vpcNatGwClient, + vpcName, overlaySubnetName, vpcNatGwName, natgwQoS, + overlaySubnetV4Cidr, overlaySubnetV4Gw, lanIP, + dockerExtNet1Name, networkAttachDefName, net1NicName, + externalSubnetProvider, + false, + ) - if !skipIperf { - if expect { - output := iperf(f, vpc1Pod, vpc2EIP) - framework.ExpectTrue(validRateLimit(output, limit)) - output = iperf(f, vpc2Pod, vpc1EIP) - framework.ExpectTrue(validRateLimit(output, limit)) - } else { - output := iperf(f, vpc1Pod, vpc2EIP) - framework.ExpectFalse(validRateLimit(output, limit)) - output = iperf(f, vpc2Pod, vpc1EIP) - framework.ExpectFalse(validRateLimit(output, limit)) + ginkgo.By("1. Get initial external subnet status") + externalSubnetName := util.GetExternalNetwork(networkAttachDefName) + initialSubnet := subnetClient.Get(externalSubnetName) + initialV4AvailableIPs := initialSubnet.Status.V4AvailableIPs + initialV4UsingIPs := initialSubnet.Status.V4UsingIPs + initialV6AvailableIPs := initialSubnet.Status.V6AvailableIPs + initialV6UsingIPs := initialSubnet.Status.V6UsingIPs + initialV4AvailableIPRange := initialSubnet.Status.V4AvailableIPRange + initialV4UsingIPRange := initialSubnet.Status.V4UsingIPRange + initialV6AvailableIPRange := initialSubnet.Status.V6AvailableIPRange + initialV6UsingIPRange := initialSubnet.Status.V6UsingIPRange + + ginkgo.By("2. Create IptablesEIP to trigger IP allocation") + eipName := "test-eip-finalizer-" + framework.RandomSuffix() + eip := framework.MakeIptablesEIP(eipName, "", "", "", vpcNatGwName, "", "") + _ = iptablesEIPClient.CreateSync(eip) + + ginkgo.By("3. Wait for IptablesEIP CR to be ready") + eipCR := waitForIptablesEIPReady(iptablesEIPClient, eipName, 60*time.Second) + framework.ExpectNotNil(eipCR, "IptablesEIP CR should be created and ready") + framework.ExpectNotEmpty(eipCR.Status.IP, "IptablesEIP should have IP assigned") + + ginkgo.By("4. Wait for IptablesEIP CR finalizer to be added") + for range 60 { + eipCR = iptablesEIPClient.Get(eipName) + if eipCR != nil && len(eipCR.Finalizers) > 0 { + break + } + time.Sleep(1 * time.Second) } - } -} - -func newVPCQoSParamsInit() *qosParams { - qosParams := &qosParams{ - vpc1Name: "qos-vpc1-" + framework.RandomSuffix(), - vpc2Name: "qos-vpc2-" + framework.RandomSuffix(), - vpc1SubnetName: "qos-vpc1-subnet-" + framework.RandomSuffix(), - vpc2SubnetName: "qos-vpc2-subnet-" + framework.RandomSuffix(), - vpcNat1GwName: "qos-vpc1-gw-" + framework.RandomSuffix(), - vpcNat2GwName: "qos-vpc2-gw-" + framework.RandomSuffix(), - vpc1EIPName: "qos-vpc1-eip-" + framework.RandomSuffix(), - vpc2EIPName: "qos-vpc2-eip-" + framework.RandomSuffix(), - vpc1FIPName: "qos-vpc1-fip-" + framework.RandomSuffix(), - vpc2FIPName: "qos-vpc2-fip-" + framework.RandomSuffix(), - vpc1PodName: "qos-vpc1-pod-" + framework.RandomSuffix(), - vpc2PodName: "qos-vpc2-pod-" + framework.RandomSuffix(), - attachDefName: "qos-ovn-vpc-external-network-" + framework.RandomSuffix(), - } - qosParams.subnetProvider = fmt.Sprintf("%s.%s", qosParams.attachDefName, framework.KubeOvnNamespace) - return qosParams -} - -func getNicDefaultQoSPolicy(limit int) apiv1.QoSPolicyBandwidthLimitRules { - return apiv1.QoSPolicyBandwidthLimitRules{ - apiv1.QoSPolicyBandwidthLimitRule{ - Name: "net1-ingress", - Interface: "net1", - RateMax: strconv.Itoa(limit), - BurstMax: strconv.Itoa(limit), - Priority: 3, - Direction: apiv1.QoSDirectionIngress, - }, - apiv1.QoSPolicyBandwidthLimitRule{ - Name: "net1-egress", - Interface: "net1", - RateMax: strconv.Itoa(limit), - BurstMax: strconv.Itoa(limit), - Priority: 3, - Direction: apiv1.QoSDirectionEgress, - }, - } -} - -func getEIPQoSRule(limit int) apiv1.QoSPolicyBandwidthLimitRules { - return apiv1.QoSPolicyBandwidthLimitRules{ - apiv1.QoSPolicyBandwidthLimitRule{ - Name: "eip-ingress", - RateMax: strconv.Itoa(limit), - BurstMax: strconv.Itoa(limit), - Priority: 1, - Direction: apiv1.QoSDirectionIngress, - }, - apiv1.QoSPolicyBandwidthLimitRule{ - Name: "eip-egress", - RateMax: strconv.Itoa(limit), - BurstMax: strconv.Itoa(limit), - Priority: 1, - Direction: apiv1.QoSDirectionEgress, - }, - } -} - -func getSpecialQoSRule(limit int, ip string) apiv1.QoSPolicyBandwidthLimitRules { - return apiv1.QoSPolicyBandwidthLimitRules{ - apiv1.QoSPolicyBandwidthLimitRule{ - Name: "net1-extip-ingress", - Interface: "net1", - RateMax: strconv.Itoa(limit), - BurstMax: strconv.Itoa(limit), - Priority: 2, - Direction: apiv1.QoSDirectionIngress, - MatchType: apiv1.QoSMatchTypeIP, - MatchValue: "src " + ip + "/32", - }, - apiv1.QoSPolicyBandwidthLimitRule{ - Name: "net1-extip-egress", - Interface: "net1", - RateMax: strconv.Itoa(limit), - BurstMax: strconv.Itoa(limit), - Priority: 2, - Direction: apiv1.QoSDirectionEgress, - MatchType: apiv1.QoSMatchTypeIP, - MatchValue: "dst " + ip + "/32", - }, - } -} - -// defaultQoSCases test default qos policy= -func defaultQoSCases(f *framework.Framework, - vpcNatGwClient *framework.VpcNatGatewayClient, - podClient *framework.PodClient, - qosPolicyClient *framework.QoSPolicyClient, - vpc1Pod *corev1.Pod, - vpc2Pod *corev1.Pod, - vpc1EIP *apiv1.IptablesEIP, - vpc2EIP *apiv1.IptablesEIP, - natgwName string, -) { - ginkgo.GinkgoHelper() - - // create nic qos policy - qosPolicyName := "default-nic-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + qosPolicyName) - rules := getNicDefaultQoSPolicy(defaultNicLimit) - - qosPolicy := framework.MakeQoSPolicy(qosPolicyName, true, apiv1.QoSBindingTypeNatGw, rules) - _ = qosPolicyClient.CreateSync(qosPolicy) - - ginkgo.By("Patch natgw " + natgwName + " with qos policy " + qosPolicyName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, qosPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) - - ginkgo.By("Delete natgw pod " + natgwName + "-0") - natGwPodName := util.GenNatGwPodName(natgwName) - podClient.DeleteSync(natGwPodName) - - ginkgo.By("Wait for natgw " + natgwName + "qos rebuild") - time.Sleep(5 * time.Second) - - ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) - - ginkgo.By("Remove qos policy " + qosPolicyName + " from natgw " + natgwName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") - - ginkgo.By("Deleting qos policy " + qosPolicyName) - qosPolicyClient.DeleteSync(qosPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is not limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, false) -} - -// eipQoSCases test default qos policy -func eipQoSCases(f *framework.Framework, - eipClient *framework.IptablesEIPClient, - podClient *framework.PodClient, - qosPolicyClient *framework.QoSPolicyClient, - vpc1Pod *corev1.Pod, - vpc2Pod *corev1.Pod, - vpc1EIP *apiv1.IptablesEIP, - vpc2EIP *apiv1.IptablesEIP, - eipName string, - natgwName string, -) { - ginkgo.GinkgoHelper() - - // create eip qos policy - qosPolicyName := "eip-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + qosPolicyName) - rules := getEIPQoSRule(eipLimit) - - qosPolicy := framework.MakeQoSPolicy(qosPolicyName, false, apiv1.QoSBindingTypeEIP, rules) - qosPolicy = qosPolicyClient.CreateSync(qosPolicy) - - ginkgo.By("Patch eip " + eipName + " with qos policy " + qosPolicyName) - _ = eipClient.PatchQoSPolicySync(eipName, qosPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(eipLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, eipLimit, true) - - ginkgo.By("Update qos policy " + qosPolicyName + " with new rate limit") - - rules = getEIPQoSRule(updatedEIPLimit) - modifiedqosPolicy := qosPolicy.DeepCopy() - modifiedqosPolicy.Spec.BandwidthLimitRules = rules - qosPolicyClient.Patch(qosPolicy, modifiedqosPolicy) - qosPolicyClient.WaitToQoSReady(qosPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is changed to " + strconv.Itoa(updatedEIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, updatedEIPLimit, true) - - ginkgo.By("Delete natgw pod " + natgwName + "-0") - natGwPodName := util.GenNatGwPodName(natgwName) - podClient.DeleteSync(natGwPodName) - - ginkgo.By("Wait for natgw " + natgwName + "qos rebuid") - time.Sleep(5 * time.Second) - - ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(updatedEIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, updatedEIPLimit, true) - - newQoSPolicyName := "new-eip-qos-policy-" + framework.RandomSuffix() - newRules := getEIPQoSRule(newEIPLimit) - newQoSPolicy := framework.MakeQoSPolicy(newQoSPolicyName, false, apiv1.QoSBindingTypeEIP, newRules) - _ = qosPolicyClient.CreateSync(newQoSPolicy) - - ginkgo.By("Change qos policy of eip " + eipName + " to " + newQoSPolicyName) - _ = eipClient.PatchQoSPolicySync(eipName, newQoSPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(newEIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, newEIPLimit, true) - - ginkgo.By("Remove qos policy " + qosPolicyName + " from natgw " + eipName) - _ = eipClient.PatchQoSPolicySync(eipName, "") - - ginkgo.By("Deleting qos policy " + qosPolicyName) - qosPolicyClient.DeleteSync(qosPolicyName) - - ginkgo.By("Deleting qos policy " + newQoSPolicyName) - qosPolicyClient.DeleteSync(newQoSPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is not limited to " + strconv.Itoa(newEIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, newEIPLimit, false) -} - -// specifyingIPQoSCases test default qos policy -func specifyingIPQoSCases(f *framework.Framework, - vpcNatGwClient *framework.VpcNatGatewayClient, - qosPolicyClient *framework.QoSPolicyClient, - vpc1Pod *corev1.Pod, - vpc2Pod *corev1.Pod, - vpc1EIP *apiv1.IptablesEIP, - vpc2EIP *apiv1.IptablesEIP, - natgwName string, -) { - ginkgo.GinkgoHelper() - - // create nic qos policy - qosPolicyName := "specifying-ip-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + qosPolicyName) - - rules := getSpecialQoSRule(specificIPLimit, vpc2EIP.Status.IP) - - qosPolicy := framework.MakeQoSPolicy(qosPolicyName, true, apiv1.QoSBindingTypeNatGw, rules) - _ = qosPolicyClient.CreateSync(qosPolicy) - - ginkgo.By("Patch natgw " + natgwName + " with qos policy " + qosPolicyName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, qosPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is limited to " + strconv.Itoa(specificIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, specificIPLimit, true) - - ginkgo.By("Remove qos policy " + qosPolicyName + " from natgw " + natgwName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") - - ginkgo.By("Deleting qos policy " + qosPolicyName) - qosPolicyClient.DeleteSync(qosPolicyName) - - ginkgo.By("Check qos " + qosPolicyName + " is not limited to " + strconv.Itoa(specificIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, specificIPLimit, false) -} - -// priorityQoSCases test qos match priority -func priorityQoSCases(f *framework.Framework, - vpcNatGwClient *framework.VpcNatGatewayClient, - eipClient *framework.IptablesEIPClient, - qosPolicyClient *framework.QoSPolicyClient, - vpc1Pod *corev1.Pod, - vpc2Pod *corev1.Pod, - vpc1EIP *apiv1.IptablesEIP, - vpc2EIP *apiv1.IptablesEIP, - natgwName string, - eipName string, -) { - ginkgo.GinkgoHelper() - - // create nic qos policy - natGwQoSPolicyName := "priority-nic-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + natGwQoSPolicyName) - // default qos policy + special qos policy - natgwRules := getNicDefaultQoSPolicy(defaultNicLimit) - natgwRules = append(natgwRules, getSpecialQoSRule(specificIPLimit, vpc2EIP.Status.IP)...) - - natgwQoSPolicy := framework.MakeQoSPolicy(natGwQoSPolicyName, true, apiv1.QoSBindingTypeNatGw, natgwRules) - _ = qosPolicyClient.CreateSync(natgwQoSPolicy) - - ginkgo.By("Patch natgw " + natgwName + " with qos policy " + natGwQoSPolicyName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, natGwQoSPolicyName) - - eipQoSPolicyName := "eip-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + eipQoSPolicyName) - eipRules := getEIPQoSRule(eipLimit) - - eipQoSPolicy := framework.MakeQoSPolicy(eipQoSPolicyName, false, apiv1.QoSBindingTypeEIP, eipRules) - _ = qosPolicyClient.CreateSync(eipQoSPolicy) - - ginkgo.By("Patch eip " + eipName + " with qos policy " + eipQoSPolicyName) - _ = eipClient.PatchQoSPolicySync(eipName, eipQoSPolicyName) - - // match qos of priority 1 - ginkgo.By("Check qos to match priority 1 is limited to " + strconv.Itoa(eipLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, eipLimit, true) - - ginkgo.By("Remove qos policy " + eipQoSPolicyName + " from natgw " + eipName) - _ = eipClient.PatchQoSPolicySync(eipName, "") - - ginkgo.By("Deleting qos policy " + eipQoSPolicyName) - qosPolicyClient.DeleteSync(eipQoSPolicyName) - - // match qos of priority 2 - ginkgo.By("Check qos to match priority 2 is limited to " + strconv.Itoa(specificIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, specificIPLimit, true) - - // change qos policy of natgw - newNatGwQoSPolicyName := "new-priority-nic-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + newNatGwQoSPolicyName) - newNatgwRules := getNicDefaultQoSPolicy(defaultNicLimit) - - newNatgwQoSPolicy := framework.MakeQoSPolicy(newNatGwQoSPolicyName, true, apiv1.QoSBindingTypeNatGw, newNatgwRules) - _ = qosPolicyClient.CreateSync(newNatgwQoSPolicy) - - ginkgo.By("Change qos policy of natgw " + natgwName + " to " + newNatGwQoSPolicyName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, newNatGwQoSPolicyName) - - // match qos of priority 3 - ginkgo.By("Check qos to match priority 3 is limited to " + strconv.Itoa(specificIPLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) - - ginkgo.By("Remove qos policy " + natGwQoSPolicyName + " from natgw " + natgwName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") - - ginkgo.By("Deleting qos policy " + natGwQoSPolicyName) - qosPolicyClient.DeleteSync(natGwQoSPolicyName) - - ginkgo.By("Deleting qos policy " + newNatGwQoSPolicyName) - qosPolicyClient.DeleteSync(newNatGwQoSPolicyName) - - ginkgo.By("Check qos " + natGwQoSPolicyName + " is not limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, false) -} - -func createNatGwAndSetQosCases(f *framework.Framework, - vpcNatGwClient *framework.VpcNatGatewayClient, - ipClient *framework.IPClient, - eipClient *framework.IptablesEIPClient, - fipClient *framework.IptablesFIPClient, - subnetClient *framework.SubnetClient, - qosPolicyClient *framework.QoSPolicyClient, - vpc1Pod *corev1.Pod, - vpc2Pod *corev1.Pod, - vpc2EIP *apiv1.IptablesEIP, - natgwName string, - eipName string, - fipName string, - vpcName string, - overlaySubnetName string, - lanIP string, - attachDefName string, -) { - ginkgo.GinkgoHelper() - - // delete fip - ginkgo.By("Deleting fip " + fipName) - fipClient.DeleteSync(fipName) - - ginkgo.By("Deleting eip " + eipName) - eipClient.DeleteSync(eipName) - - // the only pod for vpc nat gateway - vpcNatGw1PodName := util.GenNatGwPodName(natgwName) - - // delete vpc nat gw statefulset remaining ip for eth0 and net2 - ginkgo.By("Deleting custom vpc nat gw " + natgwName) - vpcNatGwClient.DeleteSync(natgwName) - - overlaySubnet1 := subnetClient.Get(overlaySubnetName) - macvlanSubnet := subnetClient.Get(attachDefName) - eth0IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, overlaySubnet1.Spec.Provider) - net1IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, macvlanSubnet.Spec.Provider) - ginkgo.By("Deleting vpc nat gw eth0 ip " + eth0IpName) - ipClient.DeleteSync(eth0IpName) - ginkgo.By("Deleting vpc nat gw net1 ip " + net1IpName) - ipClient.DeleteSync(net1IpName) - - natgwQoSPolicyName := "default-nic-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + natgwQoSPolicyName) - rules := getNicDefaultQoSPolicy(defaultNicLimit) - - qosPolicy := framework.MakeQoSPolicy(natgwQoSPolicyName, true, apiv1.QoSBindingTypeNatGw, rules) - _ = qosPolicyClient.CreateSync(qosPolicy) - - ginkgo.By("Creating custom vpc nat gw") - vpcNatGw := framework.MakeVpcNatGateway(natgwName, vpcName, overlaySubnetName, lanIP, attachDefName, natgwQoSPolicyName) - _ = vpcNatGwClient.CreateSync(vpcNatGw, f.ClientSet) - - eipQoSPolicyName := "eip-qos-policy-" + framework.RandomSuffix() - ginkgo.By("Creating qos policy " + eipQoSPolicyName) - rules = getEIPQoSRule(eipLimit) - - eipQoSPolicy := framework.MakeQoSPolicy(eipQoSPolicyName, false, apiv1.QoSBindingTypeEIP, rules) - _ = qosPolicyClient.CreateSync(eipQoSPolicy) + framework.ExpectNotNil(eipCR, "IptablesEIP CR should exist") + framework.ExpectContainElement(eipCR.Finalizers, util.KubeOVNControllerFinalizer, + "IptablesEIP CR should have finalizer after creation") - ginkgo.By("Creating eip " + eipName) - vpc1EIP := framework.MakeIptablesEIP(eipName, "", "", "", natgwName, attachDefName, eipQoSPolicyName) - vpc1EIP = eipClient.CreateSync(vpc1EIP) + ginkgo.By("5. Wait for external subnet status to be updated after IptablesEIP creation") + time.Sleep(5 * time.Second) - ginkgo.By("Creating fip " + fipName) - fip := framework.MakeIptablesFIPRule(fipName, eipName, vpc1Pod.Status.PodIP) - _ = fipClient.CreateSync(fip) + ginkgo.By("6. Verify external subnet status after IptablesEIP CR creation") + afterCreateSubnet := subnetClient.Get(externalSubnetName) + verifySubnetStatusAfterEIPOperation(subnetClient, externalSubnetName, + afterCreateSubnet.Spec.Protocol, "IptablesEIP creation", eipCR.Status.IP) - ginkgo.By("Check qos " + eipQoSPolicyName + " is limited to " + strconv.Itoa(eipLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, eipLimit, true) - - ginkgo.By("Remove qos policy " + eipQoSPolicyName + " from natgw " + natgwName) - _ = eipClient.PatchQoSPolicySync(eipName, "") - - ginkgo.By("Check qos " + natgwQoSPolicyName + " is limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, true) - - ginkgo.By("Remove qos policy " + natgwQoSPolicyName + " from natgw " + natgwName) - _ = vpcNatGwClient.PatchQoSPolicySync(natgwName, "") - - ginkgo.By("Check qos " + natgwQoSPolicyName + " is not limited to " + strconv.Itoa(defaultNicLimit) + "Mbps") - checkQos(f, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, defaultNicLimit, false) - - ginkgo.By("Deleting qos policy " + natgwQoSPolicyName) - qosPolicyClient.DeleteSync(natgwQoSPolicyName) - - ginkgo.By("Deleting qos policy " + eipQoSPolicyName) - qosPolicyClient.DeleteSync(eipQoSPolicyName) -} - -func validRateLimit(text string, limit int) bool { - maxValue := float64(limit) * 1024 * 1024 * 1.2 - minValue := float64(limit) * 1024 * 1024 * 0.8 - lines := strings.SplitSeq(text, "\n") - for line := range lines { - if line == "" { - continue - } - fields := strings.Split(line, ",") - number, err := strconv.Atoi(fields[len(fields)-1]) - if err != nil { - continue - } - if v := float64(number); v >= minValue && v <= maxValue { - return true + // Verify IP count and range changes + switch afterCreateSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: + framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1 after IptablesEIP creation") + framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1 after IptablesEIP creation") + framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IptablesEIP creation") + framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IptablesEIP creation") + case apiv1.ProtocolIPv6: + framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1 after IptablesEIP creation") + framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1 after IptablesEIP creation") + framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IptablesEIP creation") + framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IptablesEIP creation") + default: + // Dual stack + framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1 after IptablesEIP creation") + framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1 after IptablesEIP creation") + framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1 after IptablesEIP creation") + framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1 after IptablesEIP creation") + framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IptablesEIP creation") + framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IptablesEIP creation") + framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IptablesEIP creation") + framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IptablesEIP creation") } - } - return false -} - -var _ = framework.Describe("[group:qos-policy]", func() { - f := framework.NewDefaultFramework("qos-policy") - var skip bool - var cs clientset.Interface - var attachNetClient *framework.NetworkAttachmentDefinitionClient - var clusterName string - var vpcClient *framework.VpcClient - var vpcNatGwClient *framework.VpcNatGatewayClient - var subnetClient *framework.SubnetClient - var podClient *framework.PodClient - var ipClient *framework.IPClient - var iptablesEIPClient *framework.IptablesEIPClient - var iptablesFIPClient *framework.IptablesFIPClient - var qosPolicyClient *framework.QoSPolicyClient + // Store the status after creation for later comparison + afterCreateV4AvailableIPs := afterCreateSubnet.Status.V4AvailableIPs + afterCreateV4UsingIPs := afterCreateSubnet.Status.V4UsingIPs + afterCreateV6AvailableIPs := afterCreateSubnet.Status.V6AvailableIPs + afterCreateV6UsingIPs := afterCreateSubnet.Status.V6UsingIPs + afterCreateV4AvailableIPRange := afterCreateSubnet.Status.V4AvailableIPRange + afterCreateV4UsingIPRange := afterCreateSubnet.Status.V4UsingIPRange + afterCreateV6AvailableIPRange := afterCreateSubnet.Status.V6AvailableIPRange + afterCreateV6UsingIPRange := afterCreateSubnet.Status.V6UsingIPRange + + ginkgo.By("7. Delete the IptablesEIP to trigger IP release") + iptablesEIPClient.DeleteSync(eipName) + + ginkgo.By("8. Wait for IptablesEIP CR to be deleted") + deleted := false + for range 30 { + _, err := f.KubeOVNClientSet.KubeovnV1().IptablesEIPs().Get(context.Background(), eipName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + deleted = true + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectTrue(deleted, "IptablesEIP CR should be deleted") - var net1NicName string - var dockerExtNetName string - - // docker network - var dockerExtNetNetwork *dockernetwork.Inspect - - var vpcQosParams *qosParams - var vpc1Pod *corev1.Pod - var vpc2Pod *corev1.Pod - var vpc1EIP *apiv1.IptablesEIP - var vpc2EIP *apiv1.IptablesEIP - var vpc1FIP *apiv1.IptablesFIPRule - var vpc2FIP *apiv1.IptablesFIPRule - - var lanIP string - var overlaySubnetV4Cidr string - var overlaySubnetV4Gw string - var eth0Exist, net1Exist bool - var annotations1 map[string]string - var annotations2 map[string]string - var iperfServerCmd []string + ginkgo.By("9. Wait for external subnet status to be updated after IptablesEIP deletion") + time.Sleep(5 * time.Second) - ginkgo.BeforeEach(func() { - vpcQosParams = newVPCQoSParamsInit() + ginkgo.By("10. Verify external subnet status after IptablesEIP CR deletion") + afterDeleteSubnet := subnetClient.Get(externalSubnetName) + verifySubnetStatusAfterEIPOperation(subnetClient, externalSubnetName, + afterDeleteSubnet.Spec.Protocol, "IptablesEIP deletion", "") - dockerExtNetName = "kube-ovn-qos-" + framework.RandomSuffix() + // Verify IP count and range restoration + switch afterDeleteSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: + // Verify IP count is restored + framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1 after IptablesEIP deletion") + framework.ExpectEqual(afterCreateV4UsingIPs-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1 after IptablesEIP deletion") + + // Verify IP range changed + framework.ExpectNotEqual(afterCreateV4AvailableIPRange, afterDeleteSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IptablesEIP deletion") + framework.ExpectNotEqual(afterCreateV4UsingIPRange, afterDeleteSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IptablesEIP deletion") + + // Verify counts match initial state + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after IptablesEIP deletion") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after IptablesEIP deletion") + case apiv1.ProtocolIPv6: + // Verify IP count is restored + framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1 after IptablesEIP deletion") + framework.ExpectEqual(afterCreateV6UsingIPs-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1 after IptablesEIP deletion") + + // Verify IP range changed + framework.ExpectNotEqual(afterCreateV6AvailableIPRange, afterDeleteSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IptablesEIP deletion") + framework.ExpectNotEqual(afterCreateV6UsingIPRange, afterDeleteSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IptablesEIP deletion") + + // Verify counts match initial state + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after IptablesEIP deletion") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after IptablesEIP deletion") + default: + // Dual stack + framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1 after IptablesEIP deletion") + framework.ExpectEqual(afterCreateV4UsingIPs-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1 after IptablesEIP deletion") + framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1 after IptablesEIP deletion") + framework.ExpectEqual(afterCreateV6UsingIPs-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1 after IptablesEIP deletion") + + framework.ExpectNotEqual(afterCreateV4AvailableIPRange, afterDeleteSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after IptablesEIP deletion") + framework.ExpectNotEqual(afterCreateV4UsingIPRange, afterDeleteSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after IptablesEIP deletion") + framework.ExpectNotEqual(afterCreateV6AvailableIPRange, afterDeleteSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after IptablesEIP deletion") + framework.ExpectNotEqual(afterCreateV6UsingIPRange, afterDeleteSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after IptablesEIP deletion") + + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after IptablesEIP deletion") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after IptablesEIP deletion") + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after IptablesEIP deletion") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after IptablesEIP deletion") + } - vpcQosParams.vpc1SubnetName = "qos-vpc1-subnet-" + framework.RandomSuffix() - vpcQosParams.vpc2SubnetName = "qos-vpc2-subnet-" + framework.RandomSuffix() + ginkgo.By("11. Test completed: IptablesEIP CR creation and deletion properly updates external subnet status via finalizer handlers") + }) - vpcQosParams.vpcNat1GwName = "qos-gw1-" + framework.RandomSuffix() - vpcQosParams.vpcNat2GwName = "qos-gw2-" + framework.RandomSuffix() + framework.ConformanceIt("Test IptablesEIP finalizer cannot be removed when used by NAT rules", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") - vpcQosParams.vpc1EIPName = "qos-vpc1-eip-" + framework.RandomSuffix() - vpcQosParams.vpc2EIPName = "qos-vpc2-eip-" + framework.RandomSuffix() + overlaySubnetV4Cidr := "10.0.4.0/24" + overlaySubnetV4Gw := "10.0.4.1" + lanIP := "10.0.4.254" + natgwQoS := "" + setupVpcNatGwTestEnvironment( + f, dockerExtNet1Network, attachNetClient, + subnetClient, vpcClient, vpcNatGwClient, + vpcName, overlaySubnetName, vpcNatGwName, natgwQoS, + overlaySubnetV4Cidr, overlaySubnetV4Gw, lanIP, + dockerExtNet1Name, networkAttachDefName, net1NicName, + externalSubnetProvider, + false, + ) - vpcQosParams.vpc1FIPName = "qos-vpc1-fip-" + framework.RandomSuffix() - vpcQosParams.vpc2FIPName = "qos-vpc2-fip-" + framework.RandomSuffix() + ginkgo.By("1. Create a VIP for FIP") + vipName := "test-vip-" + framework.RandomSuffix() + vip := framework.MakeVip(f.Namespace.Name, vipName, overlaySubnetName, "", "", "") + _ = vipClient.CreateSync(vip) + vip = vipClient.Get(vipName) - vpcQosParams.vpc1PodName = "qos-vpc1-pod-" + framework.RandomSuffix() - vpcQosParams.vpc2PodName = "qos-vpc2-pod-" + framework.RandomSuffix() + ginkgo.By("2. Create IptablesEIP") + eipName := "test-eip-with-fip-" + framework.RandomSuffix() + eip := framework.MakeIptablesEIP(eipName, "", "", "", vpcNatGwName, "", "") + _ = iptablesEIPClient.CreateSync(eip) - vpcQosParams.attachDefName = "qos-ovn-vpc-external-network-" + framework.RandomSuffix() - vpcQosParams.subnetProvider = fmt.Sprintf("%s.%s", vpcQosParams.attachDefName, framework.KubeOvnNamespace) + ginkgo.By("3. Wait for IptablesEIP to be ready") + eipCR := waitForIptablesEIPReady(iptablesEIPClient, eipName, 60*time.Second) + framework.ExpectNotNil(eipCR, "IptablesEIP CR should be created and ready") - cs = f.ClientSet - podClient = f.PodClient() - attachNetClient = f.NetworkAttachmentDefinitionClientNS(framework.KubeOvnNamespace) - subnetClient = f.SubnetClient() - vpcClient = f.VpcClient() - vpcNatGwClient = f.VpcNatGatewayClient() - iptablesEIPClient = f.IptablesEIPClient() - ipClient = f.IPClient() - iptablesFIPClient = f.IptablesFIPClient() - qosPolicyClient = f.QoSPolicyClient() + ginkgo.By("4. Create IptablesFIP using the EIP") + fipName := "test-fip-" + framework.RandomSuffix() + fip := framework.MakeIptablesFIPRule(fipName, eipName, vip.Status.V4ip) + _ = iptablesFIPClient.CreateSync(fip) - if skip { - ginkgo.Skip("underlay spec only runs on kind clusters") + ginkgo.By("5. Wait for EIP status to show it's being used by FIP") + for range 60 { + eipCR = iptablesEIPClient.Get(eipName) + if eipCR != nil && strings.Contains(eipCR.Status.Nat, util.FipUsingEip) { + break + } + time.Sleep(1 * time.Second) } + framework.ExpectTrue(strings.Contains(eipCR.Status.Nat, util.FipUsingEip), + "EIP status.Nat should contain 'fip' when used by FIP rule") + + ginkgo.By("6. Delete the IptablesEIP (should not remove finalizer while FIP exists)") + err := f.KubeOVNClientSet.KubeovnV1().IptablesEIPs().Delete(context.Background(), eipName, metav1.DeleteOptions{}) + framework.ExpectNoError(err, "Deleting IptablesEIP should succeed") + + ginkgo.By("7. Wait and verify EIP still exists with finalizer (blocked by FIP)") + time.Sleep(5 * time.Second) + eipCR = iptablesEIPClient.Get(eipName) + framework.ExpectNotNil(eipCR, "IptablesEIP should still exist") + framework.ExpectNotNil(eipCR.DeletionTimestamp, "IptablesEIP should have DeletionTimestamp") + framework.ExpectContainElement(eipCR.Finalizers, util.KubeOVNControllerFinalizer, + "IptablesEIP should still have finalizer because it's being used by FIP") + + ginkgo.By("8. Delete the FIP to unblock EIP deletion") + iptablesFIPClient.DeleteSync(fipName) - if clusterName == "" { - ginkgo.By("Getting k8s nodes") - k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) - framework.ExpectNoError(err) + ginkgo.By("9. Wait for FIP to be deleted") + fipDeleted := false + for range 30 { + _, err := f.KubeOVNClientSet.KubeovnV1().IptablesFIPRules().Get(context.Background(), fipName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + fipDeleted = true + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectTrue(fipDeleted, "FIP should be deleted") + + ginkgo.By("10. Wait for EIP status.Nat to be cleared or EIP to be deleted") + for range 30 { + eipCR, err := f.KubeOVNClientSet.KubeovnV1().IptablesEIPs().Get(context.Background(), eipName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + // EIP already deleted, which is expected + break + } + framework.ExpectNoError(err, "Failed to get IptablesEIP") + if eipCR.Status.Nat == "" { + break + } + time.Sleep(1 * time.Second) + } - cluster, ok := kind.IsKindProvided(k8sNodes.Items[0].Spec.ProviderID) - if !ok { - skip = true - ginkgo.Skip("underlay spec only runs on kind clusters") + ginkgo.By("11. Verify EIP is now deleted after FIP is removed") + eipDeleted := false + for range 30 { + _, err := f.KubeOVNClientSet.KubeovnV1().IptablesEIPs().Get(context.Background(), eipName, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + eipDeleted = true + break } - clusterName = cluster + time.Sleep(1 * time.Second) } + framework.ExpectTrue(eipDeleted, "IptablesEIP should be deleted after FIP is removed") - ginkgo.By("Ensuring docker network " + dockerExtNetName + " exists") - network, err := docker.NetworkCreate(dockerExtNetName, true, true) - framework.ExpectNoError(err, "creating docker network "+dockerExtNetName) - dockerExtNetNetwork = network + ginkgo.By("12. Clean up VIP") + vipClient.DeleteSync(vipName) - ginkgo.By("Getting kind nodes") - nodes, err := kind.ListNodes(clusterName, "") - framework.ExpectNoError(err, "getting nodes in kind cluster") - framework.ExpectNotEmpty(nodes) - - ginkgo.By("Connecting nodes to the docker network") - err = kind.NetworkConnect(dockerExtNetNetwork.ID, nodes) - framework.ExpectNoError(err, "connecting nodes to network "+dockerExtNetName) + ginkgo.By("13. Test completed: IptablesEIP finalizer correctly blocks deletion when used by NAT rules") + }) - ginkgo.By("Getting node links that belong to the docker network") - nodes, err = kind.ListNodes(clusterName, "") - framework.ExpectNoError(err, "getting nodes in kind cluster") + framework.ConformanceIt("Test VPC NAT Gateway with no IPAM NAD and noDefaultEIP", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") - ginkgo.By("Validating node links") - network1, err := docker.NetworkInspect(dockerExtNetName) - framework.ExpectNoError(err) - for _, node := range nodes { - links, err := node.ListLinks() - framework.ExpectNoError(err, "failed to list links on node %s: %v", node.Name(), err) - net1Mac := network1.Containers[node.ID].MacAddress - for _, link := range links { - ginkgo.By("exist node nic " + link.IfName) - if link.IfName == "eth0" { - eth0Exist = true + overlaySubnetV4Cidr := "10.0.5.0/24" + overlaySubnetV4Gw := "10.0.5.1" + lanIP := "10.0.5.254" + natgwQoS := "" + noIPAMNadName := "no-ipam-nad-" + framework.RandomSuffix() + noIPAMProvider := fmt.Sprintf("%s.%s", noIPAMNadName, framework.KubeOvnNamespace) + + ginkgo.By("1. Setting up NAD without IPAM and creating subnet using standard flow") + // Create NAD without IPAM section + ginkgo.By("Getting docker network " + dockerExtNet1Name) + network, err := docker.NetworkInspect(dockerExtNet1Name) + framework.ExpectNoError(err, "getting docker network "+dockerExtNet1Name) + + ginkgo.By("Creating network attachment definition without IPAM " + noIPAMNadName) + // NAD config without ipam - this is the key difference + attachConf := fmt.Sprintf(`{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "%s", + "mode": "bridge" + }`, net1NicName) + + attachNet := framework.MakeNetworkAttachmentDefinition(noIPAMNadName, framework.KubeOvnNamespace, attachConf) + nad := attachNetClient.Create(attachNet) + ginkgo.By("Got network attachment definition " + nad.Name) + + ginkgo.By("Creating underlay macvlan subnet " + noIPAMNadName) + var cidrV4, cidrV6, gatewayV4, gatewayV6 string + for _, config := range dockerExtNet1Network.IPAM.Config { + switch util.CheckProtocol(config.Subnet.Addr().String()) { + case apiv1.ProtocolIPv4: + if f.HasIPv4() { + cidrV4 = config.Subnet.String() + gatewayV4 = config.Gateway.String() } - if link.Address == net1Mac.String() { - net1NicName = link.IfName - net1Exist = true + case apiv1.ProtocolIPv6: + if f.HasIPv6() { + cidrV6 = config.Subnet.String() + gatewayV6 = config.Gateway.String() } } - framework.ExpectTrue(eth0Exist) - framework.ExpectTrue(net1Exist) } - setupNetworkAttachmentDefinition( - f, dockerExtNetNetwork, attachNetClient, - subnetClient, vpcQosParams.attachDefName, net1NicName, vpcQosParams.subnetProvider, dockerExtNetName) - }) - - ginkgo.AfterEach(func() { - ginkgo.By("Deleting macvlan underlay subnet " + vpcQosParams.attachDefName) - subnetClient.DeleteSync(vpcQosParams.attachDefName) - - // delete net1 attachment definition - ginkgo.By("Deleting nad " + vpcQosParams.attachDefName) - attachNetClient.Delete(vpcQosParams.attachDefName) - - ginkgo.By("Getting nodes") - nodes, err := kind.ListNodes(clusterName, "") - framework.ExpectNoError(err, "getting nodes in cluster") - - if dockerExtNetNetwork != nil { - ginkgo.By("Disconnecting nodes from the docker network") - err = kind.NetworkDisconnect(dockerExtNetNetwork.ID, nodes) - framework.ExpectNoError(err, "disconnecting nodes from network "+dockerExtNetName) - ginkgo.By("Deleting docker network " + dockerExtNetName + " exists") - err := docker.NetworkRemove(dockerExtNetNetwork.ID) - framework.ExpectNoError(err, "deleting docker network "+dockerExtNetName) + cidr := make([]string, 0, 2) + gateway := make([]string, 0, 2) + if f.HasIPv4() { + cidr = append(cidr, cidrV4) + gateway = append(gateway, gatewayV4) } - }) - - _ = framework.Describe("vpc qos", func() { - ginkgo.BeforeEach(func() { - iperfServerCmd = []string{"iperf", "-s", "-i", "1", "-p", iperf2Port} - overlaySubnetV4Cidr = "10.0.0.0/24" - overlaySubnetV4Gw = "10.0.0.1" - lanIP = "10.0.0.254" - natgwQoS := "" - setupVpcNatGwTestEnvironment( - f, dockerExtNetNetwork, attachNetClient, - subnetClient, vpcClient, vpcNatGwClient, - vpcQosParams.vpc1Name, vpcQosParams.vpc1SubnetName, vpcQosParams.vpcNat1GwName, - natgwQoS, overlaySubnetV4Cidr, overlaySubnetV4Gw, lanIP, - dockerExtNetName, vpcQosParams.attachDefName, net1NicName, - vpcQosParams.subnetProvider, - true, - ) - annotations1 = map[string]string{ - util.LogicalSwitchAnnotation: vpcQosParams.vpc1SubnetName, + if f.HasIPv6() { + cidr = append(cidr, cidrV6) + gateway = append(gateway, gatewayV6) + } + excludeIPs := make([]string, 0, len(network.Containers)*2) + for _, container := range network.Containers { + if container.IPv4Address.IsValid() && f.HasIPv4() { + excludeIPs = append(excludeIPs, container.IPv4Address.Addr().String()) + } + if container.IPv6Address.IsValid() && f.HasIPv6() { + excludeIPs = append(excludeIPs, container.IPv6Address.Addr().String()) } - ginkgo.By("Creating pod " + vpcQosParams.vpc1PodName) - vpc1Pod = framework.MakePod(f.Namespace.Name, vpcQosParams.vpc1PodName, nil, annotations1, framework.AgnhostImage, iperfServerCmd, nil) - vpc1Pod = podClient.CreateSync(vpc1Pod) - - ginkgo.By("Creating eip " + vpcQosParams.vpc1EIPName) - vpc1EIP = framework.MakeIptablesEIP(vpcQosParams.vpc1EIPName, "", "", "", vpcQosParams.vpcNat1GwName, vpcQosParams.attachDefName, "") - vpc1EIP = iptablesEIPClient.CreateSync(vpc1EIP) - - ginkgo.By("Creating fip " + vpcQosParams.vpc1FIPName) - vpc1FIP = framework.MakeIptablesFIPRule(vpcQosParams.vpc1FIPName, vpcQosParams.vpc1EIPName, vpc1Pod.Status.PodIP) - _ = iptablesFIPClient.CreateSync(vpc1FIP) - - setupVpcNatGwTestEnvironment( - f, dockerExtNetNetwork, attachNetClient, - subnetClient, vpcClient, vpcNatGwClient, - vpcQosParams.vpc2Name, vpcQosParams.vpc2SubnetName, vpcQosParams.vpcNat2GwName, - natgwQoS, overlaySubnetV4Cidr, overlaySubnetV4Gw, lanIP, - dockerExtNetName, vpcQosParams.attachDefName, net1NicName, - vpcQosParams.subnetProvider, - true, - ) - - annotations2 = map[string]string{ - util.LogicalSwitchAnnotation: vpcQosParams.vpc2SubnetName, + } + macvlanSubnet := framework.MakeSubnet(noIPAMNadName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", noIPAMProvider, excludeIPs, nil, nil) + _ = subnetClient.CreateSync(macvlanSubnet) + + ginkgo.By("2. Creating custom vpc " + vpcName) + vpc := framework.MakeVpc(vpcName, lanIP, false, false, nil) + _ = vpcClient.CreateSync(vpc) + + ginkgo.By("3. Creating custom overlay subnet " + overlaySubnetName) + overlaySubnet := framework.MakeSubnet(overlaySubnetName, "", overlaySubnetV4Cidr, overlaySubnetV4Gw, vpcName, "", nil, nil, nil) + _ = subnetClient.CreateSync(overlaySubnet) + + ginkgo.By("4. Creating custom vpc nat gw with noDefaultEIP=true " + vpcNatGwName) + vpcNatGw := framework.MakeVpcNatGatewayWithNoDefaultEIP(vpcNatGwName, vpcName, overlaySubnetName, lanIP, noIPAMNadName, natgwQoS, true) + _ = vpcNatGwClient.CreateSync(vpcNatGw, f.ClientSet) + + ginkgo.By("5. Verifying VPC NAT Gateway is created") + createdGw := vpcNatGwClient.Get(vpcNatGwName) + framework.ExpectNotNil(createdGw, "VPC NAT Gateway should be created") + framework.ExpectTrue(createdGw.Spec.NoDefaultEIP, "noDefaultEIP should be true") + + ginkgo.By("6. Verifying no default EIP is created") + time.Sleep(10 * time.Second) + eips, err := f.KubeOVNClientSet.KubeovnV1().IptablesEIPs().List(context.Background(), metav1.ListOptions{}) + framework.ExpectNoError(err, "Failed to list IptablesEIPs") + hasDefaultEIP := false + for _, eip := range eips.Items { + if eip.Spec.NatGwDp == vpcNatGwName { + hasDefaultEIP = true + break } + } + framework.ExpectFalse(hasDefaultEIP, "No default EIP should be created when noDefaultEIP is true") + + ginkgo.By("7. Testing manual EIP creation") + eipName := "manual-eip-" + framework.RandomSuffix() + eip := framework.MakeIptablesEIP(eipName, "", "", "", vpcNatGwName, noIPAMNadName, "") + _ = iptablesEIPClient.CreateSync(eip) + + ginkgo.By("8. Verifying manually created EIP") + eipCR := waitForIptablesEIPReady(iptablesEIPClient, eipName, 60*time.Second) + framework.ExpectNotNil(eipCR, "Manual EIP should be created successfully") + framework.ExpectNotEmpty(eipCR.Status.IP, "Manual EIP should have IP assigned") + + ginkgo.By("9. Testing VIP and FIP with manual EIP") + vipName := "test-vip-no-ipam-" + framework.RandomSuffix() + vip := framework.MakeVip(f.Namespace.Name, vipName, overlaySubnetName, "", "", "") + _ = vipClient.CreateSync(vip) + vip = vipClient.Get(vipName) + + fipName := "test-fip-no-ipam-" + framework.RandomSuffix() + fip := framework.MakeIptablesFIPRule(fipName, eipName, vip.Status.V4ip) + _ = iptablesFIPClient.CreateSync(fip) + + ginkgo.By("10. Verifying FIP is created successfully") + createdFip := iptablesFIPClient.Get(fipName) + framework.ExpectNotNil(createdFip, "FIP should be created successfully") + framework.ExpectTrue(createdFip.Status.Ready, "FIP should be ready") + + ginkgo.By("11. Cleaning up resources") + iptablesFIPClient.DeleteSync(fipName) + vipClient.DeleteSync(vipName) + iptablesEIPClient.DeleteSync(eipName) + + vpcNatGwClient.DeleteSync(vpcNatGwName) + subnetClient.DeleteSync(overlaySubnetName) + subnetClient.DeleteSync(noIPAMNadName) + vpcClient.DeleteSync(vpcName) + attachNetClient.Delete(noIPAMNadName) - ginkgo.By("Creating pod " + vpcQosParams.vpc2PodName) - vpc2Pod = framework.MakePod(f.Namespace.Name, vpcQosParams.vpc2PodName, nil, annotations2, framework.AgnhostImage, iperfServerCmd, nil) - vpc2Pod = podClient.CreateSync(vpc2Pod) - - ginkgo.By("Creating eip " + vpcQosParams.vpc2EIPName) - vpc2EIP = framework.MakeIptablesEIP(vpcQosParams.vpc2EIPName, "", "", "", vpcQosParams.vpcNat2GwName, vpcQosParams.attachDefName, "") - vpc2EIP = iptablesEIPClient.CreateSync(vpc2EIP) - - ginkgo.By("Creating fip " + vpcQosParams.vpc2FIPName) - vpc2FIP = framework.MakeIptablesFIPRule(vpcQosParams.vpc2FIPName, vpcQosParams.vpc2EIPName, vpc2Pod.Status.PodIP) - _ = iptablesFIPClient.CreateSync(vpc2FIP) - }) - ginkgo.AfterEach(func() { - ginkgo.By("Deleting fip " + vpcQosParams.vpc1FIPName) - iptablesFIPClient.DeleteSync(vpcQosParams.vpc1FIPName) - - ginkgo.By("Deleting fip " + vpcQosParams.vpc2FIPName) - iptablesFIPClient.DeleteSync(vpcQosParams.vpc2FIPName) - - ginkgo.By("Deleting eip " + vpcQosParams.vpc1EIPName) - iptablesEIPClient.DeleteSync(vpcQosParams.vpc1EIPName) - - ginkgo.By("Deleting eip " + vpcQosParams.vpc2EIPName) - iptablesEIPClient.DeleteSync(vpcQosParams.vpc2EIPName) - - ginkgo.By("Deleting pod " + vpcQosParams.vpc1PodName) - podClient.DeleteSync(vpcQosParams.vpc1PodName) - - ginkgo.By("Deleting pod " + vpcQosParams.vpc2PodName) - podClient.DeleteSync(vpcQosParams.vpc2PodName) - - ginkgo.By("Deleting custom vpc nat gw " + vpcQosParams.vpcNat1GwName) - vpcNatGwClient.DeleteSync(vpcQosParams.vpcNat1GwName) - - ginkgo.By("Deleting custom vpc nat gw " + vpcQosParams.vpcNat2GwName) - vpcNatGwClient.DeleteSync(vpcQosParams.vpcNat2GwName) - - // the only pod for vpc nat gateway - vpcNatGw1PodName := util.GenNatGwPodName(vpcQosParams.vpcNat1GwName) - - // delete vpc nat gw statefulset remaining ip for eth0 and net2 - overlaySubnet1 := subnetClient.Get(vpcQosParams.vpc1SubnetName) - macvlanSubnet := subnetClient.Get(vpcQosParams.attachDefName) - eth0IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, overlaySubnet1.Spec.Provider) - net1IpName := ovs.PodNameToPortName(vpcNatGw1PodName, framework.KubeOvnNamespace, macvlanSubnet.Spec.Provider) - ginkgo.By("Deleting vpc nat gw eth0 ip " + eth0IpName) - ipClient.DeleteSync(eth0IpName) - ginkgo.By("Deleting vpc nat gw net1 ip " + net1IpName) - ipClient.DeleteSync(net1IpName) - ginkgo.By("Deleting overlay subnet " + vpcQosParams.vpc1SubnetName) - subnetClient.DeleteSync(vpcQosParams.vpc1SubnetName) - - ginkgo.By("Getting overlay subnet " + vpcQosParams.vpc2SubnetName) - overlaySubnet2 := subnetClient.Get(vpcQosParams.vpc2SubnetName) - - vpcNatGw2PodName := util.GenNatGwPodName(vpcQosParams.vpcNat2GwName) - eth0IpName = ovs.PodNameToPortName(vpcNatGw2PodName, framework.KubeOvnNamespace, overlaySubnet2.Spec.Provider) - net1IpName = ovs.PodNameToPortName(vpcNatGw2PodName, framework.KubeOvnNamespace, macvlanSubnet.Spec.Provider) - ginkgo.By("Deleting vpc nat gw eth0 ip " + eth0IpName) - ipClient.DeleteSync(eth0IpName) - ginkgo.By("Deleting vpc nat gw net1 ip " + net1IpName) - ipClient.DeleteSync(net1IpName) - ginkgo.By("Deleting overlay subnet " + vpcQosParams.vpc2SubnetName) - subnetClient.DeleteSync(vpcQosParams.vpc2SubnetName) - - ginkgo.By("Deleting custom vpc " + vpcQosParams.vpc1Name) - vpcClient.DeleteSync(vpcQosParams.vpc1Name) - - ginkgo.By("Deleting custom vpc " + vpcQosParams.vpc2Name) - vpcClient.DeleteSync(vpcQosParams.vpc2Name) - }) - framework.ConformanceIt("default nic qos", func() { - // case 1: set qos policy for natgw - // case 2: rebuild qos when natgw pod restart - defaultQoSCases(f, vpcNatGwClient, podClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpcNat1GwName) - }) - framework.ConformanceIt("eip qos", func() { - // case 1: set qos policy for eip - // case 2: update qos policy for eip - // case 3: change qos policy of eip - // case 4: rebuild qos when natgw pod restart - eipQoSCases(f, iptablesEIPClient, podClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpc1EIPName, vpcQosParams.vpcNat1GwName) - }) - framework.ConformanceIt("specifying ip qos", func() { - // case 1: set specific ip qos policy for natgw - specifyingIPQoSCases(f, vpcNatGwClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpcNat1GwName) - }) - framework.ConformanceIt("qos priority matching", func() { - // case 1: test qos match priority - // case 2: change qos policy of natgw - priorityQoSCases(f, vpcNatGwClient, iptablesEIPClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc1EIP, vpc2EIP, vpcQosParams.vpcNat1GwName, vpcQosParams.vpc1EIPName) - }) - framework.ConformanceIt("create resource with qos policy", func() { - // case 1: test qos when create natgw with qos policy - // case 2: test qos when create eip with qos policy - createNatGwAndSetQosCases(f, - vpcNatGwClient, ipClient, iptablesEIPClient, iptablesFIPClient, - subnetClient, qosPolicyClient, vpc1Pod, vpc2Pod, vpc2EIP, vpcQosParams.vpcNat1GwName, - vpcQosParams.vpc1EIPName, vpcQosParams.vpc1FIPName, vpcQosParams.vpc1Name, - vpcQosParams.vpc1SubnetName, lanIP, vpcQosParams.attachDefName) - }) + ginkgo.By("12. Test completed: VPC NAT Gateway with no IPAM NAD and noDefaultEIP works correctly") }) }) diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index ccc4f39bd32..dcfd67dca4f 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -11,6 +11,7 @@ import ( dockernetwork "github.com/moby/moby/api/types/network" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -30,9 +31,13 @@ import ( "github.com/kubeovn/kube-ovn/test/e2e/framework/kind" ) -const dockerNetworkName = "kube-ovn-vlan" - -const dockerExtraNetworkName = "kube-ovn-extra-vlan" +// Docker network configurations will be initialized in init() with random names and subnets +var ( + dockerNetworkName string + dockerExtraNetworkName string + dockerNetworkSubnet string + dockerExtraNetworkSubnet string +) func makeProviderNetwork(providerNetworkName string, exchangeLinkName bool, linkMap map[string]*iproute.Link) *kubeovnv1.ProviderNetwork { var defaultInterface string @@ -72,7 +77,24 @@ func makeOvnDnat(name, ovnEip, ipType, ipName, vpc, v4Ip, internalPort, external return framework.MakeOvnDnatRule(name, ovnEip, ipType, ipName, vpc, v4Ip, internalPort, externalPort, protocol) } -var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { +func makeExternalGatewayConfigMap(gwNodes, switchName string, cidr []string, gwType string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.ExternalGatewayConfig, + Namespace: framework.KubeOvnNamespace, + }, + Data: map[string]string{ + "enable-external-gw": "true", + "external-gw-nodes": gwNodes, + "external-gw-switch": switchName, + "type": gwType, + "external-gw-nic": "eth1", + "external-gw-addr": strings.Join(cidr, ","), + }, + } +} + +var _ = framework.SerialDescribe("[group:ovn-vpc-nat-gw]", func() { f := framework.NewDefaultFramework("ovn-vpc-nat-gw") var skip bool @@ -127,57 +149,64 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { // gw node is 2 means e2e HA cluster will have 2 gw nodes and a worker node // in this env, tcpdump gw nat flows will be more clear - noBfdVpcName = "no-bfd-vpc-" + framework.RandomSuffix() - bfdVpcName = "bfd-vpc-" + framework.RandomSuffix() + randomSuffix := framework.RandomSuffix() + // Use first 6 digits of random suffix for provider network names (12 byte limit) + shortSuffix := randomSuffix[:6] + noBfdVpcName = "no-bfd-vpc-" + randomSuffix + bfdVpcName = "bfd-vpc-" + randomSuffix // nats use ip crd name or vip crd - fipName = "fip-" + framework.RandomSuffix() - - countingEipName = "counting-eip-" + framework.RandomSuffix() - noBfdSubnetName = "no-bfd-subnet-" + framework.RandomSuffix() - noBfdExtraSubnetName = "no-bfd-extra-subnet-" + framework.RandomSuffix() - lrpEipSnatName = "lrp-eip-snat-" + framework.RandomSuffix() - lrpExtraEipSnatName = "lrp-extra-eip-snat-" + framework.RandomSuffix() - bfdSubnetName = "bfd-subnet-" + framework.RandomSuffix() + fipName = "fip-" + randomSuffix + + countingEipName = "counting-eip-" + randomSuffix + noBfdSubnetName = "no-bfd-subnet-" + randomSuffix + noBfdExtraSubnetName = "no-bfd-extra-subnet-" + randomSuffix + lrpEipSnatName = "lrp-eip-snat-" + randomSuffix + lrpExtraEipSnatName = "lrp-extra-eip-snat-" + randomSuffix + bfdSubnetName = "bfd-subnet-" + randomSuffix + // Default external network must use fixed "external" name for global reuse + // kube-ovn-cni creates default external subnet with name "external" when enable-eip-snat providerNetworkName = "external" - providerExtraNetworkName = "extra" - vlanName = "vlan-" + framework.RandomSuffix() - vlanExtraName = "vlan-extra-" + framework.RandomSuffix() underlaySubnetName = "external" - underlayExtraSubnetName = "extra" + vlanName = "vlan" + // Extra network uses random suffix to test multi-external-subnet scenarios + // provider network name has 12 bytes limit, use short suffix + providerExtraNetworkName = "extra-" + shortSuffix + underlayExtraSubnetName = "extra-" + shortSuffix + vlanExtraName = "vlan-extra-" + randomSuffix // sharing case - sharedVipName = "shared-vip-" + framework.RandomSuffix() - sharedEipDnatName = "shared-eip-dnat-" + framework.RandomSuffix() - sharedEipFipShoudOkName = "shared-eip-fip-should-ok-" + framework.RandomSuffix() - sharedEipFipShoudFailName = "shared-eip-fip-should-fail-" + framework.RandomSuffix() + sharedVipName = "shared-vip-" + randomSuffix + sharedEipDnatName = "shared-eip-dnat-" + randomSuffix + sharedEipFipShoudOkName = "shared-eip-fip-should-ok-" + randomSuffix + sharedEipFipShoudFailName = "shared-eip-fip-should-fail-" + randomSuffix // pod with fip - fipPodName = "fip-pod-" + framework.RandomSuffix() + fipPodName = "fip-pod-" + randomSuffix podEipName = fipPodName podFipName = fipPodName // pod with fip for extra external subnet - fipExtraPodName = "fip-extra-pod-" + framework.RandomSuffix() + fipExtraPodName = "fip-extra-pod-" + randomSuffix podExtraEipName = fipExtraPodName podExtraFipName = fipExtraPodName // fip use ip addr - ipFipVipName = "ip-fip-vip-" + framework.RandomSuffix() - ipFipEipName = "ip-fip-eip-" + framework.RandomSuffix() - ipFipName = "ip-fip-" + framework.RandomSuffix() + ipFipVipName = "ip-fip-vip-" + randomSuffix + ipFipEipName = "ip-fip-eip-" + randomSuffix + ipFipName = "ip-fip-" + randomSuffix // dnat use ip addr - ipDnatVipName = "ip-dnat-vip-" + framework.RandomSuffix() - ipDnatEipName = "ip-dnat-eip-" + framework.RandomSuffix() - ipDnatName = "ip-dnat-" + framework.RandomSuffix() + ipDnatVipName = "ip-dnat-vip-" + randomSuffix + ipDnatEipName = "ip-dnat-eip-" + randomSuffix + ipDnatName = "ip-dnat-" + randomSuffix // snat use ip cidr - cidrSnatEipName = "cidr-snat-eip-" + framework.RandomSuffix() - cidrSnatName = "cidr-snat-" + framework.RandomSuffix() - ipSnatVipName = "ip-snat-vip-" + framework.RandomSuffix() - ipSnatEipName = "ip-snat-eip-" + framework.RandomSuffix() - ipSnatName = "ip-snat-" + framework.RandomSuffix() + cidrSnatEipName = "cidr-snat-eip-" + randomSuffix + cidrSnatName = "cidr-snat-" + randomSuffix + ipSnatVipName = "ip-snat-vip-" + randomSuffix + ipSnatEipName = "ip-snat-eip-" + randomSuffix + ipSnatName = "ip-snat-" + randomSuffix if skip { ginkgo.Skip("underlay spec only runs on kind clusters") @@ -197,16 +226,16 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { } if dockerNetwork == nil { - ginkgo.By("Ensuring docker network " + dockerNetworkName + " exists") - network, err := docker.NetworkCreate(dockerNetworkName, true, true) - framework.ExpectNoError(err, "creating docker network "+dockerNetworkName) + ginkgo.By("Ensuring docker network " + dockerNetworkName + " with subnet " + dockerNetworkSubnet + " exists") + network, err := docker.NetworkCreateWithSubnet(dockerNetworkName, dockerNetworkSubnet, true, true) + framework.ExpectNoError(err, "ensuring docker network "+dockerNetworkName+" exists") dockerNetwork = network } if dockerExtraNetwork == nil { - ginkgo.By("Ensuring extra docker network " + dockerExtraNetworkName + " exists") - network, err := docker.NetworkCreate(dockerExtraNetworkName, true, true) - framework.ExpectNoError(err, "creating extra docker network "+dockerExtraNetworkName) + ginkgo.By("Ensuring extra docker network " + dockerExtraNetworkName + " with subnet " + dockerExtraNetworkSubnet + " exists") + network, err := docker.NetworkCreateWithSubnet(dockerExtraNetworkName, dockerExtraNetworkSubnet, true, true) + framework.ExpectNoError(err, "ensuring extra docker network "+dockerExtraNetworkName+" exists") dockerExtraNetwork = network } @@ -437,6 +466,40 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { ginkgo.By("Deleting provider extra network " + providerExtraNetworkName) providerNetworkClient.DeleteSync(providerExtraNetworkName) + ginkgo.By("Confirming provider networks are fully deleted") + startTime := time.Now() + checkCount := 0 + framework.WaitUntil(2*time.Second, 2*time.Minute, func(_ context.Context) (bool, error) { + checkCount++ + elapsed := time.Since(startTime) + + pn, err := providerNetworkClient.ProviderNetworkInterface.Get(context.Background(), providerNetworkName, metav1.GetOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return false, err + } + if err == nil { + // Provider network still exists + framework.Logf("Warning: provider network %s still exists after %v (check #%d), DeletionTimestamp: %v", + providerNetworkName, elapsed, checkCount, pn.DeletionTimestamp) + return false, nil + } + + extraPn, err := providerNetworkClient.ProviderNetworkInterface.Get(context.Background(), providerExtraNetworkName, metav1.GetOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return false, err + } + if err == nil { + // Extra provider network still exists + framework.Logf("Warning: provider extra network %s still exists after %v (check #%d), DeletionTimestamp: %v", + providerExtraNetworkName, elapsed, checkCount, extraPn.DeletionTimestamp) + return false, nil + } + + // Both provider networks are deleted + framework.Logf("Provider networks fully deleted after %v (%d checks)", elapsed, checkCount) + return true, nil + }, "waiting for provider networks to be fully deleted") + ginkgo.By("Getting nodes") nodes, err := kind.ListNodes(clusterName, "") framework.ExpectNoError(err, "getting nodes in cluster") @@ -470,11 +533,15 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { exchangeLinkName := false itFn(exchangeLinkName, providerNetworkName, linkMap, &providerBridgeIps) + ginkgo.By("Verifying vlan " + vlanName + " does not exist from previous test") + _, err = vlanClient.VlanInterface.Get(context.Background(), vlanName, metav1.GetOptions{}) + framework.ExpectTrue(k8serrors.IsNotFound(err), "vlan %s should not exist, AfterEach cleanup may have failed", vlanName) + ginkgo.By("Creating underlay vlan " + vlanName) vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) _ = vlanClient.Create(vlan) - ginkgo.By("Creating underlay subnet " + underlaySubnetName) + ginkgo.By("Preparing subnet configuration for " + underlaySubnetName) var cidrV4, cidrV6, gatewayV4, gatewayV6 string for _, config := range dockerNetwork.IPAM.Config { switch util.CheckProtocol(config.Subnet.String()) { @@ -511,14 +578,27 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { } vlanSubnetCidr := strings.Join(cidr, ",") vlanSubnetGw := strings.Join(gateway, ",") + + ginkgo.By("Verifying subnet " + underlaySubnetName + " does not exist from previous test") + _, err = subnetClient.SubnetInterface.Get(context.Background(), underlaySubnetName, metav1.GetOptions{}) + framework.ExpectTrue(k8serrors.IsNotFound(err), "subnet %s should not exist, AfterEach cleanup may have failed", underlaySubnetName) + + ginkgo.By("Creating underlay subnet " + underlaySubnetName) + var oldUnderlayExternalSubnet *kubeovnv1.Subnet underlaySubnet := framework.MakeSubnet(underlaySubnetName, vlanName, vlanSubnetCidr, vlanSubnetGw, "", "", excludeIPs, nil, nil) - oldUnderlayExternalSubnet := subnetClient.CreateSync(underlaySubnet) + oldUnderlayExternalSubnet = subnetClient.CreateSync(underlaySubnet) countingEip := makeOvnEip(countingEipName, underlaySubnetName, "", "", "", "") _ = ovnEipClient.CreateSync(countingEip) ginkgo.By("Checking underlay vlan " + oldUnderlayExternalSubnet.Name) framework.ExpectEqual(oldUnderlayExternalSubnet.Spec.Vlan, vlanName) framework.ExpectNotEqual(oldUnderlayExternalSubnet.Spec.CIDRBlock, "") - time.Sleep(3 * time.Second) + ginkgo.By("Wait for ovn eip finalizer to be added") + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { + eipCR := ovnEipClient.Get(countingEipName) + return eipCR != nil && len(eipCR.Finalizers) > 0, nil + }, "OvnEip should have finalizer added") + ginkgo.By("Wait for subnet status to be updated after ovn eip creation") + time.Sleep(5 * time.Second) newUnerlayExternalSubnet := subnetClient.Get(underlaySubnetName) ginkgo.By("Check status using ovn eip for subnet " + underlaySubnetName) if newUnerlayExternalSubnet.Spec.Protocol == kubeovnv1.ProtocolIPv4 { @@ -535,7 +615,8 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { // delete counting eip oldUnderlayExternalSubnet = newUnerlayExternalSubnet ovnEipClient.DeleteSync(countingEipName) - time.Sleep(3 * time.Second) + ginkgo.By("Wait for subnet status to be updated after ovn eip deletion") + time.Sleep(5 * time.Second) newUnerlayExternalSubnet = subnetClient.Get(underlaySubnetName) if newUnerlayExternalSubnet.Spec.Protocol == kubeovnv1.ProtocolIPv4 { framework.ExpectEqual(oldUnderlayExternalSubnet.Status.V4AvailableIPs+1, newUnerlayExternalSubnet.Status.V4AvailableIPs) @@ -551,20 +632,7 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { externalGwNodes := strings.Join(gwNodeNames, ",") ginkgo.By("Creating config map ovn-external-gw-config for centralized case") - cmData := map[string]string{ - "enable-external-gw": "true", - "external-gw-nodes": externalGwNodes, - "type": kubeovnv1.GWCentralizedType, - "external-gw-nic": "eth1", - "external-gw-addr": strings.Join(cidr, ","), - } - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: util.ExternalGatewayConfig, - Namespace: framework.KubeOvnNamespace, - }, - Data: cmData, - } + configMap := makeExternalGatewayConfigMap(externalGwNodes, underlaySubnetName, cidr, kubeovnv1.GWCentralizedType) _, err = cs.CoreV1().ConfigMaps(framework.KubeOvnNamespace).Create(context.Background(), configMap, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create") @@ -694,11 +762,15 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectNoError(err, "getting extra docker network "+dockerExtraNetworkName) itFn(exchangeLinkName, providerExtraNetworkName, extraLinkMap, &extraProviderBridgeIps) + ginkgo.By("Verifying extra vlan " + vlanExtraName + " does not exist from previous test") + _, err = vlanClient.VlanInterface.Get(context.Background(), vlanExtraName, metav1.GetOptions{}) + framework.ExpectTrue(k8serrors.IsNotFound(err), "vlan %s should not exist, AfterEach cleanup may have failed", vlanExtraName) + ginkgo.By("Creating underlay extra vlan " + vlanExtraName) vlan = framework.MakeVlan(vlanExtraName, providerExtraNetworkName, 0) _ = vlanClient.Create(vlan) - ginkgo.By("Creating extra underlay subnet " + underlayExtraSubnetName) + ginkgo.By("Preparing extra subnet configuration for " + underlayExtraSubnetName) cidrV4, cidrV6, gatewayV4, gatewayV6 = "", "", "", "" for _, config := range dockerExtraNetwork.IPAM.Config { switch util.CheckProtocol(config.Subnet.String()) { @@ -736,6 +808,12 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { } extraVlanSubnetCidr := strings.Join(cidr, ",") extraVlanSubnetGw := strings.Join(gateway, ",") + + ginkgo.By("Verifying extra subnet " + underlayExtraSubnetName + " does not exist from previous test") + _, err = subnetClient.SubnetInterface.Get(context.Background(), underlayExtraSubnetName, metav1.GetOptions{}) + framework.ExpectTrue(k8serrors.IsNotFound(err), "subnet %s should not exist, AfterEach cleanup may have failed", underlayExtraSubnetName) + + ginkgo.By("Creating extra underlay subnet " + underlayExtraSubnetName) underlayExtraSubnet := framework.MakeSubnet(underlayExtraSubnetName, vlanExtraName, extraVlanSubnetCidr, extraVlanSubnetGw, "", "", extraExcludeIPs, nil, nil) _ = subnetClient.CreateSync(underlayExtraSubnet) vlanExtraSubnet := subnetClient.Get(underlayExtraSubnetName) @@ -939,21 +1017,8 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { _ = podClient.CreateSync(pod) } ginkgo.By("3. Updating config map ovn-external-gw-config for distributed case") - cmData = map[string]string{ - "enable-external-gw": "true", - "external-gw-nodes": externalGwNodes, - "type": kubeovnv1.GWDistributedType, - "external-gw-nic": "eth1", - "external-gw-addr": strings.Join(cidr, ","), - } // TODO:// external-gw-nodes could be auto managed by recognizing gw chassis node which has the external-gw-nic - configMap = &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: util.ExternalGatewayConfig, - Namespace: framework.KubeOvnNamespace, - }, - Data: cmData, - } + configMap = makeExternalGatewayConfigMap(externalGwNodes, underlaySubnetName, cidr, kubeovnv1.GWDistributedType) _, err = cs.CoreV1().ConfigMaps(framework.KubeOvnNamespace).Update(context.Background(), configMap, metav1.UpdateOptions{}) framework.ExpectNoError(err, "failed to update ConfigMap") @@ -1027,9 +1092,283 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { framework.ExpectHaveKeyWithValue(node.Labels, util.NodeExtGwLabel, "false") } }) + + // Helper function to wait for OvnEip to be ready + waitForOvnEipReady := func(eipClient *framework.OvnEipClient, eipName string, timeout time.Duration) *kubeovnv1.OvnEip { + ginkgo.GinkgoHelper() + var eip *kubeovnv1.OvnEip + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + eip = eipClient.Get(eipName) + if eip != nil && eip.Status.V4Ip != "" && eip.Status.Ready { + framework.Logf("OvnEip %s is ready with V4IP: %s", eipName, eip.Status.V4Ip) + return eip + } + time.Sleep(2 * time.Second) + } + framework.Failf("Timeout waiting for OvnEip %s to be ready", eipName) + return nil + } + + framework.ConformanceIt("should properly manage OvnEip lifecycle with finalizer and update subnet status", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + ginkgo.By("Setting up provider network and underlay subnet") + network, err := docker.NetworkInspect(dockerNetworkName) + framework.ExpectNoError(err, "getting docker network "+dockerNetworkName) + + exchangeLinkName := false + itFn(exchangeLinkName, providerNetworkName, linkMap, &providerBridgeIps) + + ginkgo.By("Creating underlay vlan " + vlanName) + vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) + _ = vlanClient.Create(vlan) + + ginkgo.By("Creating underlay subnet " + underlaySubnetName) + var cidrV4, cidrV6, gatewayV4, gatewayV6 string + for _, config := range dockerNetwork.IPAM.Config { + switch util.CheckProtocol(config.Subnet.String()) { + case kubeovnv1.ProtocolIPv4: + if f.HasIPv4() { + cidrV4 = config.Subnet.String() + gatewayV4 = config.Gateway.String() + } + case kubeovnv1.ProtocolIPv6: + if f.HasIPv6() { + cidrV6 = config.Subnet.String() + gatewayV6 = config.Gateway.String() + } + } + } + cidr := make([]string, 0, 2) + gateway := make([]string, 0, 2) + if f.HasIPv4() { + cidr = append(cidr, cidrV4) + gateway = append(gateway, gatewayV4) + } + if f.HasIPv6() { + cidr = append(cidr, cidrV6) + gateway = append(gateway, gatewayV6) + } + excludeIPs := make([]string, 0, len(network.Containers)*2) + for _, container := range network.Containers { + if container.IPv4Address.IsValid() && f.HasIPv4() { + excludeIPs = append(excludeIPs, container.IPv4Address.Addr().String()) + } + if container.IPv6Address.IsValid() && f.HasIPv6() { + excludeIPs = append(excludeIPs, container.IPv6Address.Addr().String()) + } + } + vlanSubnetCidr := strings.Join(cidr, ",") + vlanSubnetGw := strings.Join(gateway, ",") + underlaySubnet := framework.MakeSubnet(underlaySubnetName, vlanName, vlanSubnetCidr, vlanSubnetGw, "", "", excludeIPs, nil, nil) + _ = subnetClient.CreateSync(underlaySubnet) + + ginkgo.By("Step 1: Recording initial underlay subnet status") + initialSubnet := subnetClient.Get(underlaySubnetName) + framework.Logf("Initial subnet status - V4: Available=%.0f Using=%.0f, V6: Available=%.0f Using=%.0f", + initialSubnet.Status.V4AvailableIPs, initialSubnet.Status.V4UsingIPs, + initialSubnet.Status.V6AvailableIPs, initialSubnet.Status.V6UsingIPs) + + // Store initial status + initialV4Available := initialSubnet.Status.V4AvailableIPs + initialV4Using := initialSubnet.Status.V4UsingIPs + initialV6Available := initialSubnet.Status.V6AvailableIPs + initialV6Using := initialSubnet.Status.V6UsingIPs + + ginkgo.By("Step 2: Creating OvnEip and waiting for it to be ready") + eipName := "test-ovn-eip-lifecycle-" + framework.RandomSuffix() + eip := makeOvnEip(eipName, underlaySubnetName, "", "", "", util.OvnEipTypeNAT) + _ = ovnEipClient.CreateSync(eip) + + eipCR := waitForOvnEipReady(ovnEipClient, eipName, 2*time.Minute) + framework.ExpectNotEmpty(eipCR.Status.V4Ip, "OvnEip should have V4 IP assigned") + + ginkgo.By("Step 3: Verifying finalizer is added to OvnEip") + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { + eipCR = ovnEipClient.Get(eipName) + return eipCR != nil && len(eipCR.Finalizers) > 0, nil + }, "OvnEip should have finalizer added") + framework.ExpectContainElement(eipCR.Finalizers, util.KubeOVNControllerFinalizer, + "OvnEip must have controller finalizer") + + ginkgo.By("Step 4: Verifying subnet status updated after OvnEip creation") + time.Sleep(5 * time.Second) + afterCreateSubnet := subnetClient.Get(underlaySubnetName) + + // Verify based on protocol + protocol := afterCreateSubnet.Spec.Protocol + framework.Logf("Verifying subnet status for protocol: %s", protocol) + + if protocol == kubeovnv1.ProtocolIPv4 || protocol == kubeovnv1.ProtocolDual { + framework.ExpectEqual(initialV4Available-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1") + framework.ExpectEqual(initialV4Using+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1") + framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V4UsingIPRange, eipCR.Status.V4Ip), + "EIP V4 IP should be in using range") + } + if protocol == kubeovnv1.ProtocolIPv6 || protocol == kubeovnv1.ProtocolDual { + framework.ExpectEqual(initialV6Available-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1") + framework.ExpectEqual(initialV6Using+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1") + } + + // Store after-create status + afterCreateV4Available := afterCreateSubnet.Status.V4AvailableIPs + afterCreateV4Using := afterCreateSubnet.Status.V4UsingIPs + afterCreateV6Available := afterCreateSubnet.Status.V6AvailableIPs + afterCreateV6Using := afterCreateSubnet.Status.V6UsingIPs + + ginkgo.By("Step 5: Deleting OvnEip and verifying cleanup") + ovnEipClient.DeleteSync(eipName) + + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { + _, err := f.KubeOVNClientSet.KubeovnV1().OvnEips().Get(context.Background(), eipName, metav1.GetOptions{}) + return k8serrors.IsNotFound(err), nil + }, "OvnEip should be deleted") + + ginkgo.By("Step 6: Verifying subnet status restored after OvnEip deletion") + time.Sleep(5 * time.Second) + afterDeleteSubnet := subnetClient.Get(underlaySubnetName) + + if protocol == kubeovnv1.ProtocolIPv4 || protocol == kubeovnv1.ProtocolDual { + framework.ExpectEqual(afterCreateV4Available+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1") + framework.ExpectEqual(afterCreateV4Using-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1") + framework.ExpectEqual(initialV4Available, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value") + framework.ExpectEqual(initialV4Using, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value") + } + if protocol == kubeovnv1.ProtocolIPv6 || protocol == kubeovnv1.ProtocolDual { + framework.ExpectEqual(afterCreateV6Available+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1") + framework.ExpectEqual(afterCreateV6Using-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1") + framework.ExpectEqual(initialV6Available, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value") + framework.ExpectEqual(initialV6Using, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value") + } + + framework.Logf("OvnEip lifecycle test completed successfully") + }) + + framework.ConformanceIt("should block OvnEip deletion when used by NAT rules", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + ginkgo.By("Setting up test environment") + network, err := docker.NetworkInspect(dockerNetworkName) + framework.ExpectNoError(err) + + exchangeLinkName := false + itFn(exchangeLinkName, providerNetworkName, linkMap, &providerBridgeIps) + + ginkgo.By("Creating underlay vlan and subnet") + vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) + _ = vlanClient.Create(vlan) + + var cidrV4, gatewayV4 string + for _, config := range dockerNetwork.IPAM.Config { + if util.CheckProtocol(config.Subnet.String()) == kubeovnv1.ProtocolIPv4 && f.HasIPv4() { + cidrV4 = config.Subnet.String() + gatewayV4 = config.Gateway.String() + break + } + } + excludeIPs := make([]string, 0, len(network.Containers)) + for _, container := range network.Containers { + if container.IPv4Address.IsValid() && f.HasIPv4() { + excludeIPs = append(excludeIPs, container.IPv4Address.Addr().String()) + } + } + underlaySubnet := framework.MakeSubnet(underlaySubnetName, vlanName, cidrV4, gatewayV4, "", "", excludeIPs, nil, nil) + _ = subnetClient.CreateSync(underlaySubnet) + + ginkgo.By("Step 1: Creating custom VPC and subnet for testing") + testVpcName := "test-vpc-dep-" + framework.RandomSuffix() + testSubnetName := "test-subnet-dep-" + framework.RandomSuffix() + testVpc := framework.MakeVpc(testVpcName, "", false, false, nil) + _ = vpcClient.CreateSync(testVpc) + + testSubnet := framework.MakeSubnet(testSubnetName, "", "192.168.100.0/24", "192.168.100.1", testVpcName, util.OvnProvider, nil, nil, nil) + _ = subnetClient.CreateSync(testSubnet) + + ginkgo.By("Step 2: Creating VIP for FIP") + vipName := "test-vip-dep-" + framework.RandomSuffix() + vip := makeOvnVip(namespaceName, vipName, testSubnetName, "", "", "") + vip = vipClient.CreateSync(vip) + framework.ExpectNotEmpty(vip.Status.V4ip) + + ginkgo.By("Step 3: Creating OvnEip") + eipName := "test-eip-with-dep-" + framework.RandomSuffix() + eip := makeOvnEip(eipName, underlaySubnetName, "", "", "", util.OvnEipTypeNAT) + _ = ovnEipClient.CreateSync(eip) + + eipCR := waitForOvnEipReady(ovnEipClient, eipName, 2*time.Minute) + + ginkgo.By("Step 4: Creating OvnFip using the EIP") + fipName := "test-fip-dep-" + framework.RandomSuffix() + fip := makeOvnFip(fipName, eipName, "", "", testVpcName, vip.Status.V4ip) + _ = ovnFipClient.CreateSync(fip) + + ginkgo.By("Step 5: Verifying EIP Status.Nat shows FIP usage") + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { + eipCR = ovnEipClient.Get(eipName) + return eipCR != nil && strings.Contains(eipCR.Status.Nat, util.FipUsingEip), nil + }, "EIP Status.Nat should contain 'fip'") + + ginkgo.By("Step 6: Attempting to delete EIP (should be blocked by FIP)") + err = f.KubeOVNClientSet.KubeovnV1().OvnEips().Delete(context.Background(), eipName, metav1.DeleteOptions{}) + framework.ExpectNoError(err, "Delete operation should succeed") + + ginkgo.By("Step 7: Verifying EIP still exists with finalizer (blocked)") + time.Sleep(5 * time.Second) + eipCR = ovnEipClient.Get(eipName) + framework.ExpectNotNil(eipCR, "EIP should still exist") + framework.ExpectNotNil(eipCR.DeletionTimestamp, "EIP should have DeletionTimestamp") + framework.ExpectContainElement(eipCR.Finalizers, util.KubeOVNControllerFinalizer, + "EIP should still have finalizer because FIP is using it") + + ginkgo.By("Step 8: Deleting FIP to unblock EIP deletion") + ovnFipClient.DeleteSync(fipName) + + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { + _, err := f.KubeOVNClientSet.KubeovnV1().OvnFips().Get(context.Background(), fipName, metav1.GetOptions{}) + return k8serrors.IsNotFound(err), nil + }, "FIP should be deleted") + + ginkgo.By("Step 9: Verifying EIP is now deleted after FIP removal") + framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) { + _, err := f.KubeOVNClientSet.KubeovnV1().OvnEips().Get(context.Background(), eipName, metav1.GetOptions{}) + return k8serrors.IsNotFound(err), nil + }, "EIP should be deleted after FIP is removed") + + ginkgo.By("Step 10: Cleaning up test resources") + vipClient.DeleteSync(vipName) + subnetClient.DeleteSync(testSubnetName) + vpcClient.DeleteSync(testVpcName) + + framework.Logf("OvnEip dependency blocking test completed successfully") + }) }) func init() { + // Generate unique network names and subnets for this test run + // This avoids conflicts with other parallel test runs on the same host + suffix := framework.RandomSuffix() + dockerNetworkName = "kube-ovn-vlan0-" + suffix + dockerExtraNetworkName = "kube-ovn-extra-vlan0-" + suffix + + // Generate random /24 subnets within 172.28.0.0/16 + // This allows up to 256 parallel test runs on the same host + subnets := docker.GenerateRandomSubnets(2) + dockerNetworkSubnet = subnets[0] + dockerExtraNetworkSubnet = subnets[1] + klog.SetOutput(ginkgo.GinkgoWriter) // Register flags. diff --git a/test/e2e/vip/e2e_test.go b/test/e2e/vip/e2e_test.go index 580cd78e765..a0ec0b1ed53 100644 --- a/test/e2e/vip/e2e_test.go +++ b/test/e2e/vip/e2e_test.go @@ -199,13 +199,198 @@ var _ = framework.Describe("[group:vip]", func() { securityGroupClient.DeleteSync(securityGroupName) }) + framework.ConformanceIt("Test vip subnet status update with finalizer", func() { + f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") + + ginkgo.By("1. Get initial subnet status") + initialSubnet := subnetClient.Get(subnetName) + initialV4AvailableIPs := initialSubnet.Status.V4AvailableIPs + initialV4UsingIPs := initialSubnet.Status.V4UsingIPs + initialV6AvailableIPs := initialSubnet.Status.V6AvailableIPs + initialV6UsingIPs := initialSubnet.Status.V6UsingIPs + initialV4AvailableIPRange := initialSubnet.Status.V4AvailableIPRange + initialV4UsingIPRange := initialSubnet.Status.V4UsingIPRange + initialV6AvailableIPRange := initialSubnet.Status.V6AvailableIPRange + initialV6UsingIPRange := initialSubnet.Status.V6UsingIPRange + + ginkgo.By("2. Create a VIP and verify finalizer is added") + testVipName := "test-vip-finalizer-" + framework.RandomSuffix() + testVip := makeOvnVip(namespaceName, testVipName, subnetName, "", "", "") + testVip = vipClient.CreateSync(testVip) + + // Verify VIP has finalizer + framework.ExpectContainElement(testVip.Finalizers, util.KubeOVNControllerFinalizer) + + ginkgo.By("3. Wait for subnet status to be updated after VIP creation") + time.Sleep(5 * time.Second) + + ginkgo.By("4. Verify subnet status after VIP creation") + afterCreateSubnet := subnetClient.Get(subnetName) + switch afterCreateSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: + // Verify IP count changed + framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1 after VIP creation") + framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1 after VIP creation") + + // Verify IP range changed + framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after VIP creation") + framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after VIP creation") + + // Verify the VIP's IP is in the using range + vipIP := testVip.Status.V4ip + framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V4UsingIPRange, vipIP), + "VIP IP %s should be in V4UsingIPRange %s", vipIP, afterCreateSubnet.Status.V4UsingIPRange) + case apiv1.ProtocolIPv6: + // Verify IP count changed + framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1 after VIP creation") + framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1 after VIP creation") + + // Verify IP range changed + framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after VIP creation") + framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after VIP creation") + + // Verify the VIP's IP is in the using range + vipIP := testVip.Status.V6ip + framework.ExpectTrue(strings.Contains(afterCreateSubnet.Status.V6UsingIPRange, vipIP), + "VIP IP %s should be in V6UsingIPRange %s", vipIP, afterCreateSubnet.Status.V6UsingIPRange) + default: + // Dual stack + framework.ExpectEqual(initialV4AvailableIPs-1, afterCreateSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should decrease by 1 after VIP creation") + framework.ExpectEqual(initialV4UsingIPs+1, afterCreateSubnet.Status.V4UsingIPs, + "V4UsingIPs should increase by 1 after VIP creation") + framework.ExpectEqual(initialV6AvailableIPs-1, afterCreateSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should decrease by 1 after VIP creation") + framework.ExpectEqual(initialV6UsingIPs+1, afterCreateSubnet.Status.V6UsingIPs, + "V6UsingIPs should increase by 1 after VIP creation") + + framework.ExpectNotEqual(initialV4AvailableIPRange, afterCreateSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after VIP creation") + framework.ExpectNotEqual(initialV4UsingIPRange, afterCreateSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after VIP creation") + framework.ExpectNotEqual(initialV6AvailableIPRange, afterCreateSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after VIP creation") + framework.ExpectNotEqual(initialV6UsingIPRange, afterCreateSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after VIP creation") + } + + // Store the status after creation for later comparison + afterCreateV4AvailableIPs := afterCreateSubnet.Status.V4AvailableIPs + afterCreateV4UsingIPs := afterCreateSubnet.Status.V4UsingIPs + afterCreateV6AvailableIPs := afterCreateSubnet.Status.V6AvailableIPs + afterCreateV6UsingIPs := afterCreateSubnet.Status.V6UsingIPs + afterCreateV4AvailableIPRange := afterCreateSubnet.Status.V4AvailableIPRange + afterCreateV4UsingIPRange := afterCreateSubnet.Status.V4UsingIPRange + afterCreateV6AvailableIPRange := afterCreateSubnet.Status.V6AvailableIPRange + afterCreateV6UsingIPRange := afterCreateSubnet.Status.V6UsingIPRange + + ginkgo.By("5. Delete the VIP") + vipClient.DeleteSync(testVipName) + + ginkgo.By("6. Wait for subnet status to be updated after VIP deletion") + time.Sleep(5 * time.Second) + + ginkgo.By("7. Verify subnet status after VIP deletion") + afterDeleteSubnet := subnetClient.Get(subnetName) + switch afterDeleteSubnet.Spec.Protocol { + case apiv1.ProtocolIPv4: + // Verify IP count is restored + framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1 after VIP deletion") + framework.ExpectEqual(afterCreateV4UsingIPs-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1 after VIP deletion") + + // Verify IP range changed + framework.ExpectNotEqual(afterCreateV4AvailableIPRange, afterDeleteSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after VIP deletion") + framework.ExpectNotEqual(afterCreateV4UsingIPRange, afterDeleteSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after VIP deletion") + + // Verify counts match initial state + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after VIP deletion") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after VIP deletion") + case apiv1.ProtocolIPv6: + // Verify IP count is restored + framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1 after VIP deletion") + framework.ExpectEqual(afterCreateV6UsingIPs-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1 after VIP deletion") + + // Verify IP range changed + framework.ExpectNotEqual(afterCreateV6AvailableIPRange, afterDeleteSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after VIP deletion") + framework.ExpectNotEqual(afterCreateV6UsingIPRange, afterDeleteSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after VIP deletion") + + // Verify counts match initial state + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after VIP deletion") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after VIP deletion") + default: + // Dual stack + framework.ExpectEqual(afterCreateV4AvailableIPs+1, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should increase by 1 after VIP deletion") + framework.ExpectEqual(afterCreateV4UsingIPs-1, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should decrease by 1 after VIP deletion") + framework.ExpectEqual(afterCreateV6AvailableIPs+1, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should increase by 1 after VIP deletion") + framework.ExpectEqual(afterCreateV6UsingIPs-1, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should decrease by 1 after VIP deletion") + + framework.ExpectNotEqual(afterCreateV4AvailableIPRange, afterDeleteSubnet.Status.V4AvailableIPRange, + "V4AvailableIPRange should change after VIP deletion") + framework.ExpectNotEqual(afterCreateV4UsingIPRange, afterDeleteSubnet.Status.V4UsingIPRange, + "V4UsingIPRange should change after VIP deletion") + framework.ExpectNotEqual(afterCreateV6AvailableIPRange, afterDeleteSubnet.Status.V6AvailableIPRange, + "V6AvailableIPRange should change after VIP deletion") + framework.ExpectNotEqual(afterCreateV6UsingIPRange, afterDeleteSubnet.Status.V6UsingIPRange, + "V6UsingIPRange should change after VIP deletion") + + framework.ExpectEqual(initialV4AvailableIPs, afterDeleteSubnet.Status.V4AvailableIPs, + "V4AvailableIPs should return to initial value after VIP deletion") + framework.ExpectEqual(initialV4UsingIPs, afterDeleteSubnet.Status.V4UsingIPs, + "V4UsingIPs should return to initial value after VIP deletion") + framework.ExpectEqual(initialV6AvailableIPs, afterDeleteSubnet.Status.V6AvailableIPs, + "V6AvailableIPs should return to initial value after VIP deletion") + framework.ExpectEqual(initialV6UsingIPs, afterDeleteSubnet.Status.V6UsingIPs, + "V6UsingIPs should return to initial value after VIP deletion") + } + + ginkgo.By("8. Test completed: VIP creation and deletion properly updates subnet status via finalizer handlers") + }) + framework.ConformanceIt("Test vip", func() { f.SkipVersionPriorTo(1, 13, "This feature was introduced in v1.13") ginkgo.By("0. Test subnet status counting vip") oldSubnet := subnetClient.Get(subnetName) countingVip := makeOvnVip(namespaceName, countingVipName, subnetName, "", "", "") - _ = vipClient.CreateSync(countingVip) - time.Sleep(3 * time.Second) + countingVip = vipClient.CreateSync(countingVip) + + // Wait for finalizer to be added + ginkgo.By("Waiting for VIP finalizer to be added") + for range 10 { + countingVip = vipClient.Get(countingVipName) + if len(countingVip.Finalizers) > 0 { + break + } + time.Sleep(1 * time.Second) + } + framework.ExpectContainElement(countingVip.Finalizers, util.KubeOVNControllerFinalizer) + + // Wait for subnet status to be updated + ginkgo.By("Waiting for subnet status to be updated after VIP creation") + time.Sleep(5 * time.Second) newSubnet := subnetClient.Get(subnetName) if newSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { framework.ExpectEqual(oldSubnet.Status.V4AvailableIPs-1, newSubnet.Status.V4AvailableIPs) @@ -220,8 +405,9 @@ var _ = framework.Describe("[group:vip]", func() { } oldSubnet = newSubnet // delete counting vip + ginkgo.By("Deleting counting VIP and waiting for subnet status update") vipClient.DeleteSync(countingVipName) - time.Sleep(3 * time.Second) + time.Sleep(5 * time.Second) newSubnet = subnetClient.Get(subnetName) if newSubnet.Spec.Protocol == apiv1.ProtocolIPv4 { framework.ExpectEqual(oldSubnet.Status.V4AvailableIPs+1, newSubnet.Status.V4AvailableIPs) diff --git a/test/e2e/webhook/subnet/subnet.go b/test/e2e/webhook/subnet/subnet.go index e08a2b1eae7..c017ec10c5f 100644 --- a/test/e2e/webhook/subnet/subnet.go +++ b/test/e2e/webhook/subnet/subnet.go @@ -4,6 +4,7 @@ import ( "context" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" "github.com/onsi/ginkgo/v2" @@ -95,26 +96,39 @@ var _ = framework.Describe("[group:webhook-subnet]", func() { f.SkipVersionPriorTo(1, 15, "vpc cannot be set to non-ovn-cluster on update is not supported before 1.15.0") ginkgo.By("Creating subnet " + subnetName) subnet := framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil) - subnet = subnetClient.CreateSync(subnet) + subnetClient.CreateSync(subnet) + // TODO: Use Patch instead of Update to modify only the Spec.Vpc field. + // This would avoid resource conflicts caused by concurrent status updates from the controller. + // Example: subnetClient.Patch(subnet, modifiedSubnet, framework.Timeout()) ginkgo.By("Validating vpc can be changed from empty to ovn-cluster") - modifiedSubnet := subnet.DeepCopy() - modifiedSubnet.Spec.Vpc = util.DefaultVpc - _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + subnet = subnetClient.Get(subnetName) + modifiedSubnet := subnet.DeepCopy() + modifiedSubnet.Spec.Vpc = util.DefaultVpc + _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + return err + }) framework.ExpectNoError(err) ginkgo.By("Validating vpc cannot be changed from ovn-cluster to another value") - subnet = subnetClient.Get(subnetName) - modifiedSubnet = subnet.DeepCopy() - modifiedSubnet.Spec.Vpc = "test-vpc" - _, err = subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + subnet = subnetClient.Get(subnetName) + modifiedSubnet := subnet.DeepCopy() + modifiedSubnet.Spec.Vpc = "test-vpc" + _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + return err + }) framework.ExpectError(err, "vpc can only be changed from empty to ovn-cluster") ginkgo.By("Validating vpc cannot be changed from ovn-cluster to empty") - subnet = subnetClient.Get(subnetName) - modifiedSubnet = subnet.DeepCopy() - modifiedSubnet.Spec.Vpc = "" - _, err = subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + subnet = subnetClient.Get(subnetName) + modifiedSubnet := subnet.DeepCopy() + modifiedSubnet.Spec.Vpc = "" + _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + return err + }) framework.ExpectError(err, "vpc can only be changed from empty to ovn-cluster") }) @@ -125,9 +139,13 @@ var _ = framework.Describe("[group:webhook-subnet]", func() { subnet = subnetClient.CreateSync(subnet) ginkgo.By("Validating vpc cannot be changed from empty to non-ovn-cluster value") - modifiedSubnet := subnet.DeepCopy() - modifiedSubnet.Spec.Vpc = "test-vpc" - _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + subnet = subnetClient.Get(subnetName) + modifiedSubnet := subnet.DeepCopy() + modifiedSubnet.Spec.Vpc = "test-vpc" + _, err := subnetClient.SubnetInterface.Update(context.TODO(), modifiedSubnet, metav1.UpdateOptions{}) + return err + }) framework.ExpectError(err, "vpc can only be changed from empty to ovn-cluster") }) })