diff --git a/plugins/meta/sbr/main.go b/plugins/meta/sbr/main.go index c89fdeda6..77793cd78 100644 --- a/plugins/meta/sbr/main.go +++ b/plugins/meta/sbr/main.go @@ -17,6 +17,7 @@ package main import ( "encoding/json" + "errors" "fmt" "log" "net" @@ -34,7 +35,11 @@ import ( bv "github.com/containernetworking/plugins/pkg/utils/buildversion" ) -const firstTableID = 100 +const ( + firstTableID = 100 + // Reserved for SBR interface rules. + sbrInterfaceRuleProtocol = 253 +) // PluginConf is the configuration document passed in. type PluginConf struct { @@ -60,6 +65,10 @@ type PluginConf struct { // enabling destination-based routing when no explicit source IP is specified. // This allows both source-based and destination-based routing to work together. AddSourceHints bool `json:"addSourceHints,omitempty"` + // UseOriginatingIf adds fallback rules for local sockets bound to the interface. + // With automatic allocation, each family uses the SBR table for its first address. + // With an explicit table, the caller provides the routes for every configured family. + UseOriginatingIf bool `json:"useOriginatingIf,omitempty"` } // Wrapper that does a lock before and unlock after operations to serialise @@ -177,9 +186,9 @@ func cmdAdd(args *skel.CmdArgs) error { // Do the actual work. err = withLockAndNetNS(args.Netns, func(_ ns.NetNS) error { if conf.Table != nil { - return doRoutesWithTable(ipCfgs, *conf.Table) + return doRoutesWithTable(ipCfgs, args.IfName, *conf.Table, conf.UseOriginatingIf) } - return doRoutes(ipCfgs, args.IfName, conf.Gateways, conf.AddSourceHints) + return doRoutes(ipCfgs, args.IfName, conf.Gateways, conf.AddSourceHints, conf.UseOriginatingIf) }) if err != nil { return err @@ -218,7 +227,7 @@ func getNextTableID(rules []netlink.Rule, routes []netlink.Route, candidateID in } // doRoutes does all the work to set up routes and rules during an add. -func doRoutes(ipCfgs []*current.IPConfig, iface string, staticGateways []string, addSourceHints bool) error { +func doRoutes(ipCfgs []*current.IPConfig, iface string, staticGateways []string, addSourceHints bool, useOriginatingIf bool) error { // Get a list of rules and routes ready. rules, err := netlinksafe.RuleList(netlink.FAMILY_ALL) if err != nil { @@ -276,6 +285,7 @@ func doRoutes(ipCfgs []*current.IPConfig, iface string, staticGateways []string, } } + var sourceRules []*netlink.Rule // Loop through setting up source based rules and default routes. for _, ipCfg := range ipCfgs { log.Printf("Set rule for source %s", ipCfg.String()) @@ -283,25 +293,28 @@ func doRoutes(ipCfgs []*current.IPConfig, iface string, staticGateways []string, rule.Table = table // Source must be restricted to a single IP, not a full subnet - var src net.IPNet - src.IP = ipCfg.Address.IP - if src.IP.To4() != nil { + src := net.IPNet{IP: ipCfg.Address.IP} + isIPv4 := ipCfg.Address.IP.To4() != nil + if isIPv4 { src.Mask = net.CIDRMask(32, 32) + rule.Family = netlink.FAMILY_V4 // oif has no src address so we set explicitly } else { src.Mask = net.CIDRMask(128, 128) + rule.Family = netlink.FAMILY_V6 } log.Printf("Source to use %s", src.String()) rule.Src = &src - if err = netlink.RuleAdd(rule); err != nil { + if useOriginatingIf { + sourceRules = append(sourceRules, rule) + } else if err = netlink.RuleAdd(rule); err != nil { return fmt.Errorf("Failed to add rule: %v", err) } // Determine which gateway to use: static gateway takes precedence // Match gateway by IP family (IPv4 vs IPv6) var gateway net.IP - isIPv4 := ipCfg.Address.IP.To4() != nil switch { case isIPv4 && staticGwV4 != nil: @@ -318,7 +331,7 @@ func doRoutes(ipCfgs []*current.IPConfig, iface string, staticGateways []string, log.Printf("Adding default route to gateway %s", gateway.String()) var dest net.IPNet - if ipCfg.Address.IP.To4() != nil { + if isIPv4 { dest.IP = net.IPv4zero dest.Mask = net.CIDRMask(0, 32) } else { @@ -372,6 +385,12 @@ func doRoutes(ipCfgs []*current.IPConfig, iface string, staticGateways []string, table = getNextTableID(rules, routes, table) } + if useOriginatingIf { + if err := installInterfaceAndSourceRules(iface, sourceRules); err != nil { + return err + } + } + // Handle routes in the default routing table if addSourceHints { // Keep or re-add subnet routes in main table with source IP hints @@ -431,7 +450,43 @@ func doRoutes(ipCfgs []*current.IPConfig, iface string, staticGateways []string, return nil } -func doRoutesWithTable(ipCfgs []*current.IPConfig, table int) error { +func installInterfaceAndSourceRules(iface string, sourceRules []*netlink.Rule) error { + var interfaceRules []*netlink.Rule + familySeen := make(map[int]bool, 2) + // Create oif rules, based on the first address seen for each family + for _, sourceRule := range sourceRules { + if familySeen[sourceRule.Family] { + continue + } + rule := netlink.NewRule() + rule.Family = sourceRule.Family + rule.Table = sourceRule.Table + rule.OifName = iface + rule.Protocol = sbrInterfaceRuleProtocol + interfaceRules = append(interfaceRules, rule) + familySeen[rule.Family] = true + } + + // Install interface rules first so automatic priorities put source rules + // ahead of the interface rule within each family. + var rules []*netlink.Rule + rules = append(append(rules, interfaceRules...), sourceRules...) + for i, rule := range rules { + if err := netlink.RuleAdd(rule); err != nil { + addErr := fmt.Errorf("failed to add SBR rule %v: %w", rule, err) + for _, added := range rules[:i] { + if err := netlink.RuleDel(added); err != nil { + addErr = errors.Join(addErr, fmt.Errorf("failed to roll back SBR rule %v: %w", added, err)) + } + } + return addErr + } + } + return nil +} + +func doRoutesWithTable(ipCfgs []*current.IPConfig, iface string, table int, useOriginatingIf bool) error { + var sourceRules []*netlink.Rule for _, ipCfg := range ipCfgs { log.Printf("Set rule for source %s", ipCfg.String()) rule := netlink.NewRule() @@ -442,18 +497,25 @@ func doRoutesWithTable(ipCfgs []*current.IPConfig, table int) error { src.IP = ipCfg.Address.IP if src.IP.To4() != nil { src.Mask = net.CIDRMask(32, 32) + rule.Family = netlink.FAMILY_V4 } else { src.Mask = net.CIDRMask(128, 128) + rule.Family = netlink.FAMILY_V6 } log.Printf("Source to use %s", src.String()) rule.Src = &src - if err := netlink.RuleAdd(rule); err != nil { + if useOriginatingIf { + sourceRules = append(sourceRules, rule) + } else if err := netlink.RuleAdd(rule); err != nil { return fmt.Errorf("failed to add rule: %v", err) } } + if useOriginatingIf { + return installInterfaceAndSourceRules(iface, sourceRules) + } return nil } @@ -500,6 +562,21 @@ func tidyRules(iface string, table *int) error { } } + deleteRule := func(rule *netlink.Rule) { + log.Printf("Delete rule %v", rule) + if err := netlink.RuleDel(rule); err != nil { + errReturn = fmt.Errorf("failed to delete rule %v: %w", rule, err) + log.Print(errReturn) + } + } + + // Interface rules are identified by name and may outlive the link. + for _, rule := range rules { + if rule.OifName == iface && rule.Protocol == sbrInterfaceRuleProtocol { + deleteRule(&rule) + } + } + link, err := netlinksafe.LinkByName(iface) if err != nil { // If interface is not found by any reason it's safe to ignore an error. Also, we don't need to raise an error @@ -507,7 +584,7 @@ func tidyRules(iface string, table *int) error { // https://github.com/containernetworking/cni/blob/main/SPEC.md#del-remove-container-from-network-or-un-apply-modifications _, notFound := err.(netlink.LinkNotFoundError) if notFound { - return nil + return errReturn } log.Printf("Failed to get link %s: %v", iface, err) return fmt.Errorf("Failed to get link %s: %v", iface, err) @@ -519,25 +596,17 @@ func tidyRules(iface string, table *int) error { return fmt.Errorf("Failed to list all addrs: %v", err) } -RULE_LOOP: for _, rule := range rules { - log.Printf("Check rule: %v", rule) - if rule.Src == nil { + if rule.Src == nil || rule.OifName != "" { continue } for _, addr := range addrs { if rule.Src.IP.Equal(addr.IP) { - log.Printf("Delete rule %v", rule) - err := netlink.RuleDel(&rule) - if err != nil { - errReturn = fmt.Errorf("Failed to delete rule %v", err) - log.Printf("... Failed! %v", err) - } - continue RULE_LOOP + deleteRule(&rule) + break } } - } return errReturn diff --git a/plugins/meta/sbr/sbr_linux_test.go b/plugins/meta/sbr/sbr_linux_test.go index 1a6b41bbc..a28944916 100644 --- a/plugins/meta/sbr/sbr_linux_test.go +++ b/plugins/meta/sbr/sbr_linux_test.go @@ -974,4 +974,240 @@ var _ = Describe("sbr test", func() { Expect(foundDefaultRouteInTable100).To(BeTrue(), "Expected default route in table 100") }) + + It("adds an oif rule when useOriginatingIf is enabled", func() { + ifname := "net1" + conf := `{ + "cniVersion": "0.3.0", + "name": "cni-plugin-sbr-test", + "type": "sbr", + "useOriginatingIf": true, + "prevResult": { + "cniVersion": "0.3.0", + "interfaces": [ + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "192.168.1.209/24", + "gateway": "192.168.1.1", + "interface": 0 + } + ], + "routes": [] + } +}` + conf = fmt.Sprintf(conf, ifname, targetNs.Path()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: targetNs.Path(), + IfName: ifname, + StdinData: []byte(conf), + } + + err := setup(targetNs, createDefaultStatus()) + Expect(err).NotTo(HaveOccurred()) + + oldStatus, err := readback(targetNs, []string{"net1", "eth0"}) + Expect(err).NotTo(HaveOccurred()) + + _, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred()) + + newStatus, err := readback(targetNs, []string{"net1", "eth0"}) + Expect(err).NotTo(HaveOccurred()) + + // Expect the non-local routes on net1 and the new default route in + // table 100, along with one source rule and one oif rule. + expNet1 := oldStatus.Devices[0] + expEth0 := oldStatus.Devices[1] + for i := range expNet1.Routes { + if expNet1.Routes[i].Table != 255 { + expNet1.Routes[i].Table = 100 + } + } + expNet1.Routes = append(expNet1.Routes, + netlink.Route{ + Gw: net.IPv4(192, 168, 1, 1), + Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.IPMask(net.IPv4zero)}, + Table: 100, + LinkIndex: expNet1.Routes[0].LinkIndex, + }) + + Expect(newStatus.Rules).To(ConsistOf( + And(HaveField("Table", 100), HaveField("OifName", ""), + HaveField("Src.String()", "192.168.1.209/32")), + And(HaveField("Table", 100), HaveField("Family", netlink.FAMILY_V4), + HaveField("OifName", ifname), HaveField("Protocol", uint8(sbrInterfaceRuleProtocol))), + )) + devNet1 := newStatus.Devices[0] + devEth0 := newStatus.Devices[1] + Expect(equalRoutes(expNet1.Routes, devNet1.Routes)).To(BeTrue()) + Expect(equalRoutes(expEth0.Routes, devEth0.Routes)).To(BeTrue()) + + conf = `{ + "cniVersion": "0.3.0", + "name": "cni-plugin-sbr-test", + "type": "sbr" +}` + + // And now check that we can back it all out. + args = &skel.CmdArgs{ + ContainerID: "dummy", + Netns: targetNs.Path(), + IfName: ifname, + StdinData: []byte(conf), + } + err = testutils.CmdDelWithArgs(args, func() error { return cmdDel(args) }) + Expect(err).NotTo(HaveOccurred()) + + retVal, err := readback(targetNs, []string{"net1", "eth0"}) + Expect(err).NotTo(HaveOccurred()) + + // Check results. We expect the rules to have been removed. + Expect(retVal.Rules).To(BeEmpty()) + }) + + It("installs one oif rule per family with lower precedence than its source rules", func() { + ifname := "net1" + conf := fmt.Sprintf(`{ + "cniVersion": "0.3.0", + "name": "cni-plugin-sbr-test", + "type": "sbr", + "useOriginatingIf": true, + "prevResult": { + "cniVersion": "0.3.0", + "interfaces": [{"name": "%s", "sandbox": "%s"}], + "ips": [ + {"version": "6", "address": "fd00::10/64", "gateway": "fd00::1", "interface": 0}, + {"version": "4", "address": "192.168.1.209/24", "gateway": "192.168.1.1", "interface": 0}, + {"version": "4", "address": "192.168.1.210/24", "gateway": "192.168.1.1", "interface": 0} + ] + } + }`, ifname, targetNs.Path()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: targetNs.Path(), + IfName: ifname, + StdinData: []byte(conf), + } + status := createDefaultStatus() + status.Devices[1].Addrs = append(status.Devices[1].Addrs, + net.IPNet{IP: net.ParseIP("fd00::10"), Mask: net.CIDRMask(64, 128)}, + net.IPNet{IP: net.ParseIP("192.168.1.210"), Mask: net.CIDRMask(24, 32)}, + ) + Expect(setup(targetNs, status)).To(Succeed()) + + _, _, err := testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred()) + + result, err := readback(targetNs, []string{ifname, "eth0"}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Rules).To(HaveLen(5)) + for _, expected := range []struct { + family int + table int + sources []string + }{ + {netlink.FAMILY_V6, 100, []string{"fd00::10/128"}}, + {netlink.FAMILY_V4, 101, []string{"192.168.1.209/32", "192.168.1.210/32"}}, + } { + var oifRule netlink.Rule + Expect(result.Rules).To(ContainElement(And( + HaveField("Family", expected.family), HaveField("Table", expected.table), + HaveField("OifName", ifname), HaveField("Src", BeNil()), + HaveField("Protocol", uint8(sbrInterfaceRuleProtocol)), + ), &oifRule)) + for offset, source := range expected.sources { + Expect(result.Rules).To(ContainElement(And( + HaveField("Family", expected.family), HaveField("Table", expected.table+offset), + HaveField("OifName", ""), HaveField("Src.String()", source), + HaveField("Priority", BeNumerically("<", oifRule.Priority)), + ))) + } + } + }) + + It("adds and removes oif rules for both families in an explicit table", func() { + const tableID = 5000 + ifname := "net1" + conf := fmt.Sprintf(`{ + "cniVersion": "0.3.0", + "name": "cni-plugin-sbr-test", + "type": "sbr", + "table": %d, + "useOriginatingIf": true, + "prevResult": { + "cniVersion": "0.3.0", + "interfaces": [{"name": "%s", "sandbox": "%s"}], + "ips": [ + {"version": "6", "address": "fd00::10/64", "interface": 0}, + {"version": "4", "address": "192.168.1.209/24", "interface": 0} + ] + } + }`, tableID, ifname, targetNs.Path()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: targetNs.Path(), + IfName: ifname, + StdinData: []byte(conf), + } + + status := createDefaultStatus() + status.Devices[1].Addrs = append(status.Devices[1].Addrs, + net.IPNet{IP: net.ParseIP("fd00::10"), Mask: net.CIDRMask(64, 128)}) + for _, prefix := range []string{"203.0.113.0/24", "2001:db8::/64"} { + _, dst, err := net.ParseCIDR(prefix) + Expect(err).NotTo(HaveOccurred()) + status.Devices[1].Routes = append(status.Devices[1].Routes, + netlink.Route{Dst: dst, Table: tableID, Scope: netlink.SCOPE_LINK}) + } + Expect(setup(targetNs, status)).To(Succeed()) + before, err := readback(targetNs, []string{ifname, "eth0"}) + Expect(err).NotTo(HaveOccurred()) + + _, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred()) + afterAdd, err := readback(targetNs, []string{ifname, "eth0"}) + Expect(err).NotTo(HaveOccurred()) + Expect(afterAdd.Devices).To(Equal(before.Devices)) + + filter := &netlink.Rule{Table: tableID} + Expect(targetNs.Do(func(_ ns.NetNS) error { + rules, err := netlinksafe.RuleListFiltered(netlink.FAMILY_ALL, filter, netlink.RT_FILTER_TABLE) + Expect(err).NotTo(HaveOccurred()) + Expect(rules).To(ConsistOf( + And(HaveField("OifName", ""), HaveField("Src.String()", "192.168.1.209/32")), + And(HaveField("OifName", ""), HaveField("Src.String()", "fd00::10/128")), + And(HaveField("OifName", ifname), HaveField("Protocol", uint8(sbrInterfaceRuleProtocol)), + HaveField("Family", netlink.FAMILY_V4)), + And(HaveField("OifName", ifname), HaveField("Protocol", uint8(sbrInterfaceRuleProtocol)), + HaveField("Family", netlink.FAMILY_V6)), + )) + // We create this rule just to ensure it's not removed when the plugin is deleted + foreignOifRule := netlink.NewRule() + foreignOifRule.Family = netlink.FAMILY_V4 + foreignOifRule.Table = tableID + foreignOifRule.OifName = ifname + foreignOifRule.Protocol = unix.RTPROT_STATIC + return netlink.RuleAdd(foreignOifRule) + })).To(Succeed()) + + Expect(testutils.CmdDelWithArgs(args, func() error { return cmdDel(args) })).To(Succeed()) + afterDel, err := readback(targetNs, []string{ifname, "eth0"}) + Expect(err).NotTo(HaveOccurred()) + Expect(afterDel.Devices).To(Equal(before.Devices)) + Expect(targetNs.Do(func(_ ns.NetNS) error { + rules, err := netlinksafe.RuleListFiltered(netlink.FAMILY_ALL, filter, netlink.RT_FILTER_TABLE) + Expect(err).NotTo(HaveOccurred()) + Expect(rules).To(ConsistOf( + And(HaveField("OifName", ifname), HaveField("Protocol", uint8(unix.RTPROT_STATIC))), + )) + return nil + })).To(Succeed()) + }) })