Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 65 additions & 4 deletions plugins/meta/vrf/vrf.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,15 +111,20 @@ func addInterface(vrf *netlink.Vrf, intf string) error {

// Save all routes that are not local and connected, before setting master,
// because otherwise those routes will be deleted after interface is moved.
//
// Only filter by scope here. Multipath (ECMP) routes have LinkIndex == 0 and
// carry per-nexthop ifindexes in MultiPath, so RT_FILTER_OIF would miss them
// (e.g. an IPv6 ::/0 shared with the primary interface). We extract the
// nexthop(s) belonging to this interface below.
filter := &netlink.Route{
LinkIndex: i.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE, // Exclude local and connected routes
Scope: netlink.SCOPE_UNIVERSE, // Exclude local and connected routes
}
filterMask := netlink.RT_FILTER_OIF | netlink.RT_FILTER_SCOPE // Filter based on link index and scope
globalRoutes, err := netlinksafe.RouteListFiltered(netlink.FAMILY_ALL, filter, filterMask)
filterMask := netlink.RT_FILTER_SCOPE
allGlobalRoutes, err := netlinksafe.RouteListFiltered(netlink.FAMILY_ALL, filter, filterMask)
if err != nil {
return fmt.Errorf("failed getting all routes for %s", intf)
}
globalRoutes := routesForInterface(allGlobalRoutes, i.Attrs().Index)

err = netlink.LinkSetMaster(i, vrf)
if err != nil {
Expand Down Expand Up @@ -240,3 +245,59 @@ func getGlobalAddresses(link netlink.Link, family int) ([]netlink.Addr, error) {

return globalAddresses, nil
}

// routesForInterface returns the subset of routes that egress via linkIndex.
// For multipath routes, only nexthops on that interface are kept (converted to a
// single-hop route when only one nexthop remains).
func routesForInterface(routes []netlink.Route, linkIndex int) []netlink.Route {
result := make([]netlink.Route, 0, len(routes))
for _, route := range routes {
if r, ok := routeForInterface(route, linkIndex); ok {
result = append(result, r)
}
}
return result
}

// routeForInterface returns a copy of route that only includes nexthops on
// linkIndex. ok is false when the route does not use that interface.
func routeForInterface(route netlink.Route, linkIndex int) (netlink.Route, bool) {
if len(route.MultiPath) > 0 {
nhs := make([]*netlink.NexthopInfo, 0, len(route.MultiPath))
for _, nh := range route.MultiPath {
if nh == nil || nh.LinkIndex != linkIndex {
continue
}
nhCopy := *nh
nhs = append(nhs, &nhCopy)
}
if len(nhs) == 0 {
return netlink.Route{}, false
}

r := route
if len(nhs) == 1 {
// Prefer a simple single-nexthop form; this is what IPAM originally

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please preserve nexthop flags when converting existing multipath routes and add a regression test for this case.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, collapsing that to one hop dropped onlink. the nexthop flags are copied onto the route now, and there is a regression for this case.

// installed before the kernel merged it into an ECMP route.
// onlink and the other nexthop flags live on the nexthop, not the
// multipath route header, so copy them onto the single-hop route.
r.MultiPath = nil
r.LinkIndex = nhs[0].LinkIndex
r.Gw = nhs[0].Gw
r.NewDst = nhs[0].NewDst
r.Encap = nhs[0].Encap
r.Via = nhs[0].Via
r.Flags |= nhs[0].Flags
} else {
r.MultiPath = nhs
r.LinkIndex = 0
r.Gw = nil
}
return r, true
}

if route.LinkIndex == linkIndex {
return route, true
}
return netlink.Route{}, false
}
290 changes: 290 additions & 0 deletions plugins/meta/vrf/vrf_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,143 @@ var _ = Describe("vrf plugin", func() {
})
})

It("moves IPv6 ECMP default nexthop into the VRF table", func() {
// Reproduce #1253: primary interface already has ::/0, IPAM adds another
// ::/0 via the secondary interface, kernel merges them into multipath.
// VRF must still move only the secondary nexthop into the VRF table.
conf := configFor("test", IF1Name, VRF0Name, "10.0.0.2/24")

By("Creating an IPv6 ECMP default route spanning both interfaces", func() {
err := targetNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()

link0, err := netlinksafe.LinkByName(IF0Name)
Expect(err).NotTo(HaveOccurred())
link1, err := netlinksafe.LinkByName(IF1Name)
Expect(err).NotTo(HaveOccurred())

addr0, err := types.ParseCIDR("2001:db8:0::2/64")
Expect(err).NotTo(HaveOccurred())
addr1, err := types.ParseCIDR("2001:db8:1::2/64")
Expect(err).NotTo(HaveOccurred())

Expect(netlink.AddrAdd(link0, &netlink.Addr{IPNet: addr0})).To(Succeed())
Expect(netlink.AddrAdd(link1, &netlink.Addr{IPNet: addr1})).To(Succeed())
Expect(netlink.LinkSetUp(link0)).To(Succeed())
Expect(netlink.LinkSetUp(link1)).To(Succeed())

// Wait for kernel host routes for the IPv6 addresses.
Eventually(func() bool {
routes, _ := netlinksafe.RouteListFiltered(netlink.FAMILY_V6, &netlink.Route{
Dst: &net.IPNet{
IP: addr1.IP,
Mask: net.CIDRMask(128, 128),
},
Table: 0,
}, netlink.RT_FILTER_DST|netlink.RT_FILTER_TABLE)
return len(routes) >= 1
}, time.Second, 100*time.Millisecond).Should(BeTrue())

defaultDst := &net.IPNet{
IP: net.IPv6zero,
Mask: net.CIDRMask(0, 128),
}
// Install multipath default explicitly. Two sequential RouteAdd
// calls with the same metric return EEXIST on some kernels
// instead of merging into ECMP (repro path from #1253).
Expect(netlink.RouteAdd(&netlink.Route{
Dst: defaultDst,
Priority: 1024,
MultiPath: []*netlink.NexthopInfo{
{LinkIndex: link0.Attrs().Index, Gw: net.ParseIP("2001:db8:0::1")},
{LinkIndex: link1.Attrs().Index, Gw: net.ParseIP("2001:db8:1::1")},
},
})).To(Succeed())

// Confirm the kernel has a multipath default including both ifaces.
routes, err := netlinksafe.RouteListFiltered(netlink.FAMILY_V6, &netlink.Route{
Dst: defaultDst,
Table: 0,
}, netlink.RT_FILTER_DST|netlink.RT_FILTER_TABLE)
Expect(err).NotTo(HaveOccurred())
Expect(routes).NotTo(BeEmpty())
// Either a multipath route or two separate defaults (depends on kernel);
// at least one must reference the secondary interface.
hasIF1 := false
for _, r := range routes {
if r.LinkIndex == link1.Attrs().Index {
hasIF1 = true
}
for _, nh := range r.MultiPath {
if nh.LinkIndex == link1.Attrs().Index {
hasIF1 = true
}
}
}
Expect(hasIF1).To(BeTrue(), "expected a default route nexthop on %s", IF1Name)
return nil
})
Expect(err).NotTo(HaveOccurred())
})

By("Adding the secondary interface to the VRF", func() {
err := originalNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
args := &skel.CmdArgs{
ContainerID: "dummy",
Netns: targetNS.Path(),
IfName: IF1Name,
StdinData: conf,
}
_, _, err := testutils.CmdAddWithArgs(args, func() error {
return cmdAdd(args)
})
Expect(err).NotTo(HaveOccurred())
return nil
})
Expect(err).NotTo(HaveOccurred())
})

By("Checking the IPv6 default via the secondary interface is in the VRF table", func() {
err := targetNS.Do(func(ns.NetNS) error {
defer GinkgoRecover()
checkInterfaceOnVRF(VRF0Name, IF1Name)

vrfLink, err := netlinksafe.LinkByName(VRF0Name)
Expect(err).NotTo(HaveOccurred())
vrf := vrfLink.(*netlink.Vrf)
link1, err := netlinksafe.LinkByName(IF1Name)
Expect(err).NotTo(HaveOccurred())

defaultDst := &net.IPNet{
IP: net.IPv6zero,
Mask: net.CIDRMask(0, 128),
}
routes, err := netlinksafe.RouteListFiltered(netlink.FAMILY_V6, &netlink.Route{
Dst: defaultDst,
Table: int(vrf.Table),
}, netlink.RT_FILTER_DST|netlink.RT_FILTER_TABLE)
Expect(err).NotTo(HaveOccurred())
Expect(routes).NotTo(BeEmpty(), "expected ::/0 in VRF table %d", vrf.Table)

found := false
for _, r := range routes {
if r.LinkIndex == link1.Attrs().Index && r.Gw.Equal(net.ParseIP("2001:db8:1::1")) {
found = true
}
for _, nh := range r.MultiPath {
if nh.LinkIndex == link1.Attrs().Index && nh.Gw.Equal(net.ParseIP("2001:db8:1::1")) {
found = true
}
}
}
Expect(found).To(BeTrue(), "expected ::/0 via 2001:db8:1::1 dev %s in VRF table", IF1Name)
return nil
})
Expect(err).NotTo(HaveOccurred())
})
})

It("fails if the interface already has a master set", func() {
conf := configFor("test", IF0Name, VRF0Name, "10.0.0.2/24")

Expand Down Expand Up @@ -932,6 +1069,159 @@ var _ = Describe("unit tests", func() {
return res
}(), uint32(1000), false),
)

Describe("routeForInterface", func() {
ipv6Default := &net.IPNet{
IP: net.IPv6zero,
Mask: net.CIDRMask(0, 128),
}
const (
eth0Index = 2
net1Index = 5
)

It("keeps a single-hop route on the target interface", func() {
route := netlink.Route{
LinkIndex: net1Index,
Dst: ipv6Default,
Gw: net.ParseIP("2001:db8:1::1"),
Priority: 1024,
Scope: netlink.SCOPE_UNIVERSE,
}

got, ok := routeForInterface(route, net1Index)
Expect(ok).To(BeTrue())
Expect(got).To(Equal(route))
})

It("rejects a single-hop route on another interface", func() {
route := netlink.Route{
LinkIndex: eth0Index,
Dst: ipv6Default,
Gw: net.ParseIP("fe80::1"),
Priority: 1024,
Scope: netlink.SCOPE_UNIVERSE,
}

_, ok := routeForInterface(route, net1Index)
Expect(ok).To(BeFalse())
})

It("extracts only the target nexthop from an IPv6 ECMP default route", func() {
// Simulates main-table ::/0 after IPAM added a second default via net1
// while eth0 already had one, so the kernel merged them into multipath.
route := netlink.Route{
LinkIndex: 0,
Dst: ipv6Default,
Priority: 1024,
Scope: netlink.SCOPE_UNIVERSE,
MultiPath: []*netlink.NexthopInfo{
{LinkIndex: eth0Index, Gw: net.ParseIP("fe80::1"), Hops: 0},
{LinkIndex: net1Index, Gw: net.ParseIP("2001:db8:1::1"), Hops: 0},
},
}

got, ok := routeForInterface(route, net1Index)
Expect(ok).To(BeTrue())
Expect(got.MultiPath).To(BeNil())
Expect(got.LinkIndex).To(Equal(net1Index))
Expect(got.Gw.Equal(net.ParseIP("2001:db8:1::1"))).To(BeTrue())
Expect(got.Dst.String()).To(Equal("::/0"))
Expect(got.Priority).To(Equal(1024))
})

It("keeps nexthop flags when a multipath route collapses to one hop", func() {
route := netlink.Route{
LinkIndex: 0,
Dst: ipv6Default,
Priority: 1024,
Flags: int(netlink.FLAG_PERVASIVE),
MultiPath: []*netlink.NexthopInfo{
{LinkIndex: eth0Index, Gw: net.ParseIP("fe80::1")},
{
LinkIndex: net1Index,
Gw: net.ParseIP("2001:db8:1::1"),
Flags: int(netlink.FLAG_ONLINK),
},
},
}

got, ok := routeForInterface(route, net1Index)
Expect(ok).To(BeTrue())
Expect(got.MultiPath).To(BeNil())
Expect(got.LinkIndex).To(Equal(net1Index))
Expect(got.Gw.Equal(net.ParseIP("2001:db8:1::1"))).To(BeTrue())
Expect(got.Flags & int(netlink.FLAG_ONLINK)).To(Equal(int(netlink.FLAG_ONLINK)))
Expect(got.Flags & int(netlink.FLAG_PERVASIVE)).To(Equal(int(netlink.FLAG_PERVASIVE)))
})

It("rejects multipath routes with no nexthop on the target interface", func() {
route := netlink.Route{
Dst: ipv6Default,
Priority: 1024,
MultiPath: []*netlink.NexthopInfo{
{LinkIndex: eth0Index, Gw: net.ParseIP("fe80::1")},
},
}

_, ok := routeForInterface(route, net1Index)
Expect(ok).To(BeFalse())
})

It("keeps multiple nexthops when more than one is on the target interface", func() {
route := netlink.Route{
Dst: ipv6Default,
MultiPath: []*netlink.NexthopInfo{
{LinkIndex: net1Index, Gw: net.ParseIP("2001:db8:1::1"), Hops: 0},
{LinkIndex: eth0Index, Gw: net.ParseIP("fe80::1"), Hops: 0},
{LinkIndex: net1Index, Gw: net.ParseIP("2001:db8:1::2"), Hops: 0},
},
}

got, ok := routeForInterface(route, net1Index)
Expect(ok).To(BeTrue())
Expect(got.MultiPath).To(HaveLen(2))
Expect(got.LinkIndex).To(Equal(0))
Expect(got.Gw).To(BeNil())
Expect(got.MultiPath[0].Gw.Equal(net.ParseIP("2001:db8:1::1"))).To(BeTrue())
Expect(got.MultiPath[1].Gw.Equal(net.ParseIP("2001:db8:1::2"))).To(BeTrue())
})

It("filters a mixed list down to routes for the target interface", func() {
routes := []netlink.Route{
{
LinkIndex: eth0Index,
Dst: ipv6Default,
Gw: net.ParseIP("fe80::1"),
},
{
LinkIndex: net1Index,
Dst: &net.IPNet{
IP: net.ParseIP("2001:db8:1::"),
Mask: net.CIDRMask(64, 128),
},
Gw: net.ParseIP("2001:db8:1::1"),
},
{
LinkIndex: 0,
Dst: ipv6Default,
Priority: 1024,
MultiPath: []*netlink.NexthopInfo{
{LinkIndex: eth0Index, Gw: net.ParseIP("fe80::1")},
{LinkIndex: net1Index, Gw: net.ParseIP("2001:db8:1::1")},
},
},
}

got := routesForInterface(routes, net1Index)
Expect(got).To(HaveLen(2))
Expect(got[0].LinkIndex).To(Equal(net1Index))
Expect(got[0].Dst.String()).To(Equal("2001:db8:1::/64"))
Expect(got[1].LinkIndex).To(Equal(net1Index))
Expect(got[1].Gw.Equal(net.ParseIP("2001:db8:1::1"))).To(BeTrue())
Expect(got[1].MultiPath).To(BeNil())
})
})
})

func configFor(name, intf, vrf, ip string) []byte {
Expand Down
Loading