Skip to content

Commit

Permalink
Merge pull request #1385 from dcbw/fix-hybrid-overlay-host-route
Browse files Browse the repository at this point in the history 
hybrid-overlay: fix host route to hybrid overlay subnets
  • Loading branch information
@dcbw
dcbw committed Jun 8, 2020
2 parents eeb8d87 + d17f517 commit 249d4cf
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 38 deletions.
2 changes: 0 additions & 2 deletions go-controller/hybrid-overlay/pkg/controller/master_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ import (
. "github.com/onsi/gomega"
)

const hoNodeCliArg string = "-no-hostsubnet-nodes=" + v1.LabelOSStable + "=windows"

func addGetPortAddressesCmds(fexec *ovntest.FakeExec, nodeName, hybMAC, hybIP string) {
addresses := hybMAC + " " + hybIP
addresses = strings.TrimSpace(addresses)
Expand Down
12 changes: 3 additions & 9 deletions go-controller/hybrid-overlay/pkg/controller/node_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -565,13 +565,8 @@ func (n *NodeController) ensureHybridOverlayBridge(node *kapi.Node) error {
n.drMAC = portMAC

// n.drIP is always 3rd address in the subnet
// TODO add support for ipv6 later
portIP := subnet.IP.To4()
if portIP == nil {
return fmt.Errorf("failed to parse local node subnet: %s", subnet.IP)
}
portIP[3] += 3
n.drIP = portIP
hybridOverlayIfAddr := util.GetNodeHybridOverlayIfAddr(subnet)
n.drIP = hybridOverlayIfAddr.IP

_, stderr, err := util.RunOVSVsctl("--may-exist", "add-br", extBridgeName,
"--", "set", "Bridge", extBridgeName, "fail_mode=secure")
Expand Down Expand Up @@ -678,13 +673,12 @@ func (n *NodeController) ensureHybridOverlayBridge(node *kapi.Node) error {
return fmt.Errorf("failed to lookup link %s: %v", util.K8sMgmtIntfName, err)
}
mgmtPortMAC := mgmtPortLink.Attrs().HardwareAddr
hybridOverlayIfAddr := util.GetNodeHybridOverlayIfAddr(subnet)
for _, clusterEntry := range config.HybridOverlay.ClusterSubnets {
route := &netlink.Route{
Dst: clusterEntry.CIDR,
LinkIndex: mgmtPortLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Gw: hybridOverlayIfAddr.IP,
Gw: n.drIP,
}
err := netlink.RouteAdd(route)
if err != nil && !os.IsExist(err) {
Expand Down
65 changes: 38 additions & 27 deletions go-controller/hybrid-overlay/pkg/controller/node_linux_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func createPod(namespace, name, node, podIP, podMAC string) *v1.Pod {
}
}

func addLink(name string) {
func addLink(name string) netlink.Link {
err := netlink.LinkAdd(&netlink.Dummy{
LinkAttrs: netlink.LinkAttrs{
Name: name,
Expand All @@ -122,6 +122,7 @@ func addLink(name string) {
Expect(err).NotTo(HaveOccurred())
err = netlink.LinkSetUp(origLink)
Expect(err).NotTo(HaveOccurred())
return origLink
}

func expectRouteForSubnet(routes []netlink.Route, subnet *net.IPNet, hoIfAddr net.IP) {
Expand All @@ -132,7 +133,7 @@ func expectRouteForSubnet(routes []netlink.Route, subnet *net.IPNet, hoIfAddr ne
break
}
}
Expect(found).To(BeTrue())
Expect(found).To(BeTrue(), fmt.Sprintf("failed to find hybrid overlay host route %s via %s", subnet, hoIfAddr))
}

func validateNetlinkState(nodeSubnet string) {
Expand All @@ -152,6 +153,7 @@ func validateNetlinkState(nodeSubnet string) {
_, ipnet, err := net.ParseCIDR(nodeSubnet)
Expect(err).NotTo(HaveOccurred())
hybridOverlayIfAddr := util.GetNodeHybridOverlayIfAddr(ipnet)
Expect(len(config.HybridOverlay.ClusterSubnets)).ToNot(BeZero())
for _, hoSubnet := range config.HybridOverlay.ClusterSubnets {
expectRouteForSubnet(routes, hoSubnet.CIDR, hybridOverlayIfAddr.IP)
}
Expand All @@ -162,13 +164,25 @@ func appRun(app *cli.App, netns ns.NetNS) {
defer GinkgoRecover()
err := app.Run([]string{
app.Name,
hoNodeCliArg,
"-enable-hybrid-overlay",
"-no-hostsubnet-nodes=" + v1.LabelOSStable + "=windows",
"-cluster-subnets=10.130.0.0/15/24",
})
Expect(err).NotTo(HaveOccurred())
return nil
})
}

func createNodeAnnotationsForSubnet(subnet string) map[string]string {
subnetAnnotations, err := util.CreateNodeHostSubnetAnnotation(ovntest.MustParseIPNets(subnet))
Expect(err).NotTo(HaveOccurred())
annotations := make(map[string]string)
for k, v := range subnetAnnotations {
annotations[k] = fmt.Sprintf("%s", v)
}
return annotations
}

var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
var (
app *cli.App
Expand All @@ -177,7 +191,10 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
f *factory.WatchFactory
stopChan chan struct{}
)
const thisNode string = "mynode"
const (
thisNode string = "mynode"
thisSubnet string = "1.2.3.0/24"
)

BeforeEach(func() {
// Restore global default values before each testcase
Expand All @@ -200,7 +217,14 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
_ = netns.Do(func(ns.NetNS) error {
defer GinkgoRecover()
addLink(extBridgeName)
addLink(util.K8sMgmtIntfName)

// Set up management interface with its address
link := addLink(util.K8sMgmtIntfName)
_, thisNet, err := net.ParseCIDR(thisSubnet)
Expect(err).NotTo(HaveOccurred())
mgmtIfAddr := util.GetNodeManagementIfAddr(thisNet)
err = netlink.AddrAdd(link, &netlink.Addr{IPNet: mgmtIfAddr})
Expect(err).NotTo(HaveOccurred())
return nil
})
})
Expand Down Expand Up @@ -258,13 +282,7 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
node1IP string = "10.0.0.2"
)

subnetAnnotations, err := util.CreateNodeHostSubnetAnnotation(ovntest.MustParseIPNets(node1Subnet))
Expect(err).NotTo(HaveOccurred())
annotations := make(map[string]string)
for k, v := range subnetAnnotations {
annotations[k] = fmt.Sprintf("%s", v)
}

annotations := createNodeAnnotationsForSubnet(node1Subnet)
fakeClient := fake.NewSimpleClientset(&v1.NodeList{
Items: []v1.Node{
*createNode(node1Name, "linux", node1IP, annotations),
Expand All @@ -278,7 +296,7 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
Output: "",
})

_, err = config.InitConfig(ctx, fexec, nil)
_, err := config.InitConfig(ctx, fexec, nil)
Expect(err).NotTo(HaveOccurred())

f, err = factory.NewWatchFactory(fakeClient)
Expand All @@ -292,7 +310,6 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {

//FIXME
//Expect(fexec.CalledMatchesExpected()).To(BeTrue(), fexec.ErrorDesc)
validateNetlinkState(node1Subnet)
return nil
}
appRun(app, netns)
Expand All @@ -301,13 +318,14 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
It("sets up local node hybrid overlay bridge", func() {
app.Action = func(ctx *cli.Context) error {
const (
thisNode string = "mynode"
thisDrMAC string = "22:33:44:55:66:77"
)

annotations := createNodeAnnotationsForSubnet(thisSubnet)
annotations[types.HybridOverlayDRMAC] = thisDrMAC
node := createNode(thisNode, "linux", "10.0.0.1", annotations)
fakeClient := fake.NewSimpleClientset(&v1.NodeList{
Items: []v1.Node{
*createNode(thisNode, "linux", "10.0.0.1", nil),
},
Items: []v1.Node{*node},
})

// Node setup from initial node sync
Expand All @@ -327,11 +345,12 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
n, err := NewNode(&kube.Kube{KClient: fakeClient}, thisNode, stopChan)
Expect(err).NotTo(HaveOccurred())

err = n.startNodeWatch(f)
err = n.ensureHybridOverlayBridge(node)
Expect(err).NotTo(HaveOccurred())

//FIXME
//Expect(fexec.CalledMatchesExpected()).To(BeTrue(), fexec.ErrorDesc)
validateNetlinkState(thisSubnet)
return nil
}
appRun(app, netns)
Expand All @@ -340,7 +359,6 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
It("sets up tunnels for Windows nodes", func() {
app.Action = func(ctx *cli.Context) error {
const (
thisNode string = "mynode"
node1Name string = "node1"
node1IP string = "10.0.0.2"
node1DrMAC string = "22:33:44:55:66:77"
Expand Down Expand Up @@ -380,7 +398,6 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {

//FIXME
//Expect(fexec.CalledMatchesExpected()).To(BeTrue(), fexec.ErrorDesc)
validateNetlinkState(node1Subnet)
return nil
}
appRun(app, netns)
Expand All @@ -389,7 +406,6 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
It("removes stale node flows on initial sync", func() {
app.Action = func(ctx *cli.Context) error {
const (
thisNode string = "mynode"
node1Name string = "node1"
node1IP string = "10.0.0.2"
)
Expand Down Expand Up @@ -433,10 +449,6 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {

It("removes stale pod flows on initial sync", func() {
app.Action = func(ctx *cli.Context) error {
const (
thisNode string = "mynode"
)

fakeClient := fake.NewSimpleClientset()

addNodeSetupCmds(fexec, thisNode)
Expand Down Expand Up @@ -474,7 +486,6 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
It("sets up local pod flows", func() {
app.Action = func(ctx *cli.Context) error {
const (
thisNode string = "mynode"
pod1IP string = "1.2.3.5"
pod1CIDR string = pod1IP + "/24"
pod1MAC string = "aa:bb:cc:dd:ee:ff"
Expand Down

0 comments on commit 249d4cf

Please sign in to comment.