Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bug 1956318: Revert "removing the hybrid overlay externalGW code" #521

Merged
merged 4 commits into from May 6, 2021
Merged

Bug 1956318: Revert "removing the hybrid overlay externalGW code" #521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8aee9e9
Revert "removing the hybrid overlay externalGW code"
trozet Apr 22, 2021
610952c
Fix egressfirewall dns test regression
trozet May 3, 2021
ef89b95
Fix linter errors in OCP Hacks
abhat Apr 9, 2021
185aebc
Fixes unit test for OF flows with geneve tunnel port
trozet May 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
139 changes: 134 additions & 5 deletions go-controller/hybrid-overlay/pkg/controller/master.go
View file
Open in desktop
Expand Up @@ -4,9 +4,11 @@ import (
"fmt"
"net"
"sync"
"time"

goovn "github.com/ebay/go-ovn"
"github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types"
hotypes "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types"
houtil "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/util"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/informer"
Expand All @@ -15,19 +17,25 @@ import (
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"

kapi "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
listers "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
utilnet "k8s.io/utils/net"
)

// MasterController is the master hybrid overlay controller
type MasterController struct {
kube kube.Interface
allocator *subnetallocator.SubnetAllocator
nodeEventHandler informer.EventHandler
ovnNBClient goovn.Client
ovnSBClient goovn.Client
kube kube.Interface
allocator *subnetallocator.SubnetAllocator
nodeEventHandler informer.EventHandler
namespaceEventHandler informer.EventHandler
podEventHandler informer.EventHandler
ovnNBClient goovn.Client
ovnSBClient goovn.Client
}

// NewMaster a new master controller that listens for node events
Expand Down Expand Up @@ -65,6 +73,36 @@ func NewMaster(kube kube.Interface,
informer.ReceiveAllUpdates,
)

m.namespaceEventHandler = eventHandlerCreateFunction("namespace", namespaceInformer,
func(obj interface{}) error {
ns, ok := obj.(*kapi.Namespace)
if !ok {
return fmt.Errorf("object is not a namespace")
}
return m.AddNamespace(ns)
},
func(obj interface{}) error {
// discard deletes
return nil
},
nsHybridAnnotationChanged,
)

m.podEventHandler = eventHandlerCreateFunction("pod", podInformer,
func(obj interface{}) error {
pod, ok := obj.(*kapi.Pod)
if !ok {
return fmt.Errorf("object is not a pod")
}
return m.AddPod(pod)
},
func(obj interface{}) error {
// discard deletes
return nil
},
informer.DiscardAllUpdates,
)

// Add our hybrid overlay CIDRs to the subnetallocator
for _, clusterEntry := range config.HybridOverlay.ClusterSubnets {
err := m.allocator.AddNetworkRange(clusterEntry.CIDR, clusterEntry.HostSubnetLength)
Expand Down Expand Up @@ -107,6 +145,22 @@ func (m *MasterController) Run(stopCh <-chan struct{}) {
klog.Error(err)
}
}()
wg.Add(1)
go func() {
defer wg.Done()
err := m.namespaceEventHandler.Run(informer.DefaultInformerThreadiness, stopCh)
if err != nil {
klog.Error(err)
}
}()
wg.Add(1)
go func() {
defer wg.Done()
err := m.podEventHandler.Run(informer.DefaultInformerThreadiness, stopCh)
if err != nil {
klog.Error(err)
}
}()
<-stopCh
klog.Info("Shutting down Hybrid Overlay Master workers")
wg.Wait()
Expand Down Expand Up @@ -287,3 +341,78 @@ func (m *MasterController) DeleteNode(node *kapi.Node) error {
klog.V(5).Infof("Node delete for %s completed", node.Name)
return nil
}

// AddNamespace copies namespace annotations to all pods in the namespace
func (m *MasterController) AddNamespace(ns *kapi.Namespace) error {
podLister := listers.NewPodLister(m.podEventHandler.GetIndexer())
pods, err := podLister.Pods(ns.Name).List(labels.Everything())
if err != nil {
return err
}
for _, pod := range pods {
if err := houtil.CopyNamespaceAnnotationsToPod(m.kube, ns, pod); err != nil {
klog.Errorf("Unable to copy hybrid-overlay namespace annotations to pod %s", pod.Name)
}
}
return nil
}

// waitForNamespace fetches a namespace from the cache, or waits until it's available
func (m *MasterController) waitForNamespace(name string) (*kapi.Namespace, error) {
var namespaceBackoff = wait.Backoff{Duration: 1 * time.Second, Steps: 7, Factor: 1.5, Jitter: 0.1}
var namespace *kapi.Namespace
if err := wait.ExponentialBackoff(namespaceBackoff, func() (bool, error) {
var err error
namespaceLister := listers.NewNamespaceLister(m.namespaceEventHandler.GetIndexer())
namespace, err = namespaceLister.Get(name)
if err != nil {
if errors.IsNotFound(err) {
// Namespace not found; retry
return false, nil
}
klog.Warningf("Error getting namespace: %v", err)
return false, err
}
return true, nil
}); err != nil {
return nil, fmt.Errorf("failed to get namespace object: %v", err)
}
return namespace, nil
}

// AddPod ensures that hybrid overlay annotations are copied to a
// pod when it's created. This allows the nodes to set up the appropriate
// flows
func (m *MasterController) AddPod(pod *kapi.Pod) error {
namespace, err := m.waitForNamespace(pod.Namespace)
if err != nil {
return err
}

namespaceExternalGw := namespace.Annotations[hotypes.HybridOverlayExternalGw]
namespaceVTEP := namespace.Annotations[hotypes.HybridOverlayVTEP]

podExternalGw := pod.Annotations[hotypes.HybridOverlayExternalGw]
podVTEP := pod.Annotations[hotypes.HybridOverlayVTEP]

if namespaceExternalGw != podExternalGw || namespaceVTEP != podVTEP {
// copy namespace annotations to the pod and return
return houtil.CopyNamespaceAnnotationsToPod(m.kube, namespace, pod)
}
return nil
}

// nsHybridAnnotationChanged returns true if any relevant NS attributes changed
func nsHybridAnnotationChanged(old, new interface{}) bool {
oldNs := old.(*kapi.Namespace)
newNs := new.(*kapi.Namespace)

nsExGwOld := oldNs.GetAnnotations()[hotypes.HybridOverlayExternalGw]
nsVTEPOld := oldNs.GetAnnotations()[hotypes.HybridOverlayVTEP]
nsExGwNew := newNs.GetAnnotations()[hotypes.HybridOverlayExternalGw]
nsVTEPNew := newNs.GetAnnotations()[hotypes.HybridOverlayVTEP]
if nsExGwOld != nsExGwNew || nsVTEPOld != nsVTEPNew {
return true
}
return false
}
187 changes: 187 additions & 0 deletions go-controller/hybrid-overlay/pkg/controller/master_test.go
View file
Open in desktop
Expand Up @@ -3,13 +3,16 @@ package controller
import (
"context"
"fmt"
"reflect"
"strings"
"sync"

goovn "github.com/ebay/go-ovn"
"github.com/urfave/cli/v2"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
k8stypes "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"

Expand Down Expand Up @@ -373,6 +376,190 @@ var _ = Describe("Hybrid SDN Master Operations", func() {
})
Expect(err).NotTo(HaveOccurred())
})

It("copies namespace annotations when a pod is added", func() {
app.Action = func(ctx *cli.Context) error {
const (
nsName string = "nstest"
nsVTEP = "1.1.1.1"
nsExGw = "2.2.2.2"
nodeName string = "node1"
nodeSubnet string = "10.1.2.0/24"
nodeHOIP string = "10.1.2.3"
nodeHOMAC string = "00:00:00:52:19:d2"
pod1Name string = "pod1"
pod1IP string = "1.2.3.5"
pod1CIDR string = pod1IP + "/24"
pod1MAC string = "aa:bb:cc:dd:ee:ff"
)

ns := &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
UID: k8stypes.UID(nsName),
Name: nsName,
Annotations: map[string]string{
hotypes.HybridOverlayVTEP: nsVTEP,
hotypes.HybridOverlayExternalGw: nsExGw,
},
},
Spec: v1.NamespaceSpec{},
Status: v1.NamespaceStatus{},
}
fakeClient := fake.NewSimpleClientset([]runtime.Object{
ns,
createPod(nsName, pod1Name, nodeName, pod1CIDR, pod1MAC),
&v1.NodeList{Items: []v1.Node{newTestNode(nodeName, "linux", nodeSubnet, "", nodeHOMAC)}},
}...)

_, err := config.InitConfig(ctx, nil, nil)
Expect(err).NotTo(HaveOccurred())

f := informers.NewSharedInformerFactory(fakeClient, informer.DefaultResyncInterval)
mockOVNNBClient := ovntest.NewMockOVNClient(goovn.DBNB)
mockOVNSBClient := ovntest.NewMockOVNClient(goovn.DBSB)
m, err := NewMaster(
&kube.Kube{KClient: fakeClient},
f.Core().V1().Nodes().Informer(),
f.Core().V1().Namespaces().Informer(),
f.Core().V1().Pods().Informer(),
mockOVNNBClient,
mockOVNSBClient,
informer.NewTestEventHandler,
)
Expect(err).NotTo(HaveOccurred())

populatePortAddresses(nodeName, nodeHOMAC, nodeHOIP, mockOVNNBClient)

f.Start(stopChan)
wg.Add(1)
go func() {
defer wg.Done()
m.Run(stopChan)
}()
f.WaitForCacheSync(stopChan)

Eventually(func() error {
pod, err := fakeClient.CoreV1().Pods(nsName).Get(context.TODO(), pod1Name, metav1.GetOptions{})
if err != nil {
return err
}
if pod.Annotations[hotypes.HybridOverlayVTEP] != nsVTEP {
return fmt.Errorf("error with annotation %s. expected: %s, got: %s", hotypes.HybridOverlayVTEP, nsVTEP, pod.Annotations[hotypes.HybridOverlayVTEP])
}
if pod.Annotations[hotypes.HybridOverlayExternalGw] != nsExGw {
return fmt.Errorf("error with annotation %s. expected: %s, got: %s", hotypes.HybridOverlayVTEP, nsExGw, pod.Annotations[hotypes.HybridOverlayExternalGw])
}
return nil
}, 2).Should(Succeed())

return nil
}

err := app.Run([]string{
app.Name,
"-loglevel=5",
"-enable-hybrid-overlay",
"-hybrid-overlay-cluster-subnets=" + hybridOverlayClusterCIDR,
})
Expect(err).NotTo(HaveOccurred())
})

It("update pod annotations when a namespace is updated", func() {
app.Action = func(ctx *cli.Context) error {
const (
nsName string = "nstest"
nsVTEP string = "1.1.1.1"
nsVTEPUpdated string = "3.3.3.3"
nsExGw string = "2.2.2.2"
nsExGwUpdated string = "4.4.4.4"
nodeName string = "node1"
nodeSubnet string = "10.1.2.0/24"
nodeHOMAC string = "00:00:00:52:19:d2"
nodeHOIP string = "10.1.2.3"
pod1Name string = "pod1"
pod1IP string = "1.2.3.5"
pod1CIDR string = pod1IP + "/24"
pod1MAC string = "aa:bb:cc:dd:ee:ff"
)

ns := &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
UID: k8stypes.UID(nsName),
Name: nsName,
Annotations: map[string]string{
hotypes.HybridOverlayVTEP: nsVTEP,
hotypes.HybridOverlayExternalGw: nsExGw,
},
},
Spec: v1.NamespaceSpec{},
Status: v1.NamespaceStatus{},
}
fakeClient := fake.NewSimpleClientset([]runtime.Object{
ns,
&v1.NodeList{Items: []v1.Node{newTestNode(nodeName, "linux", nodeSubnet, "", nodeHOMAC)}},
createPod(nsName, pod1Name, nodeName, pod1CIDR, pod1MAC),
}...)

addLinuxNodeCommands(fexec, nodeHOMAC, nodeName, nodeHOIP)
_, err := config.InitConfig(ctx, nil, nil)
Expect(err).NotTo(HaveOccurred())
f := informers.NewSharedInformerFactory(fakeClient, informer.DefaultResyncInterval)

k := &kube.Kube{KClient: fakeClient}
mockOVNNBClient := ovntest.NewMockOVNClient(goovn.DBNB)
mockOVNSBClient := ovntest.NewMockOVNClient(goovn.DBSB)
m, err := NewMaster(
k,
f.Core().V1().Nodes().Informer(),
f.Core().V1().Namespaces().Informer(),
f.Core().V1().Pods().Informer(),
mockOVNNBClient,
mockOVNSBClient,
informer.NewTestEventHandler,
)
Expect(err).NotTo(HaveOccurred())

f.Start(stopChan)
wg.Add(1)
go func() {
defer wg.Done()
m.Run(stopChan)
}()
f.WaitForCacheSync(stopChan)

updatedNs, err := fakeClient.CoreV1().Namespaces().Get(context.TODO(), nsName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
nsAnnotator := kube.NewNamespaceAnnotator(k, updatedNs)
nsAnnotator.Set(hotypes.HybridOverlayVTEP, nsVTEPUpdated)
nsAnnotator.Set(hotypes.HybridOverlayExternalGw, nsExGwUpdated)
err = nsAnnotator.Run()
Expect(err).NotTo(HaveOccurred())

Eventually(func() error {
pod, err := fakeClient.CoreV1().Pods(nsName).Get(context.TODO(), pod1Name, metav1.GetOptions{})
if err != nil {
return err
}
if reflect.DeepEqual(pod.Annotations[hotypes.HybridOverlayVTEP], nsVTEP) {
return fmt.Errorf("error with annotation %s. expected: %s, got: %s", hotypes.HybridOverlayVTEP, nsVTEPUpdated, pod.Annotations[hotypes.HybridOverlayVTEP])
}
if reflect.DeepEqual(pod.Annotations[hotypes.HybridOverlayExternalGw], nsExGw) {
return fmt.Errorf("error with annotation %s. expected: %s, got: %s", hotypes.HybridOverlayExternalGw, nsExGwUpdated, pod.Annotations[hotypes.HybridOverlayExternalGw])
}
return nil
}, 2).Should(Succeed())

return nil
}

err := app.Run([]string{
app.Name,
"-loglevel=5",
"-enable-hybrid-overlay",
"-hybrid-overlay-cluster-subnets=" + hybridOverlayClusterCIDR,
})
Expect(err).NotTo(HaveOccurred())
})
})

func addLinuxNodeCommands(fexec *ovntest.FakeExec, nodeHOMAC, nodeName, nodeHOIP string) {
Expand Down