feat(NSC): use EndpointSlice instead of Endpoints

With the advent of IPv6 integrated into the NSC we no longer get all IPs from endpoints, but rather just the primary IP of the pod (which is often, but not always the IPv4 address). In order to get all possible endpoint addresses for a given service we need to switch to using EndpointSlice which also nicely groups addresses into IPv4 and IPv6 by AddressType and also gives us more information about the endpoint status by giving us attributes for serving and terminating, instead of just ready or not ready. This does mean that users will need to add another permission to their RBAC in order for kube-router to access these objects.
cloudnativelabs · Sep 23, 2023 · 327e894 · 327e894
1 parent f8d9812
commit 327e894
Show file tree

Hide file tree

Showing 14 changed files with 299 additions and 73 deletions.
diff --git a/daemonset/generic-kuberouter-all-features-advertise-routes.yaml b/daemonset/generic-kuberouter-all-features-advertise-routes.yaml
@@ -224,6 +224,14 @@ rules:
       - services/status
     verbs:
       - update
+  - apiGroups:
+      - "discovery.k8s.io"
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
 
 ---
 kind: ClusterRoleBinding

diff --git a/daemonset/generic-kuberouter-all-features.yaml b/daemonset/generic-kuberouter-all-features.yaml
@@ -220,6 +220,14 @@ rules:
       - services/status
     verbs:
       - update
+  - apiGroups:
+      - "discovery.k8s.io"
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

diff --git a/daemonset/generic-kuberouter-only-advertise-routes.yaml b/daemonset/generic-kuberouter-only-advertise-routes.yaml
@@ -132,7 +132,15 @@ rules:
       - services/status
     verbs:
       - update
-
+  - apiGroups:
+      - "discovery.k8s.io"
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
+
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

diff --git a/daemonset/generic-kuberouter.yaml b/daemonset/generic-kuberouter.yaml
@@ -187,7 +187,15 @@ rules:
       - services/status
     verbs:
       - update
-
+  - apiGroups:
+      - "discovery.k8s.io"
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
+
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

diff --git a/daemonset/kubeadm-kuberouter-all-features-dsr.yaml b/daemonset/kubeadm-kuberouter-all-features-dsr.yaml
@@ -203,6 +203,15 @@ rules:
       - services/status
     verbs:
       - update
+  - apiGroups:
+      - "discovery.k8s.io"
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
+
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

diff --git a/daemonset/kubeadm-kuberouter-all-features-hostport.yaml b/daemonset/kubeadm-kuberouter-all-features-hostport.yaml
@@ -202,6 +202,15 @@ rules:
       - services/status
     verbs:
       - update
+  - apiGroups:
+      - "discovery.k8s.io"
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
+
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

diff --git a/daemonset/kubeadm-kuberouter-all-features.yaml b/daemonset/kubeadm-kuberouter-all-features.yaml
@@ -195,6 +195,15 @@ rules:
       - services/status
     verbs:
       - update
+  - apiGroups:
+      - "discovery.k8s.io"
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
+
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

diff --git a/daemonset/kubeadm-kuberouter.yaml b/daemonset/kubeadm-kuberouter.yaml
@@ -191,6 +191,15 @@ rules:
       - services/status
     verbs:
       - update
+  - apiGroups:
+      - "discovery.k8s.io"
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
+
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

diff --git a/pkg/cmd/kube-router.go b/pkg/cmd/kube-router.go
@@ -97,6 +97,7 @@ func (kr *KubeRouter) Run() error {
 	informerFactory := informers.NewSharedInformerFactory(kr.Client, 0)
 	svcInformer := informerFactory.Core().V1().Services().Informer()
 	epInformer := informerFactory.Core().V1().Endpoints().Informer()
+	epSliceInformer := informerFactory.Discovery().V1().EndpointSlices().Informer()
 	podInformer := informerFactory.Core().V1().Pods().Informer()
 	nodeInformer := informerFactory.Core().V1().Nodes().Informer()
 	nsInformer := informerFactory.Core().V1().Namespaces().Informer()
@@ -177,7 +178,7 @@ func (kr *KubeRouter) Run() error {
 
 	if kr.Config.RunServiceProxy {
 		nsc, err := proxy.NewNetworkServicesController(kr.Client, kr.Config,
-			svcInformer, epInformer, podInformer, &ipsetMutex)
+			svcInformer, epSliceInformer, podInformer, &ipsetMutex)
 		if err != nil {
 			return fmt.Errorf("failed to create network services controller: %v", err)
 		}
@@ -186,7 +187,7 @@ func (kr *KubeRouter) Run() error {
 		if err != nil {
 			return fmt.Errorf("failed to add ServiceEventHandler: %v", err)
 		}
-		_, err = epInformer.AddEventHandler(nsc.EndpointsEventHandler)
+		_, err = epSliceInformer.AddEventHandler(nsc.EndpointSliceEventHandler)
 		if err != nil {
 			return fmt.Errorf("failed to add EndpointsEventHandler: %v", err)
 		}