l2: exclude common virtual interfaces for announce services

In arp mode, Some common virtual interfaces should not be used to announce services, such as kube-ipvs0, docker0, etc. This PR will dispatch these common virtual interfaces. Signed-off-by: cyclinder <qifeng.guo@daocloud.io>
metallb · Mar 14, 2023 · 98eebdd · 98eebdd
1 parent 6a5ad67
commit 98eebdd
Show file tree

Hide file tree

Showing 14 changed files with 182 additions and 11 deletions.
diff --git a/charts/metallb/README.md b/charts/metallb/README.md
@@ -109,6 +109,7 @@ Kubernetes: `>= 1.19.0-0`
 | rbac.create | bool | `true` |  |
 | speaker.affinity | object | `{}` |  |
 | speaker.enabled | bool | `true` |  |
+| speaker.excludeInterfaces.enabled | bool | `true` |  |
 | speaker.frr.enabled | bool | `false` |  |
 | speaker.frr.image.pullPolicy | string | `nil` |  |
 | speaker.frr.image.repository | string | `"quay.io/frrouting/frr"` |  |

diff --git a/charts/metallb/templates/confgmap.yaml b/charts/metallb/templates/confgmap.yaml
@@ -0,0 +1,22 @@
+{{- if .Values.speaker.excludeInterfaces.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: metallb-conf
+data:
+  conf.yaml: |
+    announcedInterfacesToExclude:
+    - docker.*
+    - cbr.*
+    - dummy.*
+    - virbr.*
+    - lxcbr.*
+    - veth.*
+    - lo
+    - ^cali.*
+    - ^tunl.*
+    - flannel.*
+    - kube-ipvs.*
+    - cni.*
+    - ^nodelocaldns.*
+{{- end }}
diff --git a/charts/metallb/templates/speaker.yaml b/charts/metallb/templates/speaker.yaml
@@ -153,6 +153,12 @@ spec:
             secretName: {{ include "metallb.secretName" . }}
             defaultMode: 420
       {{- end }}
+      {{- if .Values.speaker.excludeInterfaces.enabled }}
+        - name: metallb-conf
+          configMap:
+            defaultMode: 256
+            name: metallb-conf
+      {{- end }}
       {{- if .Values.speaker.frr.enabled }}
         - name: frr-sockets
           emptyDir: {}
@@ -291,7 +297,7 @@ spec:
             - ALL
             add:
             - NET_RAW
-        {{- if or .Values.speaker.frr.enabled .Values.speaker.memberlist.enabled }}
+        {{- if or .Values.speaker.frr.enabled .Values.speaker.memberlist.enabled .Values.speaker.excludeInterfaces.enabled }}
         volumeMounts:
           {{- if .Values.speaker.memberlist.enabled }}
           - name: memberlist 
@@ -301,6 +307,10 @@ spec:
           - name: reloader
             mountPath: /etc/frr_reloader
           {{- end }}
+          {{- if .Values.speaker.excludeInterfaces.enabled }}
+          - name: metallb-conf
+            mountPath: /etc/metallb
+          {{- end }}
         {{- end }}
       {{- if .Values.speaker.frr.enabled }}
       - name: frr

diff --git a/charts/metallb/values.schema.json b/charts/metallb/values.schema.json
@@ -322,6 +322,14 @@
                 }
               }
             },
+            "excludeInterfaces": {
+              "type": "object",
+              "properties": {
+                "configPath": {
+                  "type": "boolean"
+                }
+              }
+            },
             "updateStrategy": {
               "type": "object",
               "properties": {

diff --git a/charts/metallb/values.yaml b/charts/metallb/values.yaml
@@ -260,6 +260,8 @@ speaker:
     enabled: true
     mlBindPort: 7946
     mlSecretKeyPath: "/etc/ml_secret_key"
+  excludeInterfaces:
+    enabled: true
   image:
     repository: quay.io/metallb/speaker
     tag:

diff --git a/config/controllers/speaker.yaml b/config/controllers/speaker.yaml
@@ -85,6 +85,9 @@ spec:
         - mountPath: /etc/ml_secret_key
           name: memberlist
           readOnly: true
+        - mountPath: /etc/metallb
+          name: metallb-conf
+          readOnly: true
       hostNetwork: true
       nodeSelector:
         kubernetes.io/os: linux
@@ -102,3 +105,7 @@ spec:
         secret: 
           secretName: memberlist
           defaultMode: 420
+      - name: metallb-conf
+        configMap:
+          defaultMode: 256
+          name: metallb-conf
diff --git a/config/manifests/metallb-frr-prometheus.yaml b/config/manifests/metallb-frr-prometheus.yaml
@@ -1813,6 +1813,15 @@ metadata:
   namespace: metallb-system
 ---
 apiVersion: v1
+data:
+  conf.yaml: |
+    announcedInterfacesToExclude: ["docker.*", "cbr.*", "dummy.*", "virbr.*", "lxcbr.*", "veth.*", "lo", "^cali.*", "^tunl.*", "flannel.*", "kube-ipvs.*", "cni.*", "^nodelocaldns.*"]
+kind: ConfigMap
+metadata:
+  name: metallb-conf
+  namespace: metallb-system
+---
+apiVersion: v1
 kind: Secret
 metadata:
   name: webhook-server-cert
@@ -2194,6 +2203,9 @@ spec:
         - mountPath: /etc/ml_secret_key
           name: memberlist
           readOnly: true
+        - mountPath: /etc/metallb
+          name: metallb-conf
+          readOnly: true
       hostNetwork: true
       initContainers:
       - command:
@@ -2256,6 +2268,10 @@ spec:
         secret:
           defaultMode: 420
           secretName: memberlist
+      - configMap:
+          defaultMode: 256
+          name: metallb-conf
+        name: metallb-conf
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor

diff --git a/config/manifests/metallb-frr.yaml b/config/manifests/metallb-frr.yaml
@@ -1758,6 +1758,15 @@ metadata:
   namespace: metallb-system
 ---
 apiVersion: v1
+data:
+  conf.yaml: |
+    announcedInterfacesToExclude: ["docker.*", "cbr.*", "dummy.*", "virbr.*", "lxcbr.*", "veth.*", "lo", "^cali.*", "^tunl.*", "flannel.*", "kube-ipvs.*", "cni.*", "^nodelocaldns.*"]
+kind: ConfigMap
+metadata:
+  name: metallb-conf
+  namespace: metallb-system
+---
+apiVersion: v1
 kind: Secret
 metadata:
   name: webhook-server-cert
@@ -2017,6 +2026,9 @@ spec:
         - mountPath: /etc/ml_secret_key
           name: memberlist
           readOnly: true
+        - mountPath: /etc/metallb
+          name: metallb-conf
+          readOnly: true
       hostNetwork: true
       initContainers:
       - command:
@@ -2079,6 +2091,10 @@ spec:
         secret:
           defaultMode: 420
           secretName: memberlist
+      - configMap:
+          defaultMode: 256
+          name: metallb-conf
+        name: metallb-conf
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration

diff --git a/config/manifests/metallb-native-prometheus.yaml b/config/manifests/metallb-native-prometheus.yaml
@@ -1715,6 +1715,15 @@ subjects:
   namespace: metallb-system
 ---
 apiVersion: v1
+data:
+  conf.yaml: |
+    announcedInterfacesToExclude: ["docker.*", "cbr.*", "dummy.*", "virbr.*", "lxcbr.*", "veth.*", "lo", "^cali.*", "^tunl.*", "flannel.*", "kube-ipvs.*", "cni.*", "^nodelocaldns.*"]
+kind: ConfigMap
+metadata:
+  name: metallb-conf
+  namespace: metallb-system
+---
+apiVersion: v1
 kind: Secret
 metadata:
   name: webhook-server-cert
@@ -1988,6 +1997,9 @@ spec:
         - mountPath: /etc/ml_secret_key
           name: memberlist
           readOnly: true
+        - mountPath: /etc/metallb
+          name: metallb-conf
+          readOnly: true
       hostNetwork: true
       nodeSelector:
         kubernetes.io/os: linux
@@ -2005,6 +2017,10 @@ spec:
         secret:
           defaultMode: 420
           secretName: memberlist
+      - configMap:
+          defaultMode: 256
+          name: metallb-conf
+        name: metallb-conf
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor

diff --git a/config/manifests/metallb-native.yaml b/config/manifests/metallb-native.yaml
@@ -1660,6 +1660,15 @@ subjects:
   namespace: metallb-system
 ---
 apiVersion: v1
+data:
+  conf.yaml: |
+    announcedInterfacesToExclude: ["docker.*", "cbr.*", "dummy.*", "virbr.*", "lxcbr.*", "veth.*", "lo", "^cali.*", "^tunl.*", "flannel.*", "kube-ipvs.*", "cni.*", "^nodelocaldns.*"]
+kind: ConfigMap
+metadata:
+  name: metallb-conf
+  namespace: metallb-system
+---
+apiVersion: v1
 kind: Secret
 metadata:
   name: webhook-server-cert
@@ -1842,6 +1851,9 @@ spec:
         - mountPath: /etc/ml_secret_key
           name: memberlist
           readOnly: true
+        - mountPath: /etc/metallb
+          name: metallb-conf
+          readOnly: true
       hostNetwork: true
       nodeSelector:
         kubernetes.io/os: linux
@@ -1859,6 +1871,10 @@ spec:
         secret:
           defaultMode: 420
           secretName: memberlist
+      - configMap:
+          defaultMode: 256
+          name: metallb-conf
+        name: metallb-conf
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration

diff --git a/config/native/configmap.yaml b/config/native/configmap.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: metallb-conf
+  namespace: metallb-system
+data:
+  conf.yaml: |
+    announcedInterfacesToExclude: ["docker.*", "cbr.*", "dummy.*", "virbr.*", "lxcbr.*", "veth.*", "lo", "^cali.*", "^tunl.*", "flannel.*", "kube-ipvs.*", "cni.*", "^nodelocaldns.*"]
diff --git a/config/native/kustomization.yaml b/config/native/kustomization.yaml
@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
 - ns.yaml
+- configmap.yaml
 - ../crd
 - ../rbac
 - ../controllers

diff --git a/internal/layer2/announcer.go b/internal/layer2/announcer.go
@@ -5,6 +5,7 @@ package layer2
 import (
 	"net"
 	"os"
+	"regexp"
 	"strconv"
 	"sync"
 	"time"
@@ -26,11 +27,12 @@ type Announce struct {
 
 	// This channel can block - do not write to it while holding the mutex
 	// to avoid deadlocking.
-	spamCh chan IPAdvertisement
+	spamCh        chan IPAdvertisement
+	excludeRegexp *regexp.Regexp
 }
 
 // New returns an initialized Announce.
-func New(l log.Logger) (*Announce, error) {
+func New(l log.Logger, excludeRegexp *regexp.Regexp) (*Announce, error) {
 	ret := &Announce{
 		logger:         l,
 		nodeInterfaces: []string{},
@@ -39,7 +41,9 @@ func New(l log.Logger) (*Announce, error) {
 		ips:            map[string][]IPAdvertisement{},
 		ipRefcnt:       map[string]int{},
 		spamCh:         make(chan IPAdvertisement, 1024),
+		excludeRegexp:  excludeRegexp,
 	}
+
 	go ret.interfaceScan()
 	go ret.spamLoop()
 
@@ -67,6 +71,12 @@ func (a *Announce) updateInterfaces() {
 	curIfs := make([]string, 0, len(ifs))
 	for _, intf := range ifs {
 		ifi := intf
+
+		if (a.excludeRegexp != nil) && a.excludeRegexp.MatchString(ifi.Name) {
+			level.Info(a.logger).Log("event", "announced interface to exclude", "interface", ifi.Name)
+			continue
+		}
+
 		curIfs = append(curIfs, ifi.Name)
 		l := log.With(a.logger, "interface", ifi.Name)
 		addrs, err := ifi.Addrs()