proper support/logic for CRD nodeSelectors

Signed-off-by: Travis Glenn Hansen <travisghansen@yahoo.com>

CHANGELOG.md

@@ -1,3 +1,10 @@
+# v0.3.2
+
+Released 2023-07-12
+
+- proer support for `nodeSelectors` in the CRDs (ie: limit upstream routes to only those applicable to the given node)
+- update chart to support new rbac neccessary for node logic
+
 # v0.3.1
 
 Released 2023-02-10

README.md

@@ -101,6 +101,9 @@ ip -d route show table metallb-nra
 
 # remove rules
 while ip rule delete from 0/0 to 0/0 table metallb-nra 2>/dev/null; do true; done
+
+
+ip route flush cache
 ```
 
 # TODO

agent.js

@@ -41,6 +41,8 @@ const METALLB_STATIC_FILE_WAIT = process.env.METALLB_STATIC_FILE_WAIT || 5000;
 
 const METALLB_USE_CRDS = process.env.METALLB_USE_CRDS;
 
+const NODE_NAME = process.env.NODE_NAME;
+
 // globals
 let metallb_loaded = false;
 
@@ -110,6 +112,9 @@ async function reconcile() {
 
       let args = [];
 
+      //logger.info('skipping reconcile, development debug');
+      //return;
+
       //////// step 1, create the routing table as appropriate /////////
 
       let tableExists = await ip.tableExists(TABLE_NAME);
@@ -546,9 +551,40 @@ async function processMetalLBCRDData() {
   );
 
   for (const peer of peers) {
-    metallb_peers.push(peer.spec.peerAddress);
+    let peerAllowed = true;
+
+    // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#resources-that-support-set-based-requirements
+    let nodeSelectors = _.get(peer, 'spec.nodeSelectors');
+    if (Array.isArray(nodeSelectors) && nodeSelectors.length > 0) {
+      let node = await kc.makeHttpRestRequest(
+        `/api/v1/nodes/${NODE_NAME}`,
+        'GET'
+      );
+      node = node.body;
+
+      // defaulting to false, if *any* of the selectors is true then peer is allowed
+      peerAllowed = false;
+      let i = 0;
+      do {
+        logger.debug(`asserting labelSelector %j`, nodeSelectors[i]);
+        peerAllowed = await kc.assertLabelSelector(node, nodeSelectors[i]);
+        i++;
+      } while (!peerAllowed && i < nodeSelectors.length);
+    }
+
+    if (peerAllowed) {
+      metallb_peers.push(peer.spec.peerAddress);
+    } else {
+      logger.info(
+        `ignoring peer %s due to nodeSelectors`,
+        peer.spec.peerAddress
+      );
+    }
   }
 
+  /**
+   * NOTE: purposely ignoring spec.nodeSelectors here for DSR-like scenarios
+   */
   for (const advertisement of advertisements) {
     for (const poolName of advertisement.spec.ipAddressPools) {
       for (const pool of pools) {
@@ -629,6 +665,68 @@ async function setupMetalLBCRDsWatch() {
     );
   }
 
+  if (NODE_NAME) {
+    // watching a specific node fails for some reason with the js client
+    //const resourcePath = `/api/v1/nodes/${NODE_NAME}`;
+    const resourcePath = `/api/v1/nodes`;
+    const resourceVersion = await kc.getCurrentResourceVersion(resourcePath);
+    const watch = new k8s.Watch(kc);
+    logger.info(
+      `starting ${resourcePath} watch at resourceVersion=${resourceVersion}`
+    );
+    watch.watch(
+      `${resourcePath}?resourceVersion=${resourceVersion}`,
+      {},
+      async (type, apiObj, watchObj) => {
+        if (_.get(watchObj, 'object.metadata.name') != NODE_NAME) {
+          logger.debug('ignoring node update because non-matching node');
+          return;
+        }
+
+        switch (type) {
+          case 'ADDED':
+          case 'MODIFIED': {
+            logger.info(`${resourcePath} added/modified`);
+            await processMetalLBCRDData();
+            await reconcile();
+            break;
+          }
+          case 'DELETED':
+            logger.warn(`${resourcePath} deleted from watch`);
+            await processMetalLBCRDData();
+            await reconcile();
+            break;
+          case 'BOOKMARK':
+            logger.verbose(
+              `${resourcePath} bookmarked: ${watchObj.metadata.resourceVersion}`
+            );
+            break;
+          default:
+            logger.error(
+              `unknown operation on ${resourcePath} watch: %s`,
+              type
+            );
+            break;
+        }
+      },
+      async e => {
+        metallb_loaded = false;
+        if (e) {
+          logger.error('watch failure: %s', e);
+          switch (e.code) {
+            case 'ECONNREFUSED':
+              process.exit(1);
+          }
+        } else {
+          logger.info('watch timeout');
+        }
+
+        await sleep(5000);
+        setupMetalLBCRDsWatch();
+      }
+    );
+  }
+
   await processMetalLBCRDData();
   await reconcile();
 }

charts/metallb-node-route-agent/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.0
+version: 0.3.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/metallb-node-route-agent/templates/daemonset.yaml

@@ -40,6 +40,11 @@ spec:
           - name: iproute2
             mountPath: /etc/iproute2
           env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: spec.nodeName
 {{- if .Values.extraEnv }}
 {{ toYaml .Values.extraEnv | indent 10 }}
 {{- end }}

charts/metallb-node-route-agent/templates/rbac.yaml

@@ -48,4 +48,32 @@ roleRef:
   kind: Role
   name: {{ include "metallb-node-route-agent.serviceAccountName" . }}
   apiGroup: rbac.authorization.k8s.io
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "metallb-node-route-agent.serviceAccountName" . }}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "metallb-node-route-agent.serviceAccountName" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "metallb-node-route-agent.serviceAccountName" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "metallb-node-route-agent.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace | quote }}
 {{- end }}

lib/k8s.js

@@ -54,6 +54,76 @@ class KubeConfig extends k8s.KubeConfig {
     }
   }
 
+  // https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1LabelSelector.md
+  async assertLabelSelector(apiObj, labelSelector) {
+    // An empty label selector matches all objects. A null label selector matches no objects.
+    if (labelSelector === null) {
+      return false;
+    }
+
+    let objLabels;
+    objLabels = apiObj.metadata.labels || {};
+
+    //objLabels = {};
+    if (labelSelector.matchLabels) {
+      for (const property in labelSelector.matchLabels) {
+        //console.log(
+        //  `comparing ${objLabels[property]} == ${labelSelector.matchLabels[property]}`
+        //);
+        if (objLabels[property] != labelSelector.matchLabels[property]) {
+          return false;
+        }
+      }
+    }
+
+    //objLabels = {};
+    if (labelSelector.matchExpressions) {
+      let i = 0;
+      do {
+        let matchExpression = labelSelector.matchExpressions[i];
+        if (matchExpression.key) {
+          if (matchExpression.operator) {
+            switch (matchExpression.operator) {
+              case 'In':
+                if (
+                  !matchExpression.values.includes(
+                    objLabels[matchExpression.key]
+                  )
+                ) {
+                  return false;
+                }
+                break;
+              case 'NotIn':
+                if (
+                  matchExpression.values.includes(
+                    objLabels[matchExpression.key]
+                  )
+                ) {
+                  return false;
+                }
+                break;
+              case 'Exists':
+                if (!Object.keys(objLabels).includes(matchExpression.key)) {
+                  return false;
+                }
+                break;
+              case 'DoesNotExist':
+                if (Object.keys(objLabels).includes(matchExpression.key)) {
+                  return false;
+                }
+                break;
+              default:
+                throw new Error(`unkown operator ${matchExpression.operator}`);
+            }
+          }
+        }
+        i++;
+      } while (i < labelSelector.matchExpressions.length);
+    }
+
+    return true;
+  }
+
   /**
    *
    * Make an HTTP request to the k8s api