From 188180a7122d37b8ecb8fe2cf600ff2cdf2f45af Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Wed, 12 Jul 2023 08:14:04 -0600 Subject: [PATCH] proper support/logic for CRD nodeSelectors Signed-off-by: Travis Glenn Hansen --- CHANGELOG.md | 7 ++ README.md | 3 + agent.js | 100 +++++++++++++++++- charts/metallb-node-route-agent/Chart.yaml | 2 +- .../templates/daemonset.yaml | 5 + .../templates/rbac.yaml | 28 +++++ lib/k8s.js | 70 ++++++++++++ 7 files changed, 213 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4ee8e9..34cd11a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# v0.3.2 + +Released 2023-07-12 + +- proer support for `nodeSelectors` in the CRDs (ie: limit upstream routes to only those applicable to the given node) +- update chart to support new rbac neccessary for node logic + # v0.3.1 Released 2023-02-10 diff --git a/README.md b/README.md index 9fa30e8..b573cc6 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,9 @@ ip -d route show table metallb-nra # remove rules while ip rule delete from 0/0 to 0/0 table metallb-nra 2>/dev/null; do true; done + + +ip route flush cache ``` # TODO diff --git a/agent.js b/agent.js index 5dd6dac..1941274 100644 --- a/agent.js +++ b/agent.js @@ -41,6 +41,8 @@ const METALLB_STATIC_FILE_WAIT = process.env.METALLB_STATIC_FILE_WAIT || 5000; const METALLB_USE_CRDS = process.env.METALLB_USE_CRDS; +const NODE_NAME = process.env.NODE_NAME; + // globals let metallb_loaded = false; @@ -110,6 +112,9 @@ async function reconcile() { let args = []; + //logger.info('skipping reconcile, development debug'); + //return; + //////// step 1, create the routing table as appropriate ///////// let tableExists = await ip.tableExists(TABLE_NAME); @@ -546,9 +551,40 @@ async function processMetalLBCRDData() { ); for (const peer of peers) { - metallb_peers.push(peer.spec.peerAddress); + let peerAllowed = true; + + // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#resources-that-support-set-based-requirements + let nodeSelectors = _.get(peer, 'spec.nodeSelectors'); + if (Array.isArray(nodeSelectors) && nodeSelectors.length > 0) { + let node = await kc.makeHttpRestRequest( + `/api/v1/nodes/${NODE_NAME}`, + 'GET' + ); + node = node.body; + + // defaulting to false, if *any* of the selectors is true then peer is allowed + peerAllowed = false; + let i = 0; + do { + logger.debug(`asserting labelSelector %j`, nodeSelectors[i]); + peerAllowed = await kc.assertLabelSelector(node, nodeSelectors[i]); + i++; + } while (!peerAllowed && i < nodeSelectors.length); + } + + if (peerAllowed) { + metallb_peers.push(peer.spec.peerAddress); + } else { + logger.info( + `ignoring peer %s due to nodeSelectors`, + peer.spec.peerAddress + ); + } } + /** + * NOTE: purposely ignoring spec.nodeSelectors here for DSR-like scenarios + */ for (const advertisement of advertisements) { for (const poolName of advertisement.spec.ipAddressPools) { for (const pool of pools) { @@ -629,6 +665,68 @@ async function setupMetalLBCRDsWatch() { ); } + if (NODE_NAME) { + // watching a specific node fails for some reason with the js client + //const resourcePath = `/api/v1/nodes/${NODE_NAME}`; + const resourcePath = `/api/v1/nodes`; + const resourceVersion = await kc.getCurrentResourceVersion(resourcePath); + const watch = new k8s.Watch(kc); + logger.info( + `starting ${resourcePath} watch at resourceVersion=${resourceVersion}` + ); + watch.watch( + `${resourcePath}?resourceVersion=${resourceVersion}`, + {}, + async (type, apiObj, watchObj) => { + if (_.get(watchObj, 'object.metadata.name') != NODE_NAME) { + logger.debug('ignoring node update because non-matching node'); + return; + } + + switch (type) { + case 'ADDED': + case 'MODIFIED': { + logger.info(`${resourcePath} added/modified`); + await processMetalLBCRDData(); + await reconcile(); + break; + } + case 'DELETED': + logger.warn(`${resourcePath} deleted from watch`); + await processMetalLBCRDData(); + await reconcile(); + break; + case 'BOOKMARK': + logger.verbose( + `${resourcePath} bookmarked: ${watchObj.metadata.resourceVersion}` + ); + break; + default: + logger.error( + `unknown operation on ${resourcePath} watch: %s`, + type + ); + break; + } + }, + async e => { + metallb_loaded = false; + if (e) { + logger.error('watch failure: %s', e); + switch (e.code) { + case 'ECONNREFUSED': + process.exit(1); + } + } else { + logger.info('watch timeout'); + } + + await sleep(5000); + setupMetalLBCRDsWatch(); + } + ); + } + await processMetalLBCRDData(); await reconcile(); } diff --git a/charts/metallb-node-route-agent/Chart.yaml b/charts/metallb-node-route-agent/Chart.yaml index b0bb68d..638d484 100644 --- a/charts/metallb-node-route-agent/Chart.yaml +++ b/charts/metallb-node-route-agent/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 +version: 0.3.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/metallb-node-route-agent/templates/daemonset.yaml b/charts/metallb-node-route-agent/templates/daemonset.yaml index 5e5a50f..68a998d 100644 --- a/charts/metallb-node-route-agent/templates/daemonset.yaml +++ b/charts/metallb-node-route-agent/templates/daemonset.yaml @@ -40,6 +40,11 @@ spec: - name: iproute2 mountPath: /etc/iproute2 env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName {{- if .Values.extraEnv }} {{ toYaml .Values.extraEnv | indent 10 }} {{- end }} diff --git a/charts/metallb-node-route-agent/templates/rbac.yaml b/charts/metallb-node-route-agent/templates/rbac.yaml index 9d10408..e723cbd 100644 --- a/charts/metallb-node-route-agent/templates/rbac.yaml +++ b/charts/metallb-node-route-agent/templates/rbac.yaml @@ -48,4 +48,32 @@ roleRef: kind: Role name: {{ include "metallb-node-route-agent.serviceAccountName" . }} apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "metallb-node-route-agent.serviceAccountName" . }} +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "metallb-node-route-agent.serviceAccountName" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "metallb-node-route-agent.serviceAccountName" . }} +subjects: +- kind: ServiceAccount + name: {{ include "metallb-node-route-agent.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} {{- end }} diff --git a/lib/k8s.js b/lib/k8s.js index 567d225..8621cd9 100644 --- a/lib/k8s.js +++ b/lib/k8s.js @@ -54,6 +54,76 @@ class KubeConfig extends k8s.KubeConfig { } } + // https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1LabelSelector.md + async assertLabelSelector(apiObj, labelSelector) { + // An empty label selector matches all objects. A null label selector matches no objects. + if (labelSelector === null) { + return false; + } + + let objLabels; + objLabels = apiObj.metadata.labels || {}; + + //objLabels = {}; + if (labelSelector.matchLabels) { + for (const property in labelSelector.matchLabels) { + //console.log( + // `comparing ${objLabels[property]} == ${labelSelector.matchLabels[property]}` + //); + if (objLabels[property] != labelSelector.matchLabels[property]) { + return false; + } + } + } + + //objLabels = {}; + if (labelSelector.matchExpressions) { + let i = 0; + do { + let matchExpression = labelSelector.matchExpressions[i]; + if (matchExpression.key) { + if (matchExpression.operator) { + switch (matchExpression.operator) { + case 'In': + if ( + !matchExpression.values.includes( + objLabels[matchExpression.key] + ) + ) { + return false; + } + break; + case 'NotIn': + if ( + matchExpression.values.includes( + objLabels[matchExpression.key] + ) + ) { + return false; + } + break; + case 'Exists': + if (!Object.keys(objLabels).includes(matchExpression.key)) { + return false; + } + break; + case 'DoesNotExist': + if (Object.keys(objLabels).includes(matchExpression.key)) { + return false; + } + break; + default: + throw new Error(`unkown operator ${matchExpression.operator}`); + } + } + } + i++; + } while (i < labelSelector.matchExpressions.length); + } + + return true; + } + /** * * Make an HTTP request to the k8s api