charts,salt: Allow to set Control Plane Ingress IP to an external IP

In some context the IP used to reach the Control Plane Ingress (MetalK8s UI) is not an IP available on the node, it could be a Load balancer IP or a NATed IP, so we need to be able to configure OIDC with that kind of IP That's why we now use the Control Plane Ingress IP defined in the bootstrap config and all master Control Plane node IP as `externalIPs` for the Control Plane Ingress service, so that we can reach the Control Plane Ingress with any of those IPs (but only the Control Plane Ingress Ip will be used as redirect IP for OIDC) Re-render the Control Plane Ingress DaemonSet chart salt state using: ``` ./charts/render.py ingress-nginx-control-plane --namespace metalk8s-ingress \ charts/ingress-nginx-control-plane-daemonset.yaml charts/ingress-nginx/ \ > salt/metalk8s/addons/nginx-ingress-control-plane/deployed/chart-daemonset.sls ```
scality · Apr 25, 2022 · 0fd32c7 · 0fd32c7
1 parent f816be2
commit 0fd32c7
Show file tree

Hide file tree

Showing 9 changed files with 175 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # CHANGELOG
 ## Release 2.11.7 (in development)
+### Enhancements
+
+- Allow to set Control Plane Ingress IP to an external IP (like
+  a load balancer IP)
+  (PR[#3752](https://github.com/scality/metalk8s/pull/3752))
 
 ## Release 2.11.6
 ### Enhancements

diff --git a/charts/ingress-nginx-control-plane-daemonset.yaml b/charts/ingress-nginx-control-plane-daemonset.yaml
@@ -38,8 +38,7 @@ controller:
   service:
     type: ClusterIP
 
-    externalIPs:
-    - '{%- endraw -%}{{ salt.metalk8s_network.get_control_plane_ingress_ip() }}{%- raw -%}'
+    externalIPs: '__var_tojson__(salt.metalk8s_network.get_control_plane_ingress_external_ips())'
 
     enableHttp: false
 

diff --git a/charts/render.py b/charts/render.py
@@ -168,7 +168,7 @@ def replace_magic_strings(rendered_yaml):
     # Handle __var_tojson__
     result = re.sub(
         r"__var_tojson__\((?P<varname>[\w\-_]+(?:\.[\w\-_()|]+)*)\)",
-        r"  {% endraw -%}{{ \g<varname> | tojson }}{%- raw %}",
+        r"{% endraw -%}{{ \g<varname> | tojson }}{%- raw %}",
         result,
     )
 

diff --git a/salt/_modules/metalk8s_network.py b/salt/_modules/metalk8s_network.py
@@ -257,3 +257,25 @@ def get_control_plane_ingress_endpoint():
     return "https://{}:8443".format(
         __salt__["metalk8s_network.get_control_plane_ingress_ip"]()
     )
+
+
+def get_control_plane_ingress_external_ips():
+    """Get all Control Plane Ingress external IPs
+
+    NOTE: Only the first one will be used as redirect IP for oidc
+    """
+    master_nodes = __salt__["metalk8s.minions_by_role"]("master")
+
+    # This function only run on master
+    mine_ret = __salt__["saltutil.runner"](
+        "mine.get", tgt=",".join(master_nodes), tgt_type="list", fun="control_plane_ip"
+    )
+
+    if not isinstance(mine_ret, dict):
+        raise CommandExecutionError(
+            f"Unable to get master Control Plane IPs: {mine_ret}"
+        )
+
+    return [__salt__["metalk8s_network.get_control_plane_ingress_ip"]()] + sorted(
+        list(mine_ret.values())
+    )
diff --git a/salt/metalk8s/addons/nginx-ingress-control-plane/deployed/chart-daemonset.sls b/salt/metalk8s/addons/nginx-ingress-control-plane/deployed/chart-daemonset.sls
@@ -280,9 +280,7 @@ metadata:
   name: ingress-nginx-control-plane-controller
   namespace: metalk8s-ingress
 spec:
-  externalIPs:
-  - '{%- endraw -%}{{ salt.metalk8s_network.get_control_plane_ingress_ip() }}{%- raw
-    -%}'
+  externalIPs: {% endraw -%}{{ salt.metalk8s_network.get_control_plane_ingress_external_ips() | tojson }}{%- raw %}
   ipFamilies:
   - IPv4
   ipFamilyPolicy: SingleStack

diff --git a/salt/metalk8s/orchestrate/deploy_node.sls b/salt/metalk8s/orchestrate/deploy_node.sls
@@ -326,3 +326,18 @@ Restart CoreDNS pods:
       - metalk8s_cordon: Uncordon the node
 
 {%- endif %}
+
+{%- if 'master' in roles and 'master' not in skip_roles %}
+
+# Trigger a reconfiguration of the Control Plane Ingress Controller as we
+# want to expose the Controller on every master node Control Plane IP
+Reconfigure Control Plane Ingress:
+  salt.runner:
+    - name: state.orchestrate
+    - mods:
+      - metalk8s.addons.nginx-ingress-control-plane.deployed
+    - saltenv: {{ saltenv }}
+    - require:
+      - metalk8s_cordon: Uncordon the node
+
+{%- endif %}
diff --git a/salt/tests/unit/formulas/fixtures/salt.py b/salt/tests/unit/formulas/fixtures/salt.py
@@ -424,6 +424,16 @@ def metalk8s_grafana_load_dashboard(source: str, **_kwargs: Any) -> Any:
 register_basic("metalk8s_network.get_control_plane_ingress_endpoint")(
     MagicMock(return_value="https://192.168.1.240:8443")
 )
+register_basic("metalk8s_network.get_control_plane_ingress_external_ips")(
+    MagicMock(
+        return_value=[
+            "192.168.1.240",
+            "192.168.1.100",
+            "192.168.1.101",
+            "192.168.1.102",
+        ]
+    )
+)
 
 
 @register_basic("metalk8s_network.get_ip_from_cidrs")

diff --git a/salt/tests/unit/modules/files/test_metalk8s_network.yaml b/salt/tests/unit/modules/files/test_metalk8s_network.yaml
@@ -135,3 +135,87 @@ get_control_plane_ingress_endpoint:
   - cp_ingress_ip_ret: "Error because of banana"
     raises: true
     result: "Error because of banana"
+
+get_control_plane_ingress_external_ips:
+  # 1. Nominal single node (using bootstrap IP)
+  - cp_ingress_ip_ret: 1.1.1.1
+    mine_ret:
+      bootstrap: 1.1.1.1
+    result:
+      - 1.1.1.1
+      - 1.1.1.1
+
+  # 2. Nominal single node (using non-bootstrap IP)
+  - cp_ingress_ip_ret: 1.1.1.4
+    mine_ret:
+      bootstrap: 1.1.1.1
+    result:
+      - 1.1.1.4
+      - 1.1.1.1
+
+  # 3. Nominal multi node (using bootstrap IP)
+  - cp_ingress_ip_ret: 1.1.1.1
+    master_nodes_ret:
+      - bootstrap
+      - node-1
+      - node-2
+    mine_ret:
+      bootstrap: 1.1.1.1
+      node-1: 1.1.1.2
+      node-2: 1.1.1.3
+    result:
+      - 1.1.1.1
+      - 1.1.1.1
+      - 1.1.1.2
+      - 1.1.1.3
+
+  # 4. Nominal multi node (using non-node IP)
+  - cp_ingress_ip_ret: 1.1.1.4
+    master_nodes_ret:
+      - bootstrap
+      - node-1
+      - node-2
+    mine_ret:
+      bootstrap: 1.1.1.1
+      node-1: 1.1.1.2
+      node-2: 1.1.1.3
+    result:
+      - 1.1.1.4
+      - 1.1.1.1
+      - 1.1.1.2
+      - 1.1.1.3
+
+  # 5. Nominal multi node (using non-bootstrap node IP)
+  - cp_ingress_ip_ret: 1.1.1.2
+    master_nodes_ret:
+      - bootstrap
+      - node-1
+      - node-2
+    mine_ret:
+      bootstrap: 1.1.1.1
+      node-1: 1.1.1.2
+      node-2: 1.1.1.3
+    result:
+      - 1.1.1.2
+      - 1.1.1.1
+      - 1.1.1.2
+      - 1.1.1.3
+
+  # 6. Multi node, one master node not yet in mine
+  - cp_ingress_ip_ret: 1.1.1.4
+    master_nodes_ret:
+      - bootstrap
+      - node-1
+      - node-2
+    mine_ret:
+      bootstrap: 1.1.1.1
+      node-2: 1.1.1.3
+    result:
+      - 1.1.1.4
+      - 1.1.1.1
+      - 1.1.1.3
+
+  # 7. Error unable to get from mine
+  - mine_ret: "ErROr"
+    raises: true
+    result: "Unable to get master Control Plane IPs: ErROr"
diff --git a/salt/tests/unit/modules/test_metalk8s_network.py b/salt/tests/unit/modules/test_metalk8s_network.py
@@ -363,3 +363,39 @@ def test_get_control_plane_ingress_endpoint(
                 self.assertEqual(
                     metalk8s_network.get_control_plane_ingress_endpoint(), result
                 )
+
+    @utils.parameterized_from_cases(
+        YAML_TESTS_CASES["get_control_plane_ingress_external_ips"]
+    )
+    def test_get_control_plane_ingress_external_ips(
+        self,
+        result,
+        raises=False,
+        cp_ingress_ip_ret=None,
+        master_nodes_ret=None,
+        mine_ret=None,
+    ):
+        """
+        Tests the return of `get_control_plane_ingress_external_ips` function
+        """
+        salt_dict = {
+            "metalk8s_network.get_control_plane_ingress_ip": MagicMock(
+                return_value=cp_ingress_ip_ret
+            ),
+            "metalk8s.minions_by_role": MagicMock(
+                return_value=master_nodes_ret or ["bootstrap"]
+            ),
+            "saltutil.runner": MagicMock(return_value=mine_ret),
+        }
+
+        with patch.dict(metalk8s_network.__salt__, salt_dict):
+            if raises:
+                self.assertRaisesRegex(
+                    CommandExecutionError,
+                    result,
+                    metalk8s_network.get_control_plane_ingress_external_ips,
+                )
+            else:
+                self.assertEqual(
+                    metalk8s_network.get_control_plane_ingress_external_ips(), result
+                )