Add support for automatically generating egress interface NMstate policies

component/egress-gateway-policies.jsonnet

@@ -3,10 +3,14 @@ local kap = import 'lib/kapitan.libjsonnet';
 local kube = import 'lib/kube.libjsonnet';
 
 local egw = import 'espejote-templates/egress-gateway.libsonnet';
+local ipcalc = import 'lib/cilium-ipcalc.libsonnet';
+local nm = import 'lib/openshift-nmstate.libsonnet';
 
 local inv = kap.inventory();
 local params = inv.parameters.cilium;
 
+local has_nmstate = std.member(inv.applications, 'openshift-nmstate');
+
 local EgressGatewayPolicy(name) =
   if params.release == 'enterprise' then
     egw.IsovalentEgressGatewayPolicy(name)
@@ -23,8 +27,22 @@ local egress_ip_policies = std.flattenArrays([
   local egress_range = egw.read_egress_range(interface_prefix, cfg);
   local ns_egress_ips = std.get(cfg, 'namespace_egress_ips', {});
   local dest_cidrs = com.renderArray(std.get(cfg, 'destination_cidrs', []));
-  [
-    egw.NamespaceEgressPolicy(
+  local shadow_ranges = std.get(cfg, 'shadow_ranges', {});
+  local auto_egress_interfaces =
+    local requested = std.get(cfg, 'auto_egress_interfaces', false);
+    local has_shadow_ranges = std.length(shadow_ranges) > 0;
+    if requested && !has_nmstate then
+      error
+        'User requested auto egress interfaces for "%s", ' % interface_prefix +
+        "but nmstate isn't present on the cluster."
+    else if requested && !has_shadow_ranges then
+      error
+        'User requested auto egress interfaces for "%s", ' % interface_prefix +
+        'but no shadow ranges are configured.'
+    else
+      requested && has_nmstate && has_shadow_ranges;
+  std.flattenArrays([
+    local ep = egw.NamespaceEgressPolicy(
       interface_prefix,
       '%(start)s - %(end)s' % egress_range,
       std.objectValues(std.get(cfg, 'shadow_ranges', {})),
@@ -35,10 +53,20 @@ local egress_ip_policies = std.flattenArrays([
       destination_cidrs=dest_cidrs,
       bgp_policy_labels=std.get(cfg, 'bgp_policy_labels', {}),
       policy_labels=std.get(cfg, 'policy_labels', {}),
-    )
+    );
+    [ ep ] +
+    if auto_egress_interfaces then
+      egw.EgressInterfaceNNCPs(
+        nm.NodeNetworkConfigurationPolicy,
+        ep,
+        interface_prefix,
+        egress_range,
+        shadow_ranges
+      )
+    else []
     for namespace in std.objectFields(ns_egress_ips)
     if ns_egress_ips[namespace] != null
-  ]
+  ])
   for interface_prefix in std.objectFields(params.egress_gateway.egress_ip_ranges)
   if params.egress_gateway.egress_ip_ranges[interface_prefix] != null
 ]);
@@ -49,18 +77,23 @@ local egress_ip_policies = std.flattenArrays([
 // of this object.
 local validate(policies) = std.objectValues(std.foldl(
   function(seen, p)
-    local ns =
-      p.spec.selectors[0].podSelector.matchLabels['io.kubernetes.pod.namespace'];
-    if std.objectHas(seen, ns) then
-      error 'duplicated source namespace "%s" for policies in egress ranges "%s" and "%s"' % [
-        ns,
-        seen[ns].metadata.annotations['cilium.syn.tools/interface-prefix'],
-        p.metadata.annotations['cilium.syn.tools/interface-prefix'],
-      ]
-    else
+    if p.kind != EgressGatewayPolicy('dummy').kind then
       seen {
-        [ns]: p,
-      },
+        ['%s-%s' % [ p.kind, p.metadata.name ]]: p,
+      }
+    else
+      local ns =
+        p.spec.selectors[0].podSelector.matchLabels['io.kubernetes.pod.namespace'];
+      if std.objectHas(seen, ns) then
+        error 'duplicated source namespace "%s" for policies in egress ranges "%s" and "%s"' % [
+          ns,
+          seen[ns].metadata.annotations['cilium.syn.tools/interface-prefix'],
+          p.metadata.annotations['cilium.syn.tools/interface-prefix'],
+        ]
+      else
+        seen {
+          [ns]: p,
+        },
   policies,
   {}
 ));

component/espejote-templates/egress-gateway-self-service.jsonnet

@@ -43,7 +43,12 @@ local reconcileNamespace(namespace) =
       // us) and update the namespace with an informational message.
       local range = res.range;
       local egress_range = egw.read_egress_range(range.if_prefix, range);
-      [
+      if std.get(range, 'auto_egress_interfaces', false) then [
+        setAnnotations(namespace, {
+          'cilium.syn.tools/egress-ip-status':
+            "Allocating egress IP from range with `auto_egress_interfaces=true` isn't supported",
+        }),
+      ] else [
         egw.NamespaceEgressPolicy(
           range.if_prefix,
           '%(start)s - %(end)s' % egress_range,

component/espejote-templates/egress-gateway.libsonnet

@@ -151,6 +151,70 @@ local NamespaceEgressPolicy =
       },
     };
 
+local EgressInterfaceNNCPs(NNCP, ep, interface_prefix, egress_range, shadow_ranges) = [
+  local shadow_ip =
+    local ifindex =
+      local debuginfo = std.foldl(
+        function(i, e)
+          local parts = std.splitLimit(e, '=', 1);
+          i { [parts[0]]: std.parseInt(parts[1]) },
+        std.split(
+          ep.metadata.annotations['cilium.syn.tools/debug-interface-index'],
+          ', '
+        ),
+        {}
+      );
+      debuginfo.ip - debuginfo.start;
+    local sr = ipcalc.parse_ip_range('shadow range for "%s" in "%s"' % [
+      node,
+      interface_prefix,
+    ], shadow_ranges[node]);
+    ipcalc.format_ipval(ipcalc.ipval(sr.start) + ifindex);
+  NNCP('egress-interface-%s-%s' % [
+    ep.metadata.name,
+    node,
+  ]) {
+    metadata+: {
+      annotations+: {
+        'argocd.argoproj.io/sync-wave': '-10',
+        'cilium.syn.tools/description':
+          'Generated policy to configure egress interface "%s" for shadow range "%s" associated with egress range "%s" (%s) on node "%s".' % [
+            ep.spec.egressGroups[0].interface,
+            shadow_ranges[node],
+            interface_prefix,
+            '%(start)s - %(end)s' % egress_range,
+            node,
+          ],
+      },
+      labels+: {
+        'cilium.syn.tools/egress-policy': ep.metadata.name,
+      },
+    },
+    spec: {
+      nodeSelector: {
+        'kubernetes.io/hostname': node,
+      },
+      desiredState: {
+        interfaces: [
+          {
+            name: '%s' % ep.spec.egressGroups[0].interface,
+            type: 'dummy',
+            ipv4: {
+              address: [ {
+                ip: shadow_ip,
+                'prefix-length': 32,
+              } ],
+              dhcp: false,
+              enabled: true,
+            },
+          },
+        ],
+      },
+    },
+  }
+  for node in std.objectFields(shadow_ranges)
+];
+
 local espejoteLabel = {
   'cilium.syn.tools/managed-by': 'espejote_cilium_namespace-egress-ips',
 };
@@ -230,6 +294,7 @@ local find_egress_range(ranges, egress_ip) =
   CiliumEgressGatewayPolicy: CiliumEgressGatewayPolicy,
   IsovalentEgressGatewayPolicy: IsovalentEgressGatewayPolicy,
   NamespaceEgressPolicy: NamespaceEgressPolicy,
+  EgressInterfaceNNCPs: EgressInterfaceNNCPs,
   espejoteLabel: espejoteLabel,
   find_egress_range: find_egress_range,
   read_egress_range: read_egress_range,

docs/modules/ROOT/pages/references/parameters.adoc

@@ -381,7 +381,7 @@ default:: `{}`
 This parameter allows users to configure `CiliumEgressGatewayPolicy` (or `IsovalentEgressGatewayPolicy`) resources which assign a single egress IP to a namespace according to the design selected in https://kb.vshn.ch/oc4/explanations/decisions/cloudscale-cilium-egressip.html[Floating egress IPs with Cilium on cloudscale].
 
 Each entry in the parameter is intended to describe a group of dummy interfaces that can be used in `CiliumEgressGatewayPolicy` (or `IsovalentEgressGatewayPolicy`) resources.
-The component expects that each value is an object with fields `egress_cidr`, `egress_range`, `node_selector`, `namespace_egress_ips`, `shadow_ranges`, `destination_cidrs`, `bgp_policy_labels`, and `policy_labels`.
+The component expects that each value is an object with fields `egress_cidr`, `egress_range`, `node_selector`, `namespace_egress_ips`, `shadow_ranges`, `destination_cidrs`, `bgp_policy_labels`, `policy_labels`, and `egress_auto_interfaces`.
 Fields `egress_cidr` and `egress_range` are mutually exclusive.
 The component raises an error for entries which set neither or both.
 
@@ -436,6 +436,15 @@ The component adds all labels in `policy_labels` as labels on the generated egre
 If you specify labels in this parameter and in `bgp_policy_labels` and you specify the same label key in both fields, the label value provided in `bgp_policy_labels` will win.
 ====
 
+[NOTE]
+====
+Field `egress_auto_interfaces` is optional.
+
+Setting the field to `true` raises an error for egress IP ranges which don't have any shadow ranges configured or if the `openshift-nmstate` component isn't present on the target cluster.
+
+Otherwise, the component will generate a `NodeNetworkConfigurationPolicy` to create an egress interface with the correct shadow IP on each node for which a shadow range is configured.
+====
+
 ==== Prerequisites
 
 The component expects that the key for each entry matches the prefix of the dummy interface names that are assigned the shadow IPs which map to the egress IP range defined in `egress_range`.

tests/egress-gateway.yml

@@ -1,4 +1,6 @@
 # Overwrite parameters here
+applications:
+  - openshift-nmstate
 
 parameters:
   kapitan:
@@ -12,6 +14,9 @@ parameters:
       - type: https
         source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/v6.11.3/lib/openshift4-monitoring-alert-patching.libsonnet
         output_path: vendor/lib/alert-patching.libsonnet
+      - type: https
+        source: https://raw.githubusercontent.com/appuio/component-openshift-nmstate/refs/heads/master/lib/openshift-nmstate.libsonnet
+        output_path: vendor/lib/openshift-nmstate.libsonnet
   cilium:
     egress_gateway:
       enabled: true
@@ -57,6 +62,7 @@ parameters:
           namespace_egress_ips:
             baz: 192.0.2.93
           generate_shadow_ranges_configmap: false
+          auto_egress_interfaces: true
           shadow_ranges:
             infra-8344: 198.51.100.96 - 198.51.100.127
             infra-87c9: 198.51.100.128 - 198.51.100.159

tests/golden/egress-gateway/cilium/cilium/20_namespace_egress_ip_policies.yaml

@@ -1,3 +1,84 @@
+apiVersion: nmstate.io/v1
+kind: NodeNetworkConfigurationPolicy
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+    argocd.argoproj.io/sync-wave: '-10'
+    cilium.syn.tools/description: Generated policy to configure egress interface "egress_c_29"
+      for shadow range "198.51.100.96 - 198.51.100.127" associated with egress range
+      "egress_c" (192.0.2.64 - 192.0.2.95) on node "infra-8344".
+  labels:
+    cilium.syn.tools/egress-policy: baz
+    name: egress-interface-baz-infra-8344
+  name: egress-interface-baz-infra-8344
+spec:
+  desiredState:
+    interfaces:
+      - ipv4:
+          address:
+            - ip: 198.51.100.125
+              prefix-length: 32
+          dhcp: false
+          enabled: true
+        name: egress_c_29
+        type: dummy
+  nodeSelector:
+    kubernetes.io/hostname: infra-8344
+---
+apiVersion: nmstate.io/v1
+kind: NodeNetworkConfigurationPolicy
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+    argocd.argoproj.io/sync-wave: '-10'
+    cilium.syn.tools/description: Generated policy to configure egress interface "egress_c_29"
+      for shadow range "198.51.100.128 - 198.51.100.159" associated with egress range
+      "egress_c" (192.0.2.64 - 192.0.2.95) on node "infra-87c9".
+  labels:
+    cilium.syn.tools/egress-policy: baz
+    name: egress-interface-baz-infra-87c9
+  name: egress-interface-baz-infra-87c9
+spec:
+  desiredState:
+    interfaces:
+      - ipv4:
+          address:
+            - ip: 198.51.100.157
+              prefix-length: 32
+          dhcp: false
+          enabled: true
+        name: egress_c_29
+        type: dummy
+  nodeSelector:
+    kubernetes.io/hostname: infra-87c9
+---
+apiVersion: nmstate.io/v1
+kind: NodeNetworkConfigurationPolicy
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+    argocd.argoproj.io/sync-wave: '-10'
+    cilium.syn.tools/description: Generated policy to configure egress interface "egress_c_29"
+      for shadow range "198.51.100.160 - 198.51.100.191" associated with egress range
+      "egress_c" (192.0.2.64 - 192.0.2.95) on node "infra-eba2".
+  labels:
+    cilium.syn.tools/egress-policy: baz
+    name: egress-interface-baz-infra-eba2
+  name: egress-interface-baz-infra-eba2
+spec:
+  desiredState:
+    interfaces:
+      - ipv4:
+          address:
+            - ip: 198.51.100.189
+              prefix-length: 32
+          dhcp: false
+          enabled: true
+        name: egress_c_29
+        type: dummy
+  nodeSelector:
+    kubernetes.io/hostname: infra-eba2
+---
 apiVersion: cilium.io/v2
 kind: CiliumEgressGatewayPolicy
 metadata: