diff --git a/component/egress-gateway-policies.jsonnet b/component/egress-gateway-policies.jsonnet index 9ad0a004..e1144a8d 100644 --- a/component/egress-gateway-policies.jsonnet +++ b/component/egress-gateway-policies.jsonnet @@ -3,10 +3,14 @@ local kap = import 'lib/kapitan.libjsonnet'; local kube = import 'lib/kube.libjsonnet'; local egw = import 'espejote-templates/egress-gateway.libsonnet'; +local ipcalc = import 'lib/cilium-ipcalc.libsonnet'; +local nm = import 'lib/openshift-nmstate.libsonnet'; local inv = kap.inventory(); local params = inv.parameters.cilium; +local has_nmstate = std.member(inv.applications, 'openshift-nmstate'); + local EgressGatewayPolicy(name) = if params.release == 'enterprise' then egw.IsovalentEgressGatewayPolicy(name) @@ -23,8 +27,22 @@ local egress_ip_policies = std.flattenArrays([ local egress_range = egw.read_egress_range(interface_prefix, cfg); local ns_egress_ips = std.get(cfg, 'namespace_egress_ips', {}); local dest_cidrs = com.renderArray(std.get(cfg, 'destination_cidrs', [])); - [ - egw.NamespaceEgressPolicy( + local shadow_ranges = std.get(cfg, 'shadow_ranges', {}); + local auto_egress_interfaces = + local requested = std.get(cfg, 'auto_egress_interfaces', false); + local has_shadow_ranges = std.length(shadow_ranges) > 0; + if requested && !has_nmstate then + error + 'User requested auto egress interfaces for "%s", ' % interface_prefix + + "but nmstate isn't present on the cluster." + else if requested && !has_shadow_ranges then + error + 'User requested auto egress interfaces for "%s", ' % interface_prefix + + 'but no shadow ranges are configured.' + else + requested && has_nmstate && has_shadow_ranges; + std.flattenArrays([ + local ep = egw.NamespaceEgressPolicy( interface_prefix, '%(start)s - %(end)s' % egress_range, std.objectValues(std.get(cfg, 'shadow_ranges', {})), @@ -35,10 +53,20 @@ local egress_ip_policies = std.flattenArrays([ destination_cidrs=dest_cidrs, bgp_policy_labels=std.get(cfg, 'bgp_policy_labels', {}), policy_labels=std.get(cfg, 'policy_labels', {}), - ) + ); + [ ep ] + + if auto_egress_interfaces then + egw.EgressInterfaceNNCPs( + nm.NodeNetworkConfigurationPolicy, + ep, + interface_prefix, + egress_range, + shadow_ranges + ) + else [] for namespace in std.objectFields(ns_egress_ips) if ns_egress_ips[namespace] != null - ] + ]) for interface_prefix in std.objectFields(params.egress_gateway.egress_ip_ranges) if params.egress_gateway.egress_ip_ranges[interface_prefix] != null ]); @@ -49,18 +77,23 @@ local egress_ip_policies = std.flattenArrays([ // of this object. local validate(policies) = std.objectValues(std.foldl( function(seen, p) - local ns = - p.spec.selectors[0].podSelector.matchLabels['io.kubernetes.pod.namespace']; - if std.objectHas(seen, ns) then - error 'duplicated source namespace "%s" for policies in egress ranges "%s" and "%s"' % [ - ns, - seen[ns].metadata.annotations['cilium.syn.tools/interface-prefix'], - p.metadata.annotations['cilium.syn.tools/interface-prefix'], - ] - else + if p.kind != EgressGatewayPolicy('dummy').kind then seen { - [ns]: p, - }, + ['%s-%s' % [ p.kind, p.metadata.name ]]: p, + } + else + local ns = + p.spec.selectors[0].podSelector.matchLabels['io.kubernetes.pod.namespace']; + if std.objectHas(seen, ns) then + error 'duplicated source namespace "%s" for policies in egress ranges "%s" and "%s"' % [ + ns, + seen[ns].metadata.annotations['cilium.syn.tools/interface-prefix'], + p.metadata.annotations['cilium.syn.tools/interface-prefix'], + ] + else + seen { + [ns]: p, + }, policies, {} )); diff --git a/component/espejote-templates/egress-gateway-self-service.jsonnet b/component/espejote-templates/egress-gateway-self-service.jsonnet index 5e3b1427..bead2a90 100644 --- a/component/espejote-templates/egress-gateway-self-service.jsonnet +++ b/component/espejote-templates/egress-gateway-self-service.jsonnet @@ -43,7 +43,12 @@ local reconcileNamespace(namespace) = // us) and update the namespace with an informational message. local range = res.range; local egress_range = egw.read_egress_range(range.if_prefix, range); - [ + if std.get(range, 'auto_egress_interfaces', false) then [ + setAnnotations(namespace, { + 'cilium.syn.tools/egress-ip-status': + "Allocating egress IP from range with `auto_egress_interfaces=true` isn't supported", + }), + ] else [ egw.NamespaceEgressPolicy( range.if_prefix, '%(start)s - %(end)s' % egress_range, diff --git a/component/espejote-templates/egress-gateway.libsonnet b/component/espejote-templates/egress-gateway.libsonnet index db8488ff..10e43c37 100644 --- a/component/espejote-templates/egress-gateway.libsonnet +++ b/component/espejote-templates/egress-gateway.libsonnet @@ -151,6 +151,70 @@ local NamespaceEgressPolicy = }, }; +local EgressInterfaceNNCPs(NNCP, ep, interface_prefix, egress_range, shadow_ranges) = [ + local shadow_ip = + local ifindex = + local debuginfo = std.foldl( + function(i, e) + local parts = std.splitLimit(e, '=', 1); + i { [parts[0]]: std.parseInt(parts[1]) }, + std.split( + ep.metadata.annotations['cilium.syn.tools/debug-interface-index'], + ', ' + ), + {} + ); + debuginfo.ip - debuginfo.start; + local sr = ipcalc.parse_ip_range('shadow range for "%s" in "%s"' % [ + node, + interface_prefix, + ], shadow_ranges[node]); + ipcalc.format_ipval(ipcalc.ipval(sr.start) + ifindex); + NNCP('egress-interface-%s-%s' % [ + ep.metadata.name, + node, + ]) { + metadata+: { + annotations+: { + 'argocd.argoproj.io/sync-wave': '-10', + 'cilium.syn.tools/description': + 'Generated policy to configure egress interface "%s" for shadow range "%s" associated with egress range "%s" (%s) on node "%s".' % [ + ep.spec.egressGroups[0].interface, + shadow_ranges[node], + interface_prefix, + '%(start)s - %(end)s' % egress_range, + node, + ], + }, + labels+: { + 'cilium.syn.tools/egress-policy': ep.metadata.name, + }, + }, + spec: { + nodeSelector: { + 'kubernetes.io/hostname': node, + }, + desiredState: { + interfaces: [ + { + name: '%s' % ep.spec.egressGroups[0].interface, + type: 'dummy', + ipv4: { + address: [ { + ip: shadow_ip, + 'prefix-length': 32, + } ], + dhcp: false, + enabled: true, + }, + }, + ], + }, + }, + } + for node in std.objectFields(shadow_ranges) +]; + local espejoteLabel = { 'cilium.syn.tools/managed-by': 'espejote_cilium_namespace-egress-ips', }; @@ -230,6 +294,7 @@ local find_egress_range(ranges, egress_ip) = CiliumEgressGatewayPolicy: CiliumEgressGatewayPolicy, IsovalentEgressGatewayPolicy: IsovalentEgressGatewayPolicy, NamespaceEgressPolicy: NamespaceEgressPolicy, + EgressInterfaceNNCPs: EgressInterfaceNNCPs, espejoteLabel: espejoteLabel, find_egress_range: find_egress_range, read_egress_range: read_egress_range, diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index e325b73d..74004384 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -381,7 +381,7 @@ default:: `{}` This parameter allows users to configure `CiliumEgressGatewayPolicy` (or `IsovalentEgressGatewayPolicy`) resources which assign a single egress IP to a namespace according to the design selected in https://kb.vshn.ch/oc4/explanations/decisions/cloudscale-cilium-egressip.html[Floating egress IPs with Cilium on cloudscale]. Each entry in the parameter is intended to describe a group of dummy interfaces that can be used in `CiliumEgressGatewayPolicy` (or `IsovalentEgressGatewayPolicy`) resources. -The component expects that each value is an object with fields `egress_cidr`, `egress_range`, `node_selector`, `namespace_egress_ips`, `shadow_ranges`, `destination_cidrs`, `bgp_policy_labels`, and `policy_labels`. +The component expects that each value is an object with fields `egress_cidr`, `egress_range`, `node_selector`, `namespace_egress_ips`, `shadow_ranges`, `destination_cidrs`, `bgp_policy_labels`, `policy_labels`, and `egress_auto_interfaces`. Fields `egress_cidr` and `egress_range` are mutually exclusive. The component raises an error for entries which set neither or both. @@ -436,6 +436,15 @@ The component adds all labels in `policy_labels` as labels on the generated egre If you specify labels in this parameter and in `bgp_policy_labels` and you specify the same label key in both fields, the label value provided in `bgp_policy_labels` will win. ==== +[NOTE] +==== +Field `egress_auto_interfaces` is optional. + +Setting the field to `true` raises an error for egress IP ranges which don't have any shadow ranges configured or if the `openshift-nmstate` component isn't present on the target cluster. + +Otherwise, the component will generate a `NodeNetworkConfigurationPolicy` to create an egress interface with the correct shadow IP on each node for which a shadow range is configured. +==== + ==== Prerequisites The component expects that the key for each entry matches the prefix of the dummy interface names that are assigned the shadow IPs which map to the egress IP range defined in `egress_range`. diff --git a/tests/egress-gateway.yml b/tests/egress-gateway.yml index 47163754..0a7c636c 100644 --- a/tests/egress-gateway.yml +++ b/tests/egress-gateway.yml @@ -1,4 +1,6 @@ # Overwrite parameters here +applications: + - openshift-nmstate parameters: kapitan: @@ -12,6 +14,9 @@ parameters: - type: https source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/v6.11.3/lib/openshift4-monitoring-alert-patching.libsonnet output_path: vendor/lib/alert-patching.libsonnet + - type: https + source: https://raw.githubusercontent.com/appuio/component-openshift-nmstate/refs/heads/master/lib/openshift-nmstate.libsonnet + output_path: vendor/lib/openshift-nmstate.libsonnet cilium: egress_gateway: enabled: true @@ -57,6 +62,7 @@ parameters: namespace_egress_ips: baz: 192.0.2.93 generate_shadow_ranges_configmap: false + auto_egress_interfaces: true shadow_ranges: infra-8344: 198.51.100.96 - 198.51.100.127 infra-87c9: 198.51.100.128 - 198.51.100.159 diff --git a/tests/golden/egress-gateway/cilium/cilium/20_namespace_egress_ip_policies.yaml b/tests/golden/egress-gateway/cilium/cilium/20_namespace_egress_ip_policies.yaml index e5fc575d..27dd2cb1 100644 --- a/tests/golden/egress-gateway/cilium/cilium/20_namespace_egress_ip_policies.yaml +++ b/tests/golden/egress-gateway/cilium/cilium/20_namespace_egress_ip_policies.yaml @@ -1,3 +1,84 @@ +apiVersion: nmstate.io/v1 +kind: NodeNetworkConfigurationPolicy +metadata: + annotations: + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + argocd.argoproj.io/sync-wave: '-10' + cilium.syn.tools/description: Generated policy to configure egress interface "egress_c_29" + for shadow range "198.51.100.96 - 198.51.100.127" associated with egress range + "egress_c" (192.0.2.64 - 192.0.2.95) on node "infra-8344". + labels: + cilium.syn.tools/egress-policy: baz + name: egress-interface-baz-infra-8344 + name: egress-interface-baz-infra-8344 +spec: + desiredState: + interfaces: + - ipv4: + address: + - ip: 198.51.100.125 + prefix-length: 32 + dhcp: false + enabled: true + name: egress_c_29 + type: dummy + nodeSelector: + kubernetes.io/hostname: infra-8344 +--- +apiVersion: nmstate.io/v1 +kind: NodeNetworkConfigurationPolicy +metadata: + annotations: + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + argocd.argoproj.io/sync-wave: '-10' + cilium.syn.tools/description: Generated policy to configure egress interface "egress_c_29" + for shadow range "198.51.100.128 - 198.51.100.159" associated with egress range + "egress_c" (192.0.2.64 - 192.0.2.95) on node "infra-87c9". + labels: + cilium.syn.tools/egress-policy: baz + name: egress-interface-baz-infra-87c9 + name: egress-interface-baz-infra-87c9 +spec: + desiredState: + interfaces: + - ipv4: + address: + - ip: 198.51.100.157 + prefix-length: 32 + dhcp: false + enabled: true + name: egress_c_29 + type: dummy + nodeSelector: + kubernetes.io/hostname: infra-87c9 +--- +apiVersion: nmstate.io/v1 +kind: NodeNetworkConfigurationPolicy +metadata: + annotations: + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + argocd.argoproj.io/sync-wave: '-10' + cilium.syn.tools/description: Generated policy to configure egress interface "egress_c_29" + for shadow range "198.51.100.160 - 198.51.100.191" associated with egress range + "egress_c" (192.0.2.64 - 192.0.2.95) on node "infra-eba2". + labels: + cilium.syn.tools/egress-policy: baz + name: egress-interface-baz-infra-eba2 + name: egress-interface-baz-infra-eba2 +spec: + desiredState: + interfaces: + - ipv4: + address: + - ip: 198.51.100.189 + prefix-length: 32 + dhcp: false + enabled: true + name: egress_c_29 + type: dummy + nodeSelector: + kubernetes.io/hostname: infra-eba2 +--- apiVersion: cilium.io/v2 kind: CiliumEgressGatewayPolicy metadata: diff --git a/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml b/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml index 79586021..b8a1abac 100644 --- a/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml +++ b/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml @@ -91,6 +91,7 @@ spec: } }, { + "auto_egress_interfaces": true, "bgp_policy_labels": { }, @@ -306,6 +307,70 @@ spec: }, }; + local EgressInterfaceNNCPs(NNCP, ep, interface_prefix, egress_range, shadow_ranges) = [ + local shadow_ip = + local ifindex = + local debuginfo = std.foldl( + function(i, e) + local parts = std.splitLimit(e, '=', 1); + i { [parts[0]]: std.parseInt(parts[1]) }, + std.split( + ep.metadata.annotations['cilium.syn.tools/debug-interface-index'], + ', ' + ), + {} + ); + debuginfo.ip - debuginfo.start; + local sr = ipcalc.parse_ip_range('shadow range for "%s" in "%s"' % [ + node, + interface_prefix, + ], shadow_ranges[node]); + ipcalc.format_ipval(ipcalc.ipval(sr.start) + ifindex); + NNCP('egress-interface-%s-%s' % [ + ep.metadata.name, + node, + ]) { + metadata+: { + annotations+: { + 'argocd.argoproj.io/sync-wave': '-10', + 'cilium.syn.tools/description': + 'Generated policy to configure egress interface "%s" for shadow range "%s" associated with egress range "%s" (%s) on node "%s".' % [ + ep.spec.egressGroups[0].interface, + shadow_ranges[node], + interface_prefix, + '%(start)s - %(end)s' % egress_range, + node, + ], + }, + labels+: { + 'cilium.syn.tools/egress-policy': ep.metadata.name, + }, + }, + spec: { + nodeSelector: { + 'kubernetes.io/hostname': node, + }, + desiredState: { + interfaces: [ + { + name: '%s' % ep.spec.egressGroups[0].interface, + type: 'dummy', + ipv4: { + address: [ { + ip: shadow_ip, + 'prefix-length': 32, + } ], + dhcp: false, + enabled: true, + }, + }, + ], + }, + }, + } + for node in std.objectFields(shadow_ranges) + ]; + local espejoteLabel = { 'cilium.syn.tools/managed-by': 'espejote_cilium_namespace-egress-ips', }; @@ -385,6 +450,7 @@ spec: CiliumEgressGatewayPolicy: CiliumEgressGatewayPolicy, IsovalentEgressGatewayPolicy: IsovalentEgressGatewayPolicy, NamespaceEgressPolicy: NamespaceEgressPolicy, + EgressInterfaceNNCPs: EgressInterfaceNNCPs, espejoteLabel: espejoteLabel, find_egress_range: find_egress_range, read_egress_range: read_egress_range, @@ -629,7 +695,12 @@ spec: // us) and update the namespace with an informational message. local range = res.range; local egress_range = egw.read_egress_range(range.if_prefix, range); - [ + if std.get(range, 'auto_egress_interfaces', false) then [ + setAnnotations(namespace, { + 'cilium.syn.tools/egress-ip-status': + "Allocating egress IP from range with `auto_egress_interfaces=true` isn't supported", + }), + ] else [ egw.NamespaceEgressPolicy( range.if_prefix, '%(start)s - %(end)s' % egress_range,