From 74757745a75004273600d29d3fc284d9d11f47c4 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Wed, 5 Nov 2025 13:17:14 +0100 Subject: [PATCH 1/5] Extend `NamespaceEgressPolicy` to support BGP-backed egress IPs For BGP-backed egress IPs, we don't configure the policy's egress IP based on the computed interface, but instead directly specify the egress IP in the policy. Additionally, we always set `maxGatewayNodes: 1` for BGP-backed egress IPs. Note that it's still the responsibility of the cluster operator to ensure that all possible BGP-backed egress IPs are assigned to an interface on each egress node. --- .../espejote-templates/egress-gateway.libsonnet | 13 +++++++++++-- .../cilium/40_egress_ip_managed_resource.yaml | 13 +++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/component/espejote-templates/egress-gateway.libsonnet b/component/espejote-templates/egress-gateway.libsonnet index 67593ba79..8a9c42a55 100644 --- a/component/espejote-templates/egress-gateway.libsonnet +++ b/component/espejote-templates/egress-gateway.libsonnet @@ -43,6 +43,7 @@ local NamespaceEgressPolicy = namespace, policy_resource_fn, destination_cidrs=null, + bgp_policy_labels={}, ) // Helper which computes the interface index of the egress IP. // Assumes that the IPs in egress_range are assigned to dummy interfaces @@ -99,11 +100,14 @@ local NamespaceEgressPolicy = : 'Expected `destination_cidrs` to be an array, got %s' % std.type(destination_cidrs); destination_cidrs; + local bgp_egress_ip = std.length(bgp_policy_labels) > 0; + policy_resource_fn(namespace) { metadata+: { annotations+: { 'cilium.syn.tools/description': - 'Generated policy to assign egress IP %s in egress range "%s" (%s) to namespace %s.' % [ + 'Generated policy to assign %segress IP %s in egress range "%s" (%s) to namespace %s.' % [ + if bgp_egress_ip then 'BGP ' else '', egress_ip, interface_prefix, egress_range, @@ -113,18 +117,23 @@ local NamespaceEgressPolicy = 'cilium.syn.tools/interface-prefix': interface_prefix, 'cilium.syn.tools/egress-range': egress_range, 'cilium.syn.tools/source-namespace': namespace, - 'cilium.syn.tools/debug-interface-index': ifindex.debug, + [if !bgp_egress_ip then 'cilium.syn.tools/debug-interface-index']: ifindex.debug, [if std.length(shadow_ips) > 0 then 'cilium.syn.tools/shadow-ips']: std.manifestJsonMinified(shadow_ips), }, + labels+: bgp_policy_labels, }, spec: { destinationCIDRs: dest_cidrs, + [if bgp_egress_ip then 'egressCIDRs']: [ '%s/32' % egress_ip ], egressGroups: [ { nodeSelector: { matchLabels: node_selector, }, + } + if bgp_egress_ip then { + maxGatewayNodes: 1, + } else { interface: ifindex.ifname, }, ], diff --git a/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml b/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml index 6da0be906..a02b76c73 100644 --- a/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml +++ b/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml @@ -187,6 +187,7 @@ spec: namespace, policy_resource_fn, destination_cidrs=null, + bgp_policy_labels={}, ) // Helper which computes the interface index of the egress IP. // Assumes that the IPs in egress_range are assigned to dummy interfaces @@ -243,11 +244,14 @@ spec: : 'Expected `destination_cidrs` to be an array, got %s' % std.type(destination_cidrs); destination_cidrs; + local bgp_egress_ip = std.length(bgp_policy_labels) > 0; + policy_resource_fn(namespace) { metadata+: { annotations+: { 'cilium.syn.tools/description': - 'Generated policy to assign egress IP %s in egress range "%s" (%s) to namespace %s.' % [ + 'Generated policy to assign %segress IP %s in egress range "%s" (%s) to namespace %s.' % [ + if bgp_egress_ip then 'BGP ' else '', egress_ip, interface_prefix, egress_range, @@ -257,18 +261,23 @@ spec: 'cilium.syn.tools/interface-prefix': interface_prefix, 'cilium.syn.tools/egress-range': egress_range, 'cilium.syn.tools/source-namespace': namespace, - 'cilium.syn.tools/debug-interface-index': ifindex.debug, + [if !bgp_egress_ip then 'cilium.syn.tools/debug-interface-index']: ifindex.debug, [if std.length(shadow_ips) > 0 then 'cilium.syn.tools/shadow-ips']: std.manifestJsonMinified(shadow_ips), }, + labels+: bgp_policy_labels, }, spec: { destinationCIDRs: dest_cidrs, + [if bgp_egress_ip then 'egressCIDRs']: [ '%s/32' % egress_ip ], egressGroups: [ { nodeSelector: { matchLabels: node_selector, }, + } + if bgp_egress_ip then { + maxGatewayNodes: 1, + } else { interface: ifindex.ifname, }, ], From f1105476501f03cacc0bf824d807d402b716ba0b Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Wed, 5 Nov 2025 13:48:59 +0100 Subject: [PATCH 2/5] Add support for configuring BGP-backed namespace egress IPs in the component --- component/egress-gateway-policies.jsonnet | 1 + tests/egress-gateway.yml | 2 ++ .../cilium/cilium/20_namespace_egress_ip_policies.yaml | 8 +++++--- .../cilium/cilium/40_egress_ip_managed_resource.yaml | 3 +++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/component/egress-gateway-policies.jsonnet b/component/egress-gateway-policies.jsonnet index 0ed6872ee..c0c88bb30 100644 --- a/component/egress-gateway-policies.jsonnet +++ b/component/egress-gateway-policies.jsonnet @@ -32,6 +32,7 @@ local egress_ip_policies = std.flattenArrays([ namespace, EgressGatewayPolicy, destination_cidrs=dest_cidrs, + bgp_policy_labels=std.get(cfg, 'bgp_policy_labels', {}), ) for namespace in std.objectFields(ns_egress_ips) if ns_egress_ips[namespace] != null diff --git a/tests/egress-gateway.yml b/tests/egress-gateway.yml index a6c5c196d..1f89ccd8a 100644 --- a/tests/egress-gateway.yml +++ b/tests/egress-gateway.yml @@ -80,3 +80,5 @@ parameters: destination_cidrs: - 203.0.113.0/25 - ~203.0.113.128/25 + bgp_policy_labels: + cilium.syn.tools/egress-ip: bgp diff --git a/tests/golden/egress-gateway/cilium/cilium/20_namespace_egress_ip_policies.yaml b/tests/golden/egress-gateway/cilium/cilium/20_namespace_egress_ip_policies.yaml index 4462b01cd..ed2d23355 100644 --- a/tests/golden/egress-gateway/cilium/cilium/20_namespace_egress_ip_policies.yaml +++ b/tests/golden/egress-gateway/cilium/cilium/20_namespace_egress_ip_policies.yaml @@ -90,21 +90,23 @@ kind: CiliumEgressGatewayPolicy metadata: annotations: argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true,Prune=false - cilium.syn.tools/debug-interface-index: start=3221226144, end=3221226175, ip=3221226144 - cilium.syn.tools/description: Generated policy to assign egress IP 192.0.2.160 + cilium.syn.tools/description: Generated policy to assign BGP egress IP 192.0.2.160 in egress range "egress_f" (192.0.2.160 - 192.0.2.191) to namespace qux. cilium.syn.tools/egress-ip: 192.0.2.160 cilium.syn.tools/egress-range: 192.0.2.160 - 192.0.2.191 cilium.syn.tools/interface-prefix: egress_f cilium.syn.tools/source-namespace: qux labels: + cilium.syn.tools/egress-ip: bgp name: qux name: qux spec: destinationCIDRs: - 203.0.113.0/25 + egressCIDRs: + - 192.0.2.160/32 egressGroups: - - interface: egress_f_0 + - maxGatewayNodes: 1 nodeSelector: matchLabels: node-role.kubernetes.io/infra: '' diff --git a/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml b/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml index a02b76c73..9c5a21a8e 100644 --- a/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml +++ b/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml @@ -127,6 +127,9 @@ spec: "shadow_ranges": null }, { + "bgp_policy_labels": { + "cilium.syn.tools/egress-ip": "bgp" + }, "destination_cidrs": [ "203.0.113.0/25" ], From f1d017c17f6cd8a34b1b2c1d7214d2327ac056f4 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Wed, 5 Nov 2025 14:00:47 +0100 Subject: [PATCH 3/5] Update documentation --- docs/modules/ROOT/pages/references/parameters.adoc | 12 +++++++++++- tests/egress-gateway.yml | 1 + .../cilium/cilium/40_egress_ip_managed_resource.yaml | 3 +++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index dd0df7bd1..567c2f0ac 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -381,7 +381,7 @@ default:: `{}` This parameter allows users to configure `CiliumEgressGatewayPolicy` (or `IsovalentEgressGatewayPolicy`) resources which assign a single egress IP to a namespace according to the design selected in https://kb.vshn.ch/oc4/explanations/decisions/cloudscale-cilium-egressip.html[Floating egress IPs with Cilium on cloudscale]. Each entry in the parameter is intended to describe a group of dummy interfaces that can be used in `CiliumEgressGatewayPolicy` (or `IsovalentEgressGatewayPolicy`) resources. -The component expects that each value is an object with fields `egress_range`, `node_selector`, `namespace_egress_ips`, `shadow_ranges`, and `destination_cidrs`. +The component expects that each value is an object with fields `egress_range`, `node_selector`, `namespace_egress_ips`, `shadow_ranges`, `destination_cidrs`, and `bgp_policy_labels`. NOTE: Field `shadow_ranges` is optional, see the section on <<_shadow_ranges,shadow ranges>> for more details. @@ -398,6 +398,16 @@ When this field is omitted or empty, the component generates egress policies wit Such policies assign the egress IP to all traffic originating in the namespace. ==== +[NOTE] +==== +Field `bgp_policy_labels` is optional. + +When the field is provide, and not an empty object, the component adds the contents as entries in `metadata.labels` of the resulting policies. +In this case, the component also configures the egress policies directly with the egress IP and `mmaxGatewayNodes: 1` in the `spec.egressGroups` entry. + +If the field is omitted or empty, the component configures the egress group with the interface name and doesn't set `maxGatewayNodes`. +==== + ==== Prerequisites The component expects that the key for each entry matches the prefix of the dummy interface names that are assigned the shadow IPs which map to the egress IP range defined in `egress_range`. diff --git a/tests/egress-gateway.yml b/tests/egress-gateway.yml index 1f89ccd8a..06a37d668 100644 --- a/tests/egress-gateway.yml +++ b/tests/egress-gateway.yml @@ -60,6 +60,7 @@ parameters: infra-eba2: 198.51.100.160 - 198.51.100.191 destination_cidrs: - 203.0.113.0/24 + bgp_policy_labels: {} egress_d: egress_range: '192.0.2.96 - 192.0.2.127' node_selector: diff --git a/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml b/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml index 9c5a21a8e..0d7b3aeab 100644 --- a/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml +++ b/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml @@ -88,6 +88,9 @@ spec: } }, { + "bgp_policy_labels": { + + }, "destination_cidrs": [ "203.0.113.0/24" ], From 940221c0973365a548824972472f654ba94f5caa Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Wed, 5 Nov 2025 14:06:39 +0100 Subject: [PATCH 4/5] Update EIP self-service to support BGP labels We extend the Espejote managed resource Jsonnet to also read the egress IP's range's `bgp_policy_labels` field when creating the egress policy. --- component/espejote-templates/egress-gateway-self-service.jsonnet | 1 + .../cilium/cilium/40_egress_ip_managed_resource.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/component/espejote-templates/egress-gateway-self-service.jsonnet b/component/espejote-templates/egress-gateway-self-service.jsonnet index f03c3ae02..65bb0b40e 100644 --- a/component/espejote-templates/egress-gateway-self-service.jsonnet +++ b/component/espejote-templates/egress-gateway-self-service.jsonnet @@ -52,6 +52,7 @@ local reconcileNamespace(namespace) = ns_meta.name, egw.IsovalentEgressGatewayPolicy, destination_cidrs=std.get(range, 'destination_cidrs', []), + bgp_policy_labels=std.get(range, 'bgp_policy_labels', {}), ) { metadata+: { labels+: egw.espejoteLabel, diff --git a/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml b/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml index 0d7b3aeab..8719b5aec 100644 --- a/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml +++ b/tests/golden/egress-gateway/cilium/cilium/40_egress_ip_managed_resource.yaml @@ -513,6 +513,7 @@ spec: ns_meta.name, egw.IsovalentEgressGatewayPolicy, destination_cidrs=std.get(range, 'destination_cidrs', []), + bgp_policy_labels=std.get(range, 'bgp_policy_labels', {}), ) { metadata+: { labels+: egw.espejoteLabel, From 771299a7eff9d622f63d432d2d948668e7d307f2 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 11 Nov 2025 09:18:41 +0100 Subject: [PATCH 5/5] Fix typos in documentation Co-authored-by: Sebastian Widmer --- docs/modules/ROOT/pages/references/parameters.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index 567c2f0ac..9e7dff920 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -402,8 +402,8 @@ Such policies assign the egress IP to all traffic originating in the namespace. ==== Field `bgp_policy_labels` is optional. -When the field is provide, and not an empty object, the component adds the contents as entries in `metadata.labels` of the resulting policies. -In this case, the component also configures the egress policies directly with the egress IP and `mmaxGatewayNodes: 1` in the `spec.egressGroups` entry. +When the field is provided, and not an empty object, the component adds the contents as entries in `metadata.labels` of the resulting policies. +In this case, the component also configures the egress policies directly with the egress IP and `maxGatewayNodes: 1` in the `spec.egressGroups` entry. If the field is omitted or empty, the component configures the egress group with the interface name and doesn't set `maxGatewayNodes`. ====