Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

multi-pool: Determine IP pool based on ipam.cilium.io/ip-pool annotation #25511

Merged
merged 8 commits into from
May 30, 2023
Merged

multi-pool: Determine IP pool based on ipam.cilium.io/ip-pool annotation #25511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f479c3f
k8s: Add stream for local pod resources
gandro May 17, 2023
f6fbd84
api: Add IPAM pool name to allocated addresses
gandro May 17, 2023
082016e
endpoint: Track IPAM pool of endpoint address
gandro May 17, 2023
14f6806
annotation: Add `ipam.cilium.io/ip-pool`
gandro May 17, 2023
45fcb9d
ipam: Add IPAM metadata manager
gandro May 17, 2023
91b4d6f
ipam: Remove unused ability to release IPs by owner name
gandro May 23, 2023
5fd0df3
ipam: Make allocateIPsLocked a bit more robust
gandro May 25, 2023
580382f
ipam: Hook up IPAM metadata manager
gandro May 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/v1/client/ipam/delete_ipam_ip_parameters.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions api/v1/models/address_pair.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 7 additions & 7 deletions api/v1/openapi.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,7 @@ paths:
tags:
- ipam
parameters:
- "$ref": "#/parameters/ipam-release-arg"
- "$ref": "#/parameters/ipam-ip"
- "$ref": "#/parameters/ipam-pool"
responses:
'200':
Expand Down Expand Up @@ -1115,12 +1115,6 @@ parameters:
in: path
required: true
type: string
ipam-release-arg:
name: ip
description: IP address or owner name
in: path
required: true
type: string
ipam-family:
name: family
in: query
Expand Down Expand Up @@ -1686,12 +1680,18 @@ definitions:
ipv4-expiration-uuid:
description: UUID of IPv4 expiration timer
type: string
ipv4-pool-name:
description: IPAM pool from which this IPv4 address was allocated
type: string
ipv6:
description: IPv6 address
type: string
ipv6-expiration-uuid:
description: UUID of IPv6 expiration timer
type: string
ipv6-pool-name:
description: IPAM pool from which this IPv6 address was allocated
type: string
Address:
description: IP address
type: string
Expand Down
34 changes: 18 additions & 16 deletions api/v1/server/embedded_spec.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion api/v1/server/restapi/ipam/delete_ipam_ip_parameters.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions cilium-health/launch/endpoint.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,11 +245,13 @@ func LaunchAsEndpoint(baseCtx context.Context,

if healthIPv6 := node.GetEndpointHealthIPv6(); healthIPv6 != nil {
info.Addressing.IPV6 = healthIPv6.String()
info.Addressing.IPV6PoolName = ipamOption.PoolDefault
ip6Address = &net.IPNet{IP: healthIPv6, Mask: defaults.ContainerIPv6Mask}
healthIP = healthIPv6
}
if healthIPv4 := node.GetEndpointHealthIPv4(); healthIPv4 != nil {
info.Addressing.IPV4 = healthIPv4.String()
info.Addressing.IPV4PoolName = ipamOption.PoolDefault
ip4Address = &net.IPNet{IP: healthIPv4, Mask: defaults.ContainerIPv4Mask}
healthIP = healthIPv4
}
Expand Down
4 changes: 4 additions & 0 deletions daemon/cmd/cells.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/cilium/cilium/pkg/endpointmanager"
"github.com/cilium/cilium/pkg/gops"
"github.com/cilium/cilium/pkg/hive/cell"
ipamMetadata "github.com/cilium/cilium/pkg/ipam/metadata"
"github.com/cilium/cilium/pkg/k8s"
k8sClient "github.com/cilium/cilium/pkg/k8s/client"
"github.com/cilium/cilium/pkg/node"
Expand Down Expand Up @@ -109,6 +110,9 @@ var (
// IPCache, policy.Repository and CachingIdentityAllocator.
cell.Provide(newPolicyTrifecta),

// IPAM metadata manager, determines which IPAM pool a pod should allocate from
ipamMetadata.Cell,

// Egress Gateway allows originating traffic from specific IPv4 addresses.
egressgateway.Cell,

Expand Down
4 changes: 4 additions & 0 deletions daemon/cmd/daemon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ import (
"github.com/cilium/cilium/pkg/identity"
"github.com/cilium/cilium/pkg/identity/identitymanager"
"github.com/cilium/cilium/pkg/ipam"
ipamMetadata "github.com/cilium/cilium/pkg/ipam/metadata"
ipamOption "github.com/cilium/cilium/pkg/ipam/option"
"github.com/cilium/cilium/pkg/ipcache"
"github.com/cilium/cilium/pkg/k8s"
Expand Down Expand Up @@ -189,6 +190,8 @@ type Daemon struct {

cgroupManager *manager.CgroupManager

ipamMetadata *ipamMetadata.Manager

apiLimiterSet *rate.APILimiterSet

// event queue for serializing configuration updates to the daemon.
Expand Down Expand Up @@ -533,6 +536,7 @@ func newDaemon(ctx context.Context, cleaner *daemonCleanup, params *daemonParams
policy: params.Policy,
policyUpdater: params.PolicyUpdater,
egressGatewayManager: params.EgressGatewayManager,
ipamMetadata: params.IPAMMetadataManager,
cniConfigManager: params.CNIConfigManager,
clustermesh: params.ClusterMesh,
}
Expand Down
2 changes: 2 additions & 0 deletions daemon/cmd/daemon_main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ import (
"github.com/cilium/cilium/pkg/hubble/exporter/exporteroption"
"github.com/cilium/cilium/pkg/hubble/observer/observeroption"
"github.com/cilium/cilium/pkg/identity"
ipamMetadata "github.com/cilium/cilium/pkg/ipam/metadata"
ipamOption "github.com/cilium/cilium/pkg/ipam/option"
"github.com/cilium/cilium/pkg/ipcache"
"github.com/cilium/cilium/pkg/ipmasq"
Expand Down Expand Up @@ -1622,6 +1623,7 @@ type daemonParams struct {
PolicyUpdater *policy.Updater
IPCache *ipcache.IPCache
EgressGatewayManager *egressgateway.Manager
IPAMMetadataManager *ipamMetadata.Manager
CNIConfigManager cni.CNIConfigManager
SwaggerSpec *server.Spec
HealthAPISpec *healthApi.Spec
Expand Down
10 changes: 6 additions & 4 deletions daemon/cmd/endpoint.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -541,14 +541,16 @@ func (d *Daemon) createEndpoint(ctx context.Context, owner regeneration.Owner, e
if addressing := epTemplate.Addressing; addressing != nil {
if uuid := addressing.IPV4ExpirationUUID; uuid != "" {
if ip := net.ParseIP(addressing.IPV4); ip != nil {
if err := d.ipam.StopExpirationTimer(ip, ipam.PoolDefault, uuid); err != nil {
pool := ipam.PoolOrDefault(addressing.IPV4PoolName)
if err := d.ipam.StopExpirationTimer(ip, pool, uuid); err != nil {
return d.errorDuringCreation(ep, err)
}
}
}
if uuid := addressing.IPV6ExpirationUUID; uuid != "" {
if ip := net.ParseIP(addressing.IPV6); ip != nil {
if err := d.ipam.StopExpirationTimer(ip, ipam.PoolDefault, uuid); err != nil {
pool := ipam.PoolOrDefault(addressing.IPV4PoolName)
if err := d.ipam.StopExpirationTimer(ip, pool, uuid); err != nil {
return d.errorDuringCreation(ep, err)
}
}
Expand Down Expand Up @@ -758,13 +760,13 @@ func (d *Daemon) EndpointDeleted(ep *endpoint.Endpoint, conf endpoint.DeleteConf

if !conf.NoIPRelease {
if option.Config.EnableIPv4 {
if err := d.ipam.ReleaseIP(ep.IPv4.AsSlice(), ipam.PoolDefault); err != nil {
if err := d.ipam.ReleaseIP(ep.IPv4.AsSlice(), ipam.PoolOrDefault(ep.IPv4IPAMPool)); err != nil {
scopedLog := ep.Logger(daemonSubsys).WithError(err)
scopedLog.Warning("Unable to release IPv4 address during endpoint deletion")
}
}
if option.Config.EnableIPv6 {
if err := d.ipam.ReleaseIP(ep.IPv6.AsSlice(), ipam.PoolDefault); err != nil {
if err := d.ipam.ReleaseIP(ep.IPv6.AsSlice(), ipam.PoolOrDefault(ep.IPv6IPAMPool)); err != nil {
scopedLog := ep.Logger(daemonSubsys).WithError(err)
scopedLog.Warning("Unable to release IPv6 address during endpoint deletion")
}
Expand Down
18 changes: 14 additions & 4 deletions daemon/cmd/ipam.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func NewPostIPAMHandler(d *Daemon) ipamapi.PostIpamHandler {
func (h *postIPAM) Handle(params ipamapi.PostIpamParams) middleware.Responder {
family := strings.ToLower(swag.StringValue(params.Family))
owner := swag.StringValue(params.Owner)
pool := ipam.PoolOrDefault(swag.StringValue(params.Pool))
pool := ipam.Pool(swag.StringValue(params.Pool))
var expirationTimeout time.Duration
if swag.BoolValue(params.Expiration) {
expirationTimeout = defaults.IPAMExpiration
Expand All @@ -58,6 +58,7 @@ func (h *postIPAM) Handle(params ipamapi.PostIpamParams) middleware.Responder {

if ipv4Result != nil {
resp.Address.IPV4 = ipv4Result.IP.String()
resp.Address.IPV4PoolName = ipv4Result.IPPoolName.String()
resp.IPV4 = &models.IPAMAddressResponse{
Cidrs: ipv4Result.CIDRs,
IP: ipv4Result.IP.String(),
Expand All @@ -70,6 +71,7 @@ func (h *postIPAM) Handle(params ipamapi.PostIpamParams) middleware.Responder {

if ipv6Result != nil {
resp.Address.IPV6 = ipv6Result.IP.String()
resp.Address.IPV6PoolName = ipv6Result.IPPoolName.String()
resp.IPV6 = &models.IPAMAddressResponse{
Cidrs: ipv6Result.CIDRs,
IP: ipv6Result.IP.String(),
Expand Down Expand Up @@ -97,7 +99,7 @@ func NewPostIPAMIPHandler(d *Daemon) ipamapi.PostIpamIPHandler {
// Handle incoming requests address allocation requests for the daemon.
func (h *postIPAMIP) Handle(params ipamapi.PostIpamIPParams) middleware.Responder {
owner := swag.StringValue(params.Owner)
pool := ipam.PoolOrDefault(swag.StringValue(params.Pool))
pool := ipam.Pool(swag.StringValue(params.Pool))
if err := h.daemon.ipam.AllocateIPString(params.IP, owner, pool); err != nil {
return api.Error(ipamapi.PostIpamIPFailureCode, err)
}
Expand All @@ -123,8 +125,13 @@ func (h *deleteIPAMIP) Handle(params ipamapi.DeleteIpamIPParams) middleware.Resp
return api.Error(ipamapi.DeleteIpamIPFailureCode, fmt.Errorf("IP is in use by endpoint %d", ep.ID))
}

pool := ipam.PoolOrDefault(swag.StringValue(params.Pool))
if err := h.daemon.ipam.ReleaseIPString(params.IP, pool); err != nil {
ip := net.ParseIP(params.IP)
if ip == nil {
return api.Error(ipamapi.DeleteIpamIPInvalidCode, fmt.Errorf("Invalid IP address: %s", params.IP))
}

pool := ipam.Pool(swag.StringValue(params.Pool))
if err := h.daemon.ipam.ReleaseIP(ip, pool); err != nil {
return api.Error(ipamapi.DeleteIpamIPFailureCode, err)
}

Expand Down Expand Up @@ -554,6 +561,9 @@ func (d *Daemon) startIPAM() {
log.Info("Initializing node addressing")
// Set up ipam conf after init() because we might be running d.conf.KVStoreIPv4Registration
d.ipam = ipam.NewIPAM(d.datapath.LocalNodeAddressing(), option.Config, d.nodeDiscovery, d.k8sWatcher, &d.mtuConfig, d.clientset)
if d.ipamMetadata != nil {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I read the code correctly, d.ipamMetadata is already created via hive? Why add a fallback here, and not where the ipamMetaddata is initialized?

Copy link
Member Author

@gandro gandro May 25, 2023 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The hive ipamMetaddata constructor will return nil if it's running in an unsupported IPAM mode:
https://github.com/cilium/cilium/blob/0d4ba6a6d4edb21c92f6e469566d4d408c2ac31b/pkg/ipam/metadata/cell.go#L34-L36

And I do not want to pass in a potentially nil value as an interface type into WithMetadata because it will break the nil checks here (i.e. they will not detect that metadata is nil due to nil interface types):
https://github.com/cilium/cilium/blob/0d4ba6a6d4edb21c92f6e469566d4d408c2ac31b/pkg/ipam/allocator.go#L34-L36

Example: https://go.dev/play/p/z8C_cU0eMdU

d.ipam.WithMetadata(d.ipamMetadata)
}
bootstrapStats.ipam.End(true)
}

Expand Down
12 changes: 6 additions & 6 deletions daemon/cmd/state.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,24 +359,24 @@ func (d *Daemon) regenerateRestoredEndpoints(state *endpointRestoreState) (resto
return
}

func (d *Daemon) allocateIPsLocked(ep *endpoint.Endpoint) error {
var err error

func (d *Daemon) allocateIPsLocked(ep *endpoint.Endpoint) (err error) {
if option.Config.EnableIPv6 && ep.IPv6.IsValid() {
_, err = d.ipam.AllocateIPWithoutSyncUpstream(ep.IPv6.AsSlice(), ep.HumanStringLocked()+" [restored]", ipam.PoolDefault)
ipv6Pool := ipam.PoolOrDefault(ep.IPv6IPAMPool)
_, err = d.ipam.AllocateIPWithoutSyncUpstream(ep.IPv6.AsSlice(), ep.HumanStringLocked()+" [restored]", ipv6Pool)
if err != nil {
return fmt.Errorf("unable to reallocate %s IPv6 address: %w", ep.IPv6, err)
}

defer func() {
if err != nil {
d.ipam.ReleaseIP(ep.IPv6.AsSlice(), ipam.PoolDefault)
d.ipam.ReleaseIP(ep.IPv6.AsSlice(), ipv6Pool)
}
}()
}

if option.Config.EnableIPv4 && ep.IPv4.IsValid() {
_, err = d.ipam.AllocateIPWithoutSyncUpstream(ep.IPv4.AsSlice(), ep.HumanStringLocked()+" [restored]", ipam.PoolDefault)
ipv4Pool := ipam.PoolOrDefault(ep.IPv4IPAMPool)
_, err = d.ipam.AllocateIPWithoutSyncUpstream(ep.IPv4.AsSlice(), ep.HumanStringLocked()+" [restored]", ipv4Pool)
switch {
// We only check for BypassIPAllocUponRestore for IPv4 because we
// assume that this flag is only turned on for IPv4-only IPAM modes
Expand Down
7 changes: 7 additions & 0 deletions pkg/annotation/k8s.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ const (
// ServicePrefix is the common prefix for service related annotations.
ServicePrefix = "service.cilium.io"

// IPAMPrefix is the common prefix for IPAM related annotations.
IPAMPrefix = "ipam.cilium.io"

// PolicyName / PolicyNameAlias is an optional annotation to the NetworkPolicy
// resource which specifies the name of the policy node to which all
// rules should be applied to.
Expand Down Expand Up @@ -119,6 +122,10 @@ const (
// BGPVRouterAnnoPrefix is the prefix used for all Virtual Router annotations
// Its just a prefix, because the ASN of the Router is part of the annotation itself
BGPVRouterAnnoPrefix = "cilium.io/bgp-virtual-router."

// IPAMPoolKey is the annotation name used to store the IPAM pool name from
// which workloads should allocate their IP from
IPAMPoolKey = IPAMPrefix + "/ip-pool"
)

// Get returns the annotation value associated with the given key, or any of
Expand Down