From da076b384e3a068ddecf09ff355bb920197e8145 Mon Sep 17 00:00:00 2001 From: Abhisek Dwivedi Date: Wed, 29 Mar 2023 09:03:59 +0530 Subject: [PATCH] KO-130: Added support for adding custom interfaces (#200) * Added support for custom interface type --- .../aerospikecluster_mutating_webhook.go | 46 +- api/v1beta1/aerospikecluster_types.go | 97 ++- .../aerospikecluster_validating_webhook.go | 113 +++ api/v1beta1/utils.go | 4 + api/v1beta1/zz_generated.deepcopy.go | 34 +- .../asdb.aerospike.com_aerospikeclusters.yaml | 128 +++ controllers/configmap.go | 2 +- controllers/pod.go | 15 +- controllers/scripts/common-env.sh | 74 +- controllers/scripts/create-aerospike-conf.sh | 24 +- test/aero_info.go | 8 + test/cluster_helper.go | 1 + test/network_policy_test.go | 804 ++++++++++++++++-- 13 files changed, 1227 insertions(+), 123 deletions(-) diff --git a/api/v1beta1/aerospikecluster_mutating_webhook.go b/api/v1beta1/aerospikecluster_mutating_webhook.go index cdf9b537..9a6490cc 100644 --- a/api/v1beta1/aerospikecluster_mutating_webhook.go +++ b/api/v1beta1/aerospikecluster_mutating_webhook.go @@ -60,7 +60,7 @@ func (c *AerospikeCluster) Default() admission.Response { func (c *AerospikeCluster) setDefaults(asLog logr.Logger) error { // Set network defaults - c.Spec.AerospikeNetworkPolicy.SetDefaults() + c.Spec.AerospikeNetworkPolicy.setDefaults(c.ObjectMeta.Namespace) // Set common storage defaults. c.Spec.Storage.SetDefaults() @@ -284,6 +284,37 @@ func (c *AerospikeCluster) setDefaultAerospikeConfigs( return nil } +// setDefaults applies default to unspecified fields on the network policy. +func (n *AerospikeNetworkPolicy) setDefaults(namespace string) { + if n.AccessType == AerospikeNetworkTypeUnspecified { + n.AccessType = AerospikeNetworkTypeHostInternal + } + + if n.AlternateAccessType == AerospikeNetworkTypeUnspecified { + n.AlternateAccessType = AerospikeNetworkTypeHostExternal + } + + if n.TLSAccessType == AerospikeNetworkTypeUnspecified { + n.TLSAccessType = AerospikeNetworkTypeHostInternal + } + + if n.TLSAlternateAccessType == AerospikeNetworkTypeUnspecified { + n.TLSAlternateAccessType = AerospikeNetworkTypeHostExternal + } + + // Set network namespace if not present + n.setNetworkNamespace(namespace) +} + +func (n *AerospikeNetworkPolicy) setNetworkNamespace(namespace string) { + setNamespaceDefault(n.CustomAccessNetworkNames, namespace) + setNamespaceDefault(n.CustomAlternateAccessNetworkNames, namespace) + setNamespaceDefault(n.CustomTLSAccessNetworkNames, namespace) + setNamespaceDefault(n.CustomTLSAlternateAccessNetworkNames, namespace) + setNamespaceDefault(n.CustomFabricNetworkNames, namespace) + setNamespaceDefault(n.CustomTLSFabricNetworkNames, namespace) +} + // ***************************************************************************** // Helper // ***************************************************************************** @@ -746,3 +777,16 @@ func escapeString(str string) string { return str } + +func setNamespaceDefault(networks []string, namespace string) { + for idx := range networks { + netName := strings.TrimSpace(networks[idx]) + namespacedName := strings.Split(netName, "/") + + if len(namespacedName) == 1 { + netName = namespace + "/" + netName + } + + networks[idx] = netName + } +} diff --git a/api/v1beta1/aerospikecluster_types.go b/api/v1beta1/aerospikecluster_types.go index b41622a9..dc4eba5b 100644 --- a/api/v1beta1/aerospikecluster_types.go +++ b/api/v1beta1/aerospikecluster_types.go @@ -718,7 +718,6 @@ type AerospikeClusterStatus struct { //nolint:govet // for readability } // AerospikeNetworkType specifies the type of network address to use. -// +kubebuilder:validation:Enum=pod;hostInternal;hostExternal;configuredIP // +k8s:openapi-gen=true type AerospikeNetworkType string @@ -743,44 +742,104 @@ const ( // label "aerospike.com/configured-access-address" in k8s node will be used as `accessAddress` // label "aerospike.com/configured-alternate-access-address" in k8s node will be used as `alternateAccessAddress` AerospikeNetworkTypeConfigured AerospikeNetworkType = "configuredIP" + + // AerospikeNetworkTypeCustomInterface specifies any other custom interface to be used with Aerospike + AerospikeNetworkTypeCustomInterface AerospikeNetworkType = "customInterface" ) // AerospikeNetworkPolicy specifies how clients and tools access the Aerospike cluster. type AerospikeNetworkPolicy struct { // AccessType is the type of network address to use for Aerospike access address. // Defaults to hostInternal. + // +kubebuilder:validation:Enum=pod;hostInternal;hostExternal;configuredIP;customInterface AccessType AerospikeNetworkType `json:"access,omitempty"` + // CustomAccessNetworkNames is the list of the pod's network interfaces used for Aerospike access address. + // Each element in the list is specified with a namespace and the name of a NetworkAttachmentDefinition, + // separated by a forward slash (/). + // These elements must be defined in the pod annotation k8s.v1.cni.cncf.io/networks in order to assign + // network interfaces to the pod. + // Required with 'customInterface' access type. + // +kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:MaxItems:=1 + CustomAccessNetworkNames []string `json:"customAccessNetworkNames,omitempty"` + // AlternateAccessType is the type of network address to use for Aerospike alternate access address. // Defaults to hostExternal. + // +kubebuilder:validation:Enum=pod;hostInternal;hostExternal;configuredIP;customInterface AlternateAccessType AerospikeNetworkType `json:"alternateAccess,omitempty"` + // CustomAlternateAccessNetworkNames is the list of the pod's network interfaces used for Aerospike + // alternate access address. + // Each element in the list is specified with a namespace and the name of a NetworkAttachmentDefinition, + // separated by a forward slash (/). + // These elements must be defined in the pod annotation k8s.v1.cni.cncf.io/networks in order to assign + // network interfaces to the pod. + // Required with 'customInterface' alternateAccess type + // +kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:MaxItems:=1 + CustomAlternateAccessNetworkNames []string `json:"customAlternateAccessNetworkNames,omitempty"` + // TLSAccessType is the type of network address to use for Aerospike TLS access address. // Defaults to hostInternal. + // +kubebuilder:validation:Enum=pod;hostInternal;hostExternal;configuredIP;customInterface TLSAccessType AerospikeNetworkType `json:"tlsAccess,omitempty"` + // CustomTLSAccessNetworkNames is the list of the pod's network interfaces used for Aerospike TLS access address. + // Each element in the list is specified with a namespace and the name of a NetworkAttachmentDefinition, + // separated by a forward slash (/). + // These elements must be defined in the pod annotation k8s.v1.cni.cncf.io/networks in order to assign + // network interfaces to the pod. + // Required with 'customInterface' tlsAccess type + // +kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:MaxItems:=1 + CustomTLSAccessNetworkNames []string `json:"customTLSAccessNetworkNames,omitempty"` + // TLSAlternateAccessType is the type of network address to use for Aerospike TLS alternate access address. // Defaults to hostExternal. + // +kubebuilder:validation:Enum=pod;hostInternal;hostExternal;configuredIP;customInterface TLSAlternateAccessType AerospikeNetworkType `json:"tlsAlternateAccess,omitempty"` -} - -// SetDefaults applies default to unspecified fields on the network policy. -func (n *AerospikeNetworkPolicy) SetDefaults() { - if n.AccessType == AerospikeNetworkTypeUnspecified { - n.AccessType = AerospikeNetworkTypeHostInternal - } - - if n.AlternateAccessType == AerospikeNetworkTypeUnspecified { - n.AlternateAccessType = AerospikeNetworkTypeHostExternal - } - if n.TLSAccessType == AerospikeNetworkTypeUnspecified { - n.TLSAccessType = AerospikeNetworkTypeHostInternal - } - - if n.TLSAlternateAccessType == AerospikeNetworkTypeUnspecified { - n.TLSAlternateAccessType = AerospikeNetworkTypeHostExternal - } + // CustomTLSAlternateAccessNetworkNames is the list of the pod's network interfaces used for Aerospike TLS + // alternate access address. + // Each element in the list is specified with a namespace and the name of a NetworkAttachmentDefinition, + // separated by a forward slash (/). + // These elements must be defined in the pod annotation k8s.v1.cni.cncf.io/networks in order to assign + // network interfaces to the pod. + // Required with 'customInterface' tlsAlternateAccess type + // +kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:MaxItems:=1 + CustomTLSAlternateAccessNetworkNames []string `json:"customTLSAlternateAccessNetworkNames,omitempty"` + + // FabricType is the type of network address to use for Aerospike fabric address. + // Defaults is empty meaning all interfaces 'any'. + // +kubebuilder:validation:Enum:=customInterface + FabricType AerospikeNetworkType `json:"fabric,omitempty"` + + // CustomFabricNetworkNames is the list of the pod's network interfaces used for Aerospike fabric address. + // Each element in the list is specified with a namespace and the name of a NetworkAttachmentDefinition, + // separated by a forward slash (/). + // These elements must be defined in the pod annotation k8s.v1.cni.cncf.io/networks in order to assign + // network interfaces to the pod. + // Required with 'customInterface' fabric type + // +kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:MaxItems:=1 + CustomFabricNetworkNames []string `json:"customFabricNetworkNames,omitempty"` + + // TLSFabricType is the type of network address to use for Aerospike TLS fabric address. + // Defaults is empty meaning all interfaces 'any'. + // +kubebuilder:validation:Enum:=customInterface + TLSFabricType AerospikeNetworkType `json:"tlsFabric,omitempty"` + + // CustomTLSFabricNetworkNames is the list of the pod's network interfaces used for Aerospike TLS fabric address. + // Each element in the list is specified with a namespace and the name of a NetworkAttachmentDefinition, + // separated by a forward slash (/). + // These elements must be defined in the pod annotation k8s.v1.cni.cncf.io/networks in order to assign network + // interfaces to the pod. + // Required with 'customInterface' tlsFabric type + // +kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:MaxItems:=1 + CustomTLSFabricNetworkNames []string `json:"customTLSFabricNetworkNames,omitempty"` } // AerospikeInstanceSummary defines the observed state of a pod's Aerospike Server Instance. diff --git a/api/v1beta1/aerospikecluster_validating_webhook.go b/api/v1beta1/aerospikecluster_validating_webhook.go index 32985ac7..1325c2a0 100644 --- a/api/v1beta1/aerospikecluster_validating_webhook.go +++ b/api/v1beta1/aerospikecluster_validating_webhook.go @@ -111,6 +111,12 @@ func (c *AerospikeCluster) ValidateUpdate(oldObj runtime.Object) error { return fmt.Errorf("cannot update MultiPodPerHost setting") } + if err := validateNetworkPolicyUpdate( + &old.Spec.AerospikeNetworkPolicy, &c.Spec.AerospikeNetworkPolicy, + ); err != nil { + return err + } + // Validate AerospikeConfig update if err := validateAerospikeConfigUpdate( aslog, incomingVersion, outgoingVersion, @@ -252,6 +258,10 @@ func (c *AerospikeCluster) validate(aslog logr.Logger) error { return err } + if err := c.validateNetworkPolicy(c.Namespace); err != nil { + return err + } + // Validate Sidecars if err := c.validatePodSpec(); err != nil { return err @@ -1334,6 +1344,28 @@ func validateNetworkConnectionUpdate( return nil } +func validateNetworkPolicyUpdate(oldPolicy, newPolicy *AerospikeNetworkPolicy) error { + if oldPolicy.FabricType != newPolicy.FabricType { + return fmt.Errorf("cannot update fabric type") + } + + if oldPolicy.TLSFabricType != newPolicy.TLSFabricType { + return fmt.Errorf("cannot update tlsFabric type") + } + + if newPolicy.FabricType == AerospikeNetworkTypeCustomInterface && + !reflect.DeepEqual(oldPolicy.CustomFabricNetworkNames, newPolicy.CustomFabricNetworkNames) { + return fmt.Errorf("cannot change/update customFabricNetworkNames") + } + + if newPolicy.TLSFabricType == AerospikeNetworkTypeCustomInterface && + !reflect.DeepEqual(oldPolicy.CustomTLSFabricNetworkNames, newPolicy.CustomTLSFabricNetworkNames) { + return fmt.Errorf("cannot change/update customTLSFabricNetworkNames") + } + + return nil +} + func validateNsConfUpdate(newConfSpec, oldConfSpec, currentStatus *AerospikeConfigSpec) error { newConf := newConfSpec.Value oldConf := oldConfSpec.Value @@ -1833,3 +1865,84 @@ func validateDNS(dnsPolicy v1.DNSPolicy, dnsConfig *v1.PodDNSConfig) error { return nil } + +func (c *AerospikeCluster) validateNetworkPolicy(namespace string) error { + networkPolicy := &c.Spec.AerospikeNetworkPolicy + + annotations := c.Spec.PodSpec.AerospikeObjectMeta.Annotations + networks := annotations["k8s.v1.cni.cncf.io/networks"] + networkList := strings.Split(networks, ",") + networkSet := sets.NewString() + + setNamespaceDefault(networkList, namespace) + + networkSet.Insert(networkList...) + + validateNetworkList := func(netList []string, addressType AerospikeNetworkType, listName string) error { + if netList == nil { + return fmt.Errorf("%s is required with 'customInterface' %s type", listName, addressType) + } + + if c.Spec.PodSpec.HostNetwork { + return fmt.Errorf("hostNetwork is not allowed with 'customInterface' network type") + } + + if !networkSet.HasAll(netList...) { + return fmt.Errorf( + "required networks %v not present in pod metadata annotations key \"k8s.v1.cni.cncf.io/networks\"", + netList) + } + + return nil + } + + if networkPolicy.AccessType == AerospikeNetworkTypeCustomInterface { + if err := validateNetworkList( + networkPolicy.CustomAccessNetworkNames, + "access", "customAccessNetworkNames"); err != nil { + return err + } + } + + if networkPolicy.AlternateAccessType == AerospikeNetworkTypeCustomInterface { + if err := validateNetworkList( + networkPolicy.CustomAlternateAccessNetworkNames, + "alternateAccess", "customAlternateAccessNetworkNames"); err != nil { + return err + } + } + + if networkPolicy.TLSAccessType == AerospikeNetworkTypeCustomInterface { + if err := validateNetworkList( + networkPolicy.CustomTLSAccessNetworkNames, + "tlsAccess", "customTLSAccessNetworkNames"); err != nil { + return err + } + } + + if networkPolicy.TLSAlternateAccessType == AerospikeNetworkTypeCustomInterface { + if err := validateNetworkList( + networkPolicy.CustomTLSAlternateAccessNetworkNames, + "tlsAlternateAccess", "customTLSAlternateAccessNetworkNames"); err != nil { + return err + } + } + + if networkPolicy.FabricType == AerospikeNetworkTypeCustomInterface { + if err := validateNetworkList( + networkPolicy.CustomFabricNetworkNames, + "fabric", "customFabricNetworkNames"); err != nil { + return err + } + } + + if networkPolicy.TLSFabricType == AerospikeNetworkTypeCustomInterface { + if err := validateNetworkList( + networkPolicy.CustomTLSFabricNetworkNames, + "tlsFabric", "customTLSFabricNetworkNames"); err != nil { + return err + } + } + + return nil +} diff --git a/api/v1beta1/utils.go b/api/v1beta1/utils.go index 8af03dc2..e6fa2aee 100644 --- a/api/v1beta1/utils.go +++ b/api/v1beta1/utils.go @@ -410,6 +410,10 @@ func GetFabricPort(aeroConf *AerospikeConfigSpec) *int { return GetPortFromConfig(aeroConf, confKeyNetworkFabric, "port") } +func GetFabricTLSPort(aeroConf *AerospikeConfigSpec) *int { + return GetPortFromConfig(aeroConf, confKeyNetworkFabric, "tls-port") +} + func GetPortFromConfig( aeroConf *AerospikeConfigSpec, connectionType string, paramName string, ) *int { diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index e82f031d..2dda6aba 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -170,7 +170,7 @@ func (in *AerospikeClusterSpec) DeepCopyInto(out *AerospikeClusterSpec) { **out = **in } in.RackConfig.DeepCopyInto(&out.RackConfig) - out.AerospikeNetworkPolicy = in.AerospikeNetworkPolicy + in.AerospikeNetworkPolicy.DeepCopyInto(&out.AerospikeNetworkPolicy) if in.OperatorClientCertSpec != nil { in, out := &in.OperatorClientCertSpec, &out.OperatorClientCertSpec *out = new(AerospikeOperatorClientCertSpec) @@ -242,7 +242,7 @@ func (in *AerospikeClusterStatusSpec) DeepCopyInto(out *AerospikeClusterStatusSp **out = **in } in.RackConfig.DeepCopyInto(&out.RackConfig) - out.AerospikeNetworkPolicy = in.AerospikeNetworkPolicy + in.AerospikeNetworkPolicy.DeepCopyInto(&out.AerospikeNetworkPolicy) if in.OperatorClientCertSpec != nil { in, out := &in.OperatorClientCertSpec, &out.OperatorClientCertSpec *out = new(AerospikeOperatorClientCertSpec) @@ -361,6 +361,36 @@ func (in *AerospikeInstanceSummary) DeepCopy() *AerospikeInstanceSummary { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AerospikeNetworkPolicy) DeepCopyInto(out *AerospikeNetworkPolicy) { *out = *in + if in.CustomAccessNetworkNames != nil { + in, out := &in.CustomAccessNetworkNames, &out.CustomAccessNetworkNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.CustomAlternateAccessNetworkNames != nil { + in, out := &in.CustomAlternateAccessNetworkNames, &out.CustomAlternateAccessNetworkNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.CustomTLSAccessNetworkNames != nil { + in, out := &in.CustomTLSAccessNetworkNames, &out.CustomTLSAccessNetworkNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.CustomTLSAlternateAccessNetworkNames != nil { + in, out := &in.CustomTLSAlternateAccessNetworkNames, &out.CustomTLSAlternateAccessNetworkNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.CustomFabricNetworkNames != nil { + in, out := &in.CustomFabricNetworkNames, &out.CustomFabricNetworkNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.CustomTLSFabricNetworkNames != nil { + in, out := &in.CustomTLSFabricNetworkNames, &out.CustomTLSFabricNetworkNames + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AerospikeNetworkPolicy. diff --git a/config/crd/bases/asdb.aerospike.com_aerospikeclusters.yaml b/config/crd/bases/asdb.aerospike.com_aerospikeclusters.yaml index c43718b4..d564db5c 100644 --- a/config/crd/bases/asdb.aerospike.com_aerospikeclusters.yaml +++ b/config/crd/bases/asdb.aerospike.com_aerospikeclusters.yaml @@ -136,6 +136,7 @@ spec: - hostInternal - hostExternal - configuredIP + - customInterface type: string alternateAccess: description: AlternateAccessType is the type of network address @@ -145,6 +146,61 @@ spec: - hostInternal - hostExternal - configuredIP + - customInterface + type: string + customAccessNetworkNames: + description: CustomAccessNetworkNames is the list of the pod's + network interfaces u + items: + type: string + maxItems: 1 + minItems: 1 + type: array + customAlternateAccessNetworkNames: + description: CustomAlternateAccessNetworkNames is the list of + the pod's network int + items: + type: string + maxItems: 1 + minItems: 1 + type: array + customFabricNetworkNames: + description: CustomFabricNetworkNames is the list of the pod's + network interfaces u + items: + type: string + maxItems: 1 + minItems: 1 + type: array + customTLSAccessNetworkNames: + description: CustomTLSAccessNetworkNames is the list of the pod's + network interface + items: + type: string + maxItems: 1 + minItems: 1 + type: array + customTLSAlternateAccessNetworkNames: + description: 'CustomTLSAlternateAccessNetworkNames is the list + of the pod''s network ' + items: + type: string + maxItems: 1 + minItems: 1 + type: array + customTLSFabricNetworkNames: + description: CustomTLSFabricNetworkNames is the list of the pod's + network interface + items: + type: string + maxItems: 1 + minItems: 1 + type: array + fabric: + description: 'FabricType is the type of network address to use + for Aerospike fabric ' + enum: + - customInterface type: string tlsAccess: description: 'TLSAccessType is the type of network address to @@ -154,6 +210,7 @@ spec: - hostInternal - hostExternal - configuredIP + - customInterface type: string tlsAlternateAccess: description: TLSAlternateAccessType is the type of network address @@ -163,6 +220,13 @@ spec: - hostInternal - hostExternal - configuredIP + - customInterface + type: string + tlsFabric: + description: 'TLSFabricType is the type of network address to + use for Aerospike TLS ' + enum: + - customInterface type: string type: object image: @@ -5939,6 +6003,7 @@ spec: - hostInternal - hostExternal - configuredIP + - customInterface type: string alternateAccess: description: AlternateAccessType is the type of network address @@ -5948,6 +6013,61 @@ spec: - hostInternal - hostExternal - configuredIP + - customInterface + type: string + customAccessNetworkNames: + description: CustomAccessNetworkNames is the list of the pod's + network interfaces u + items: + type: string + maxItems: 1 + minItems: 1 + type: array + customAlternateAccessNetworkNames: + description: CustomAlternateAccessNetworkNames is the list of + the pod's network int + items: + type: string + maxItems: 1 + minItems: 1 + type: array + customFabricNetworkNames: + description: CustomFabricNetworkNames is the list of the pod's + network interfaces u + items: + type: string + maxItems: 1 + minItems: 1 + type: array + customTLSAccessNetworkNames: + description: CustomTLSAccessNetworkNames is the list of the pod's + network interface + items: + type: string + maxItems: 1 + minItems: 1 + type: array + customTLSAlternateAccessNetworkNames: + description: 'CustomTLSAlternateAccessNetworkNames is the list + of the pod''s network ' + items: + type: string + maxItems: 1 + minItems: 1 + type: array + customTLSFabricNetworkNames: + description: CustomTLSFabricNetworkNames is the list of the pod's + network interface + items: + type: string + maxItems: 1 + minItems: 1 + type: array + fabric: + description: 'FabricType is the type of network address to use + for Aerospike fabric ' + enum: + - customInterface type: string tlsAccess: description: 'TLSAccessType is the type of network address to @@ -5957,6 +6077,7 @@ spec: - hostInternal - hostExternal - configuredIP + - customInterface type: string tlsAlternateAccess: description: TLSAlternateAccessType is the type of network address @@ -5966,6 +6087,13 @@ spec: - hostInternal - hostExternal - configuredIP + - customInterface + type: string + tlsFabric: + description: 'TLSFabricType is the type of network address to + use for Aerospike TLS ' + enum: + - customInterface type: string type: object image: diff --git a/controllers/configmap.go b/controllers/configmap.go index c7032e7b..f7c0a811 100644 --- a/controllers/configmap.go +++ b/controllers/configmap.go @@ -41,8 +41,8 @@ const ( ) type initializeTemplateInput struct { - NetworkPolicy asdbv1beta1.AerospikeNetworkPolicy WorkDir string + NetworkPolicy asdbv1beta1.AerospikeNetworkPolicy FabricPort int32 PodPort int32 PodTLSPort int32 diff --git a/controllers/pod.go b/controllers/pod.go index 17654c2b..f8b32508 100644 --- a/controllers/pod.go +++ b/controllers/pod.go @@ -709,15 +709,20 @@ func getFQDNForPod(aeroCluster *asdbv1beta1.AerospikeCluster, host string) strin return fmt.Sprintf("%s.%s.%s", host, aeroCluster.Name, aeroCluster.Namespace) } -// GetEndpointsFromInfo returns the aerospike service endpoints as a slice of host:port elements named addressName +// GetEndpointsFromInfo returns the aerospike endpoints as a slice of host:port based on context and addressName passed // from the info endpointsMap. It returns an empty slice if the access address with addressName is not found in // endpointsMap. -// // E.g. addressName are access, alternate-access func GetEndpointsFromInfo( - addressName string, endpointsMap map[string]string, + aeroCtx, addressName string, endpointsMap map[string]string, ) []string { - portStr, ok := endpointsMap["service."+addressName+"-port"] + addressKeyPrefix := aeroCtx + "." + + if addressName != "" { + addressKeyPrefix += addressName + "-" + } + + portStr, ok := endpointsMap[addressKeyPrefix+"port"] if !ok { return nil } @@ -727,7 +732,7 @@ func GetEndpointsFromInfo( return nil } - hostsStr, ok := endpointsMap["service."+addressName+"-addresses"] + hostsStr, ok := endpointsMap[addressKeyPrefix+"addresses"] if !ok { return nil } diff --git a/controllers/scripts/common-env.sh b/controllers/scripts/common-env.sh index 608ff852..a681a244 100644 --- a/controllers/scripts/common-env.sh +++ b/controllers/scripts/common-env.sh @@ -104,14 +104,6 @@ export EXTERNALIP=$(echo $HOSTIPS | awk '{print $2}') export CONFIGURED_ACCESSIP=$(echo $HOSTIPS | awk '{print $3}') export CONFIGURED_ALTERNATE_ACCESSIP=$(echo $HOSTIPS | awk '{print $4}') -if [[ "$CONFIGURED_ACCESSIP" == "NIL" ]]; then - CONFIGURED_ACCESSIP='' -fi - -if [[ "$CONFIGURED_ALTERNATE_ACCESSIP" == "NIL" ]]; then - CONFIGURED_ALTERNATE_ACCESSIP='' -fi - # Sets up port related variables. export POD_PORT="{{.PodPort}}" export POD_TLSPORT="{{.PodTLSPort}}" @@ -127,8 +119,70 @@ export MAPPED_PORT="$POD_PORT" export MAPPED_TLSPORT="$POD_TLSPORT" {{- end}} +# parseCustomNetworkIP function parses the network IPs for the given list of network names +# It parses network status information from pod annotations key `k8s.v1.cni.cncf.io/network-status` which is added by CNI +parseCustomNetworkIP() { + local networks=$1 + networks="$(echo "${networks}" | sed 's/\[//g'| sed 's/\]//g')" + + DATA="$(curl --cacert $CA_CERT -H "Authorization: Bearer $TOKEN" "$KUBE_API_SERVER/api/v1/namespaces/$NAMESPACE/pods/$MY_POD_NAME")" + + NETWORKIPS="$(echo $DATA | python3 -c "import sys, json +data = json.load(sys.stdin); +networks = '${networks}'; +networks = networks.strip().split(' ') +def getNetworkIP(data, networks): + annotations = data['metadata']['annotations'] + key = 'k8s.v1.cni.cncf.io/network-status' + if key in annotations: + netIPs = [] + for net in json.loads(annotations[key]): + if net['name'] in networks: + if 'ips' in net: + if len(net['ips']) == 0: + raise ValueError(\"ips list empty for network {} in pod annotations key k8s.v1.cni.cncf.io/network-status\".format(net['name'])) + netIPs.extend(net['ips']) + else: + raise KeyError(\"ips key missing for network {} in pod annotations key k8s.v1.cni.cncf.io/network-status\".format(net['name'])) + + if len(netIPs) == 0: + raise ValueError(\"networks {} not found in pod annotations key k8s.v1.cni.cncf.io/network-status\".format(networks)) + return ' '.join(netIPs) +print(getNetworkIP(data, networks))")" +} + +{{- if eq .NetworkPolicy.AccessType "customInterface"}} +parseCustomNetworkIP "{{.NetworkPolicy.CustomAccessNetworkNames}}" +export CUSTOM_ACCESS_NETWORK_IPS=${NETWORKIPS} +{{- end}} + +{{- if eq .NetworkPolicy.TLSAccessType "customInterface"}} +parseCustomNetworkIP "{{.NetworkPolicy.CustomTLSAccessNetworkNames}}" +export CUSTOM_TLS_ACCESS_NETWORK_IPS=${NETWORKIPS} +{{- end}} + +{{- if eq .NetworkPolicy.AlternateAccessType "customInterface"}} +parseCustomNetworkIP "{{.NetworkPolicy.CustomAlternateAccessNetworkNames}}" +export CUSTOM_ALTERNATE_ACCESS_NETWORK_IPS=${NETWORKIPS} +{{- end}} + +{{- if eq .NetworkPolicy.TLSAlternateAccessType "customInterface"}} +parseCustomNetworkIP "{{.NetworkPolicy.CustomTLSAlternateAccessNetworkNames}}" +export CUSTOM_TLS_ALTERNATE_ACCESS_NETWORK_IPS=${NETWORKIPS} +{{- end}} + +{{- if eq .NetworkPolicy.FabricType "customInterface"}} +parseCustomNetworkIP "{{.NetworkPolicy.CustomFabricNetworkNames}}" +export CUSTOM_FABRIC_NETWORK_IPS=${NETWORKIPS} +{{- end}} + +{{- if eq .NetworkPolicy.TLSFabricType "customInterface"}} +parseCustomNetworkIP "{{.NetworkPolicy.CustomTLSFabricNetworkNames}}" +export CUSTOM_TLS_FABRIC_NETWORK_IPS=${NETWORKIPS} +{{- end}} + # Parse out cluster name, formatted as: stsname-rackid-index -IFS='-' read -ra ADDR <<< "${MY_POD_NAME}" +IFS='-' read -ra ADDR <<<"${MY_POD_NAME}" POD_ORDINAL="${ADDR[-1]}" @@ -138,5 +192,5 @@ export NODE_ID="${RACK_ID}a${POD_ORDINAL}" GENERATED_ENV="/tmp/generate-env.sh" if [ -f $GENERATED_ENV ]; then - source $GENERATED_ENV + source $GENERATED_ENV fi diff --git a/controllers/scripts/create-aerospike-conf.sh b/controllers/scripts/create-aerospike-conf.sh index cc0eb101..bdaa31cd 100644 --- a/controllers/scripts/create-aerospike-conf.sh +++ b/controllers/scripts/create-aerospike-conf.sh @@ -49,6 +49,7 @@ substituteEndpoint() { local podPort=$6 local mappedPort=$7 local configuredIP=$8 + local interfaceIP=$9 case $networkType in pod) @@ -70,13 +71,18 @@ substituteEndpoint() { accessAddress=$configuredIP accessPort=$mappedPort - if [ -z "$configuredIP" ] + if [[ "$accessAddress" == "NIL" ]]; then echo "Please set '${CONFIGURED_ACCESSIP_LABEL}' and '${CONFIGURED_ALTERNATE_ACCESSIP_LABEL}' node label to use NetworkPolicy configuredIP for access and alternateAccess addresses" exit 1 fi ;; + customInterface) + accessAddress=$interfaceIP + accessPort=$podPort + ;; + *) accessAddress=$podIP accessPort=$podPort @@ -94,14 +100,22 @@ substituteEndpoint() { sed -i "s/^\(\s*\)${addressType}-port\s*${podPort}/\1${addressType}-port ${accessPort}/" ${CFG} } -substituteEndpoint "access" {{.NetworkPolicy.AccessType}} $PODIP $INTERNALIP $EXTERNALIP $POD_PORT $MAPPED_PORT $CONFIGURED_ACCESSIP -substituteEndpoint "alternate-access" {{.NetworkPolicy.AlternateAccessType}} $PODIP $INTERNALIP $EXTERNALIP $POD_PORT $MAPPED_PORT $CONFIGURED_ALTERNATE_ACCESSIP +substituteEndpoint "access" {{.NetworkPolicy.AccessType}} $PODIP $INTERNALIP $EXTERNALIP $POD_PORT $MAPPED_PORT $CONFIGURED_ACCESSIP $CUSTOM_ACCESS_NETWORK_IPS +substituteEndpoint "alternate-access" {{.NetworkPolicy.AlternateAccessType}} $PODIP $INTERNALIP $EXTERNALIP $POD_PORT $MAPPED_PORT $CONFIGURED_ALTERNATE_ACCESSIP $CUSTOM_ALTERNATE_ACCESS_NETWORK_IPS if [ "true" == "$MY_POD_TLS_ENABLED" ]; then - substituteEndpoint "tls-access" {{.NetworkPolicy.TLSAccessType}} $PODIP $INTERNALIP $EXTERNALIP $POD_TLSPORT $MAPPED_TLSPORT $CONFIGURED_ACCESSIP - substituteEndpoint "tls-alternate-access" {{.NetworkPolicy.TLSAlternateAccessType}} $PODIP $INTERNALIP $EXTERNALIP $POD_TLSPORT $MAPPED_TLSPORT $CONFIGURED_ALTERNATE_ACCESSIP + substituteEndpoint "tls-access" {{.NetworkPolicy.TLSAccessType}} $PODIP $INTERNALIP $EXTERNALIP $POD_TLSPORT $MAPPED_TLSPORT $CONFIGURED_ACCESSIP $CUSTOM_TLS_ACCESS_NETWORK_IPS + substituteEndpoint "tls-alternate-access" {{.NetworkPolicy.TLSAlternateAccessType}} $PODIP $INTERNALIP $EXTERNALIP $POD_TLSPORT $MAPPED_TLSPORT $CONFIGURED_ALTERNATE_ACCESSIP $CUSTOM_TLS_ALTERNATE_ACCESS_NETWORK_IPS fi +{{- if eq .NetworkPolicy.FabricType "customInterface"}} +sed -i -e "/fabric {/a \\ address ${CUSTOM_FABRIC_NETWORK_IPS}" ${CFG} +{{- end}} + +{{- if eq .NetworkPolicy.TLSFabricType "customInterface"}} +sed -i -e "/fabric {/a \\ tls-address ${CUSTOM_TLS_FABRIC_NETWORK_IPS}" ${CFG} +{{- end}} + # ------------------------------------------------------------------------------ # Update mesh seeds in the configuration file # ------------------------------------------------------------------------------ diff --git a/test/aero_info.go b/test/aero_info.go index 57a5f29b..86efff1c 100644 --- a/test/aero_info.go +++ b/test/aero_info.go @@ -164,6 +164,10 @@ func getEndpointIP( return "", fmt.Errorf( "can not use configured network type: %s", networkType, ) + case asdbv1beta1.AerospikeNetworkTypeCustomInterface: + return "", fmt.Errorf( + "%s not support yet", networkType, + ) case asdbv1beta1.AerospikeNetworkTypeUnspecified: return "", fmt.Errorf( "unknown network type: %s", networkType, @@ -210,6 +214,10 @@ func createHost(pod *asdbv1beta1.AerospikePodStatus) (*as.Host, error) { return nil, fmt.Errorf( "can not use configured network type: %s", networkType, ) + case asdbv1beta1.AerospikeNetworkTypeCustomInterface: + return nil, fmt.Errorf( + "%s not support yet", networkType, + ) case asdbv1beta1.AerospikeNetworkTypeUnspecified: return nil, fmt.Errorf( "unknown network type: %s", networkType, diff --git a/test/cluster_helper.go b/test/cluster_helper.go index 668401f6..b72f33e1 100644 --- a/test/cluster_helper.go +++ b/test/cluster_helper.go @@ -932,6 +932,7 @@ func createDummyAerospikeClusterWithRFAndStorage( return aeroCluster } +//nolint:unparam // generic func func createNonSCDummyAerospikeCluster( clusterNamespacedName types.NamespacedName, size int32, ) *asdbv1beta1.AerospikeCluster { diff --git a/test/network_policy_test.go b/test/network_policy_test.go index 1f00902e..4fbc2d8b 100644 --- a/test/network_policy_test.go +++ b/test/network_policy_test.go @@ -9,6 +9,7 @@ import ( "fmt" "net" "reflect" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -16,6 +17,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" asdbv1beta1 "github.com/aerospike/aerospike-kubernetes-operator/api/v1beta1" aerospikecluster "github.com/aerospike/aerospike-kubernetes-operator/controllers" @@ -29,6 +31,16 @@ const ( valueAccessAddress = "192.168.1.1" labelAlternateAccessAddress = "aerospike.com/configured-alternate-access-address" valueAlternateAccessAddress = "192.168.1.2" + networkOne = "ipvlan-conf-1" + networkTwo = "ipvlan-conf-2" + networkThree = "ipvlan-conf-3" + nsNetworkOne = "test1/ipvlan-conf-1" + nsNetworkTwo = "test1/ipvlan-conf-2" + customNetIPVlanOne = "10.0.4.70" + customNetIPVlanTwo = "10.0.6.70" + customNetIPVlanThree = "0.0.0.0" + networkAnnotationKey = "k8s.v1.cni.cncf.io/networks" + networkStatusAnnotationKey = "k8s.v1.cni.cncf.io/network-status" ) var _ = Describe( @@ -69,18 +81,214 @@ var _ = Describe( Context( "Negative cases for the NetworkPolicy", func() { - doNegativeTestNetworkPolicy(true, true, ctx) + negativeAerospikeNetworkPolicyTest(ctx, true, true) }, ) - }, ) -func doNegativeTestNetworkPolicy( - multiPodPerHost bool, enableTLS bool, ctx goctx.Context, -) { +func negativeAerospikeNetworkPolicyTest(ctx goctx.Context, multiPodPerHost, enableTLS bool) { Context( - "Negative cases for the configuredIP", func() { + "NegativeDeployNetworkPolicyTest", func() { + negativeDeployNetworkPolicyTest(ctx, multiPodPerHost, enableTLS) + }, + ) + + Context( + "NegativeUpdateNetworkPolicyTest", func() { + negativeUpdateNetworkPolicyTest(ctx) + }, + ) +} + +func negativeDeployNetworkPolicyTest(ctx goctx.Context, multiPodPerHost, enableTLS bool) { + Context( + "Negative cases for customInterface", func() { + clusterNamespacedName := getClusterNamespacedName("np-custom-interface", namespace) + + It( + "MissingCustomAccessNetworkNames: should fail when access is set to 'customInterface' and "+ + "customAccessNetworkNames is not given", + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.AccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "MissingCustomAlternateAccessNetworkNames: should fail when alternateAccess is set to "+ + "'customInterface' and customAlternateAccessNetworkNames is not given", + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.AlternateAccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "MissingCustomTLSAccessNetworkNames: should fail when tlsAccess is set to 'customInterface' and "+ + "customTLSAccessNetworkNames is not given", + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.TLSAccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "MissingCustomTLSAlternateAccessNetworkNames: should fail when tlsAlternateAccess is set to"+ + " 'customInterface' and customTLSAlternateAccessNetworkNames is not given", + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.TLSAlternateAccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "MissingCustomFabricNetworkNames: should fail when fabric is set to 'customInterface' and "+ + "customFabricNetworkNames is not given", + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.FabricType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "MissingCustomTLSFabricNetworkNames: should fail when tlsFabric is set to 'customInterface' and "+ + "customTLSFabricNetworkNames is not given", + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.TLSFabricType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "InvalidFabricType: should fail when fabric is set to value other than 'customInterface'", + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.FabricType = "invalid-enum" + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + // Following test-cases are applicable for all custom Interfaces + // Added test-case for only 'customAccessNetworkNames`, rest of the types will be similar to this only + It( + "MissingNetworkNameInPodAnnotations: should fail when access is set to 'customInterface' and "+ + "customAccessNetworkNames is not present in pod annotations", + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.AccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomAccessNetworkNames = []string{networkOne} + + // define different networks than the ones defined in CustomAccessNetworkNames + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: "ipvlan-conf-2, ipvlan-conf-3", + } + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "UsingHostNetworkAndCustomInterface: should fail when access is set to 'customInterface' and "+ + "Host network is used", + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.AccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomAccessNetworkNames = []string{networkOne} + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: networkOne, + } + aeroCluster.Spec.PodSpec.HostNetwork = true + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "NetworkNameOfDifferentNamespace: should fail when access is set to 'customInterface' and "+ + "customAccessNetworkNames present in pod annotations is of different namespace", + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.AccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomAccessNetworkNames = []string{"random/ipvlan-conf-1"} + + // define different networks than the ones defined in CustomAccessNetworkNames + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: networkOne, + } + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "InvalidCustomAccessNetworkNames: should fail when access is set to 'customInterface' and "+ + "customAccessNetworkNames is given and CNI not working (network-status annotation missing)", + func() { + aeroCluster := createNonSCDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.AccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomAccessNetworkNames = []string{networkOne} + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: networkOne, + } + + // cluster will crash as there will be no network status annotations + err := deployClusterWithTO( + k8sClient, ctx, aeroCluster, retryInterval, 2*time.Minute, + ) + Expect(err).Should(HaveOccurred()) + + err = deleteCluster(k8sClient, ctx, aeroCluster) + Expect(err).ToNot(HaveOccurred()) + }, + ) + + It( + "EmptyCustomAccessNetworkNames: should fail when access is set to 'customInterface' and "+ + "customAccessNetworkNames is set to empty slice", + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.AccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomAccessNetworkNames = []string{} + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "MaxAllowedNetworkNames: should fail when more than the max allowed network names(only 1) are "+ + "given in CR.", + func() { + aeroCluster := createDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.AccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomAccessNetworkNames = []string{ + nsNetworkOne, nsNetworkTwo, + } + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: "test1/ipvlan-conf-1, test1/ipvlan-conf-2", + } + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + }, + ) + + Context( + "Negative cases for configuredIP", func() { clusterNamespacedName := getClusterNamespacedName("np-configured-ip", multiClusterNs1) BeforeEach( @@ -108,7 +316,7 @@ func doNegativeTestNetworkPolicy( ) Expect(err).ToNot(HaveOccurred()) - networkPolicy := asdbv1beta1.AerospikeNetworkPolicy{ + networkPolicy := &asdbv1beta1.AerospikeNetworkPolicy{ AccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, TLSAccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, } @@ -126,7 +334,7 @@ func doNegativeTestNetworkPolicy( err := setNodeLabels(ctx, map[string]string{labelAccessAddress: valueAccessAddress}) Expect(err).ToNot(HaveOccurred()) - networkPolicy := asdbv1beta1.AerospikeNetworkPolicy{ + networkPolicy := &asdbv1beta1.AerospikeNetworkPolicy{ AlternateAccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, TLSAlternateAccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, } @@ -141,7 +349,7 @@ func doNegativeTestNetworkPolicy( ) It( "setting configured access-address and alternate-access-address without label", func() { - networkPolicy := asdbv1beta1.AerospikeNetworkPolicy{ + networkPolicy := &asdbv1beta1.AerospikeNetworkPolicy{ AccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, TLSAccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, AlternateAccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, @@ -160,9 +368,299 @@ func doNegativeTestNetworkPolicy( ) } +func negativeUpdateNetworkPolicyTest(ctx goctx.Context) { + Context("Negative cases for customInterface", func() { + clusterNamespacedName := getClusterNamespacedName("np-custom-interface", namespace) + Context( + "InvalidAerospikeCustomInterface", func() { + BeforeEach( + func() { + aeroCluster := createDummyAerospikeCluster( + clusterNamespacedName, 3, + ) + + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).ToNot(HaveOccurred()) + }, + ) + + AfterEach( + func() { + aeroCluster, err := getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + + _ = deleteCluster(k8sClient, ctx, aeroCluster) + }, + ) + + It( + "MissingCustomAccessNetworkNames: should fail when access is set to 'customInterface' and "+ + "customAccessNetworkNames is not given", + func() { + aeroCluster, err := getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + aeroCluster.Spec.AerospikeNetworkPolicy.AccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + err = updateCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "MissingCustomAlternateAccessNetworkNames: should fail when alternateAccess is set to "+ + "'customInterface' and customAlternateAccessNetworkNames is not given", + func() { + aeroCluster, err := getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + aeroCluster.Spec.AerospikeNetworkPolicy.AlternateAccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + err = updateCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "MissingCustomTLSAccessNetworkNames: should fail when tlsAccess is set to 'customInterface' and "+ + "customTLSAccessNetworkNames is not given", + func() { + aeroCluster, err := getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + aeroCluster.Spec.AerospikeNetworkPolicy.TLSAccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + err = updateCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "MissingCustomTLSAlternateAccessNetworkNames: should fail when tlsAlternateAccess is set "+ + "to 'customInterface' and customTLSAlternateAccessNetworkNames is not given", + func() { + aeroCluster, err := getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + aeroCluster.Spec.AerospikeNetworkPolicy.TLSAlternateAccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + err = updateCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "UpdatingFabricTypeInNetworkPolicy: should fail when fabric type is changed", + func() { + aeroCluster, err := getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + aeroCluster.Spec.AerospikeNetworkPolicy.FabricType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomFabricNetworkNames = []string{networkOne} + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: networkOne, + } + err = updateCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "UpdatingTLSFabricTypeInNetworkPolicy: should fail when TLS fabric type is changed", + func() { + aeroCluster, err := getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + aeroCluster.Spec.AerospikeNetworkPolicy.TLSFabricType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomTLSFabricNetworkNames = []string{networkOne} + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: networkOne, + } + err = updateCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + // Following test-cases are applicable for all custom Interfaces + // Added test-case for only 'customAccessNetworkNames`, rest of the types will be similar to this only + It( + "MissingNetworkNameInPodAnnotations: should fail when access is set to 'customInterface' and "+ + "customAccessNetworkNames is not present in pod annotations", + func() { + aeroCluster, err := getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + aeroCluster.Spec.AerospikeNetworkPolicy.AccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomAccessNetworkNames = []string{networkOne} + + // define different networks than the ones defined in CustomAccessNetworkNames + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: "ipvlan-conf-2, ipvlan-conf-3", + } + err = updateCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "UsingHostNetworkAndCustomInterface: should fail when access is set to 'customInterface' and "+ + "Host network is used", + func() { + aeroCluster, err := getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + aeroCluster.Spec.AerospikeNetworkPolicy.AccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomAccessNetworkNames = []string{networkOne} + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: networkOne, + } + aeroCluster.Spec.PodSpec.HostNetwork = true + err = updateCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "NetworkNameOfDifferentNamespace: should fail when access is set to 'customInterface' and "+ + "customAccessNetworkNames present in pod annotations is of different namespace", + func() { + aeroCluster, err := getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + aeroCluster.Spec.AerospikeNetworkPolicy.AccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomAccessNetworkNames = []string{"random/ipvlan-conf-1"} + + // define different networks than the ones defined in CustomAccessNetworkNames + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: networkOne, + } + err = updateCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + }, + ) + + Context("InvalidFabricNetworkNamesUpdate", func() { + var ( + aeroCluster *asdbv1beta1.AerospikeCluster + fabricNetStatusOne = `[{ + "name": "test1/ipvlan-conf-1", + "interface": "net1", + "ips": [ + "0.0.0.0" + ], + "mac": "06:18:e7:3e:50:65", + "dns": {} +}]` + fabricNetStatusTwo = `[{ + "name": "test1/ipvlan-conf-2", + "interface": "net2", + "ips": [ + "0.0.0.0" + ], + "mac": "06:18:e7:3e:50:65", + "dns": {} +}]` + ) + + AfterEach( + func() { + err := deleteCluster(k8sClient, ctx, aeroCluster) + Expect(err).ToNot(HaveOccurred()) + }, + ) + + It( + "UpdateCustomFabricNetworkNames: should fail when fabric is set to 'customInterface' and "+ + "customFabricNetworkNames list is updated", + func() { + aeroCluster = createDummyAerospikeCluster( + clusterNamespacedName, 2, + ) + aeroCluster.Spec.AerospikeNetworkPolicy.FabricType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomFabricNetworkNames = []string{nsNetworkOne} + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: nsNetworkOne, + networkStatusAnnotationKey: fabricNetStatusOne, + } + + By("Creating cluster with custom fabric interface") + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).ToNot(HaveOccurred()) + + aeroCluster, err = getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Updating custom fabric interface network list") + aeroCluster.Spec.AerospikeNetworkPolicy.CustomFabricNetworkNames = []string{nsNetworkTwo} + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: nsNetworkTwo, + networkStatusAnnotationKey: fabricNetStatusTwo, + } + + err = updateCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + + It( + "UpdateCustomTLSFabricNetworkNames: should fail when tlsFabric is set to 'customInterface' and "+ + "customTLSFabricNetworkNames list is updated", + func() { + aeroCluster = createDummyAerospikeCluster( + clusterNamespacedName, 2, + ) + aeroCluster.Spec.AerospikeNetworkPolicy.TLSFabricType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomTLSFabricNetworkNames = []string{nsNetworkOne} + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: nsNetworkOne, + networkStatusAnnotationKey: fabricNetStatusOne, + } + + By("Creating cluster with custom tlsfabric interface") + err := deployCluster(k8sClient, ctx, aeroCluster) + Expect(err).ToNot(HaveOccurred()) + + aeroCluster, err = getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Updating custom tlsFabric interface network list") + aeroCluster.Spec.AerospikeNetworkPolicy.CustomTLSFabricNetworkNames = []string{nsNetworkTwo} + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: nsNetworkTwo, + networkStatusAnnotationKey: fabricNetStatusTwo, + } + + err = updateCluster(k8sClient, ctx, aeroCluster) + Expect(err).Should(HaveOccurred()) + }, + ) + }) + }) +} + func doTestNetworkPolicy( multiPodPerHost bool, enableTLS bool, ctx goctx.Context, ) { + var aeroCluster *asdbv1beta1.AerospikeCluster + + AfterEach(func() { + err := deleteCluster(k8sClient, ctx, aeroCluster) + Expect(err).ToNot(HaveOccurred()) + }) + It( "DefaultNetworkPolicy", func() { clusterNamespacedName := getClusterNamespacedName( @@ -171,8 +669,8 @@ func doTestNetworkPolicy( // Ensures that default network policy is applied. defaultNetworkPolicy := asdbv1beta1.AerospikeNetworkPolicy{} - aeroCluster := getAerospikeClusterSpecWithNetworkPolicy( - clusterNamespacedName, defaultNetworkPolicy, multiPodPerHost, + aeroCluster = getAerospikeClusterSpecWithNetworkPolicy( + clusterNamespacedName, &defaultNetworkPolicy, multiPodPerHost, enableTLS, ) @@ -181,9 +679,6 @@ func doTestNetworkPolicy( err = validateNetworkPolicy(ctx, aeroCluster) Expect(err).ToNot(HaveOccurred()) - - err = deleteCluster(k8sClient, ctx, aeroCluster) - Expect(err).ToNot(HaveOccurred()) }, ) @@ -200,8 +695,8 @@ func doTestNetworkPolicy( TLSAccessType: asdbv1beta1.AerospikeNetworkTypePod, TLSAlternateAccessType: asdbv1beta1.AerospikeNetworkTypeHostExternal, } - aeroCluster := getAerospikeClusterSpecWithNetworkPolicy( - clusterNamespacedName, networkPolicy, multiPodPerHost, + aeroCluster = getAerospikeClusterSpecWithNetworkPolicy( + clusterNamespacedName, &networkPolicy, multiPodPerHost, enableTLS, ) @@ -210,16 +705,12 @@ func doTestNetworkPolicy( err = validateNetworkPolicy(ctx, aeroCluster) Expect(err).ToNot(HaveOccurred()) - - err = deleteCluster(k8sClient, ctx, aeroCluster) - Expect(err).ToNot(HaveOccurred()) }, ) Context( "When using configuredIP", func() { clusterNamespacedName := getClusterNamespacedName("np-configured-ip", multiClusterNs1) - BeforeEach( func() { err := deleteNodeLabels(ctx, []string{labelAccessAddress, labelAlternateAccessAddress}) @@ -227,26 +718,16 @@ func doTestNetworkPolicy( }, ) - AfterEach( - func() { - aeroCluster, err := getCluster(k8sClient, ctx, clusterNamespacedName) - Expect(err).ToNot(HaveOccurred()) - - err = deleteCluster(k8sClient, ctx, aeroCluster) - Expect(err).ToNot(HaveOccurred()) - }, - ) - It( "setting configured access-address", func() { err := setNodeLabels(ctx, map[string]string{labelAccessAddress: valueAccessAddress}) Expect(err).ToNot(HaveOccurred()) - networkPolicy := asdbv1beta1.AerospikeNetworkPolicy{ + networkPolicy := &asdbv1beta1.AerospikeNetworkPolicy{ AccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, TLSAccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, } - aeroCluster := getAerospikeClusterSpecWithNetworkPolicy( + aeroCluster = getAerospikeClusterSpecWithNetworkPolicy( clusterNamespacedName, networkPolicy, multiPodPerHost, enableTLS, ) @@ -267,11 +748,11 @@ func doTestNetworkPolicy( ) Expect(err).ToNot(HaveOccurred()) - networkPolicy := asdbv1beta1.AerospikeNetworkPolicy{ + networkPolicy := &asdbv1beta1.AerospikeNetworkPolicy{ AlternateAccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, TLSAlternateAccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, } - aeroCluster := getAerospikeClusterSpecWithNetworkPolicy( + aeroCluster = getAerospikeClusterSpecWithNetworkPolicy( clusterNamespacedName, networkPolicy, multiPodPerHost, enableTLS, ) @@ -293,13 +774,13 @@ func doTestNetworkPolicy( ) Expect(err).ToNot(HaveOccurred()) - networkPolicy := asdbv1beta1.AerospikeNetworkPolicy{ + networkPolicy := &asdbv1beta1.AerospikeNetworkPolicy{ AccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, AlternateAccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, TLSAccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, TLSAlternateAccessType: asdbv1beta1.AerospikeNetworkTypeConfigured, } - aeroCluster := getAerospikeClusterSpecWithNetworkPolicy( + aeroCluster = getAerospikeClusterSpecWithNetworkPolicy( clusterNamespacedName, networkPolicy, multiPodPerHost, enableTLS, ) @@ -313,6 +794,134 @@ func doTestNetworkPolicy( ) }, ) + + // All the custom interface related test-cases are tested by mocking the behavior of CNI plugins. + // Mocking is done by adding k8s.v1.cni.cncf.io/network-status manually in pod metadata annotations + // which is ideally added by CNI at runtime. + // Test cases with NetworkAttachmentDefinition of different namespaces can't be tested with current mocking. + Context("customInterface", func() { + clusterNamespacedName := getClusterNamespacedName( + "np-custom-interface", multiClusterNs1, + ) + + It( + "Should add all custom interface IPs in aerospike.conf file", func() { + networkPolicy := asdbv1beta1.AerospikeNetworkPolicy{ + AccessType: asdbv1beta1.AerospikeNetworkTypeCustomInterface, + CustomAccessNetworkNames: []string{networkOne}, + AlternateAccessType: asdbv1beta1.AerospikeNetworkTypeCustomInterface, + CustomAlternateAccessNetworkNames: []string{networkTwo}, + FabricType: asdbv1beta1.AerospikeNetworkTypeCustomInterface, + CustomFabricNetworkNames: []string{networkThree}, + } + + if enableTLS { + networkPolicy.TLSAccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + networkPolicy.CustomTLSAccessNetworkNames = []string{networkOne} + networkPolicy.TLSAlternateAccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + networkPolicy.CustomTLSAlternateAccessNetworkNames = []string{networkTwo} + networkPolicy.TLSFabricType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + networkPolicy.CustomTLSFabricNetworkNames = []string{networkThree} + } + + aeroCluster = getAerospikeClusterSpecWithNetworkPolicy( + clusterNamespacedName, &networkPolicy, multiPodPerHost, + enableTLS, + ) + + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + // Network Interfaces to be used + networkAnnotationKey: "test1/ipvlan-conf-1, ipvlan-conf-2, ipvlan-conf-3", + // CNI updates this network-status in the pod annotations + networkStatusAnnotationKey: `[{ + "name": "aws-cni", + "interface": "eth0", + "ips": [ + "10.0.2.114" + ], + "default": true, + "dns": {} +},{ + "name": "test1/ipvlan-conf-1", + "interface": "net1", + "ips": [ + "10.0.4.70" + ], + "mac": "06:18:e7:3e:50:65", + "dns": {} +},{ + "name": "test1/ipvlan-conf-2", + "interface": "net2", + "ips": [ + "10.0.6.70" + ], + "mac": "06:0d:8f:a4:be:f9", + "dns": {} +},{ + "name": "test1/ipvlan-conf-3", + "interface": "net3", + "ips": [ + "0.0.0.0" + ], + "mac": "06:0d:8f:a4:be:f8", + "dns": {} +}]`, + } + + err := aerospikeClusterCreateUpdate(k8sClient, aeroCluster, ctx) + Expect(err).ToNot(HaveOccurred()) + + err = validateNetworkPolicy(ctx, aeroCluster) + Expect(err).ToNot(HaveOccurred()) + }, + ) + + It("Should recover when correct custom network names are updated", func() { + aeroCluster = createNonSCDummyAerospikeCluster(clusterNamespacedName, 2) + aeroCluster.Spec.AerospikeNetworkPolicy.AccessType = asdbv1beta1.AerospikeNetworkTypeCustomInterface + aeroCluster.Spec.AerospikeNetworkPolicy.CustomAccessNetworkNames = []string{"missing-network"} + aeroCluster.Spec.PodSpec.AerospikeObjectMeta.Annotations = map[string]string{ + networkAnnotationKey: "missing-network, test1/ipvlan-conf-1 ", + networkStatusAnnotationKey: `[{ + "name": "aws-cni", + "interface": "eth0", + "ips": [ + "10.0.2.114" + ], + "default": true, + "dns": {} +},{ + "name": "test1/ipvlan-conf-1", + "interface": "net1", + "ips": [ + "10.0.4.70" + ], + "mac": "06:18:e7:3e:50:65", + "dns": {} +}]`, + } + + By("Creating cluster with wrong custom network name") + // cluster will crash as wrong custom interface is passed in CustomAccessNetworkNames + err := deployClusterWithTO( + k8sClient, ctx, aeroCluster, retryInterval, 2*time.Minute, + ) + Expect(err).Should(HaveOccurred()) + + aeroCluster, err = getCluster( + k8sClient, ctx, clusterNamespacedName, + ) + Expect(err).ToNot(HaveOccurred()) + + By("Updating correct custom network name") + aeroCluster.Spec.AerospikeNetworkPolicy.CustomAccessNetworkNames = []string{nsNetworkOne} + err = updateCluster(k8sClient, ctx, aeroCluster) + Expect(err).ToNot(HaveOccurred()) + + err = validateNetworkPolicy(ctx, aeroCluster) + Expect(err).ToNot(HaveOccurred()) + }) + }) } // validateNetworkPolicy validates that the new network policy is applied correctly. @@ -382,54 +991,66 @@ func validateNetworkPolicy( // Validate the returned endpoints. err = validatePodEndpoint( ctx, &podList.Items[podIndex], current, networkPolicy.AccessType, false, - aerospikecluster.GetEndpointsFromInfo("access", endpointsMap), valueAccessAddress, - ) + aerospikecluster.GetEndpointsFromInfo("service", "access", endpointsMap), + valueAccessAddress, customNetIPVlanOne, 0) if err != nil { return err } - err = validatePodEndpoint( - ctx, &podList.Items[podIndex], current, networkPolicy.AlternateAccessType, false, - aerospikecluster.GetEndpointsFromInfo( - "alternate-access", endpointsMap, - ), valueAlternateAccessAddress, - ) + err = validatePodEndpoint(ctx, &podList.Items[podIndex], current, networkPolicy.AlternateAccessType, false, + aerospikecluster.GetEndpointsFromInfo("service", "alternate-access", endpointsMap), + valueAlternateAccessAddress, customNetIPVlanTwo, 0) if err != nil { return err } + if networkPolicy.FabricType == asdbv1beta1.AerospikeNetworkTypeCustomInterface { + err = validatePodEndpoint( + ctx, &podList.Items[podIndex], current, networkPolicy.FabricType, false, + aerospikecluster.GetEndpointsFromInfo("fabric", "", endpointsMap), + "", customNetIPVlanThree, int32(*asdbv1beta1.GetFabricPort(desired.Spec.AerospikeConfig))) + if err != nil { + return err + } + } + tlsName := getServiceTLSName(current) if tlsName != "" { - err = validatePodEndpoint( - ctx, &podList.Items[podIndex], current, networkPolicy.TLSAccessType, true, - aerospikecluster.GetEndpointsFromInfo( - "tls-access", endpointsMap, - ), valueAccessAddress, - ) + err = validatePodEndpoint(ctx, &podList.Items[podIndex], current, networkPolicy.TLSAccessType, true, + aerospikecluster.GetEndpointsFromInfo("service", "tls-access", endpointsMap), + valueAccessAddress, customNetIPVlanOne, 0) if err != nil { return err } - err = validatePodEndpoint( - ctx, &podList.Items[podIndex], current, networkPolicy.TLSAlternateAccessType, true, - aerospikecluster.GetEndpointsFromInfo( - "tls-alternate-access", endpointsMap, - ), valueAlternateAccessAddress, - ) + err = validatePodEndpoint(ctx, &podList.Items[podIndex], current, networkPolicy.TLSAlternateAccessType, true, + aerospikecluster.GetEndpointsFromInfo("service", "tls-alternate-access", + endpointsMap), valueAlternateAccessAddress, customNetIPVlanTwo, 0) if err != nil { return err } + + if networkPolicy.TLSFabricType == asdbv1beta1.AerospikeNetworkTypeCustomInterface { + err = validatePodEndpoint( + ctx, &podList.Items[podIndex], current, networkPolicy.TLSFabricType, false, + aerospikecluster.GetEndpointsFromInfo("fabric", "tls", endpointsMap), + "", customNetIPVlanThree, int32(*asdbv1beta1.GetFabricTLSPort(desired.Spec.AerospikeConfig))) + if err != nil { + return err + } + } } } return nil } +// TODO: refactor this func to reduce number of params func validatePodEndpoint( - ctx goctx.Context, pod *corev1.Pod, - aeroCluster *asdbv1beta1.AerospikeCluster, - networkType asdbv1beta1.AerospikeNetworkType, isTLS bool, actual []string, configuredIP string, + ctx goctx.Context, pod *corev1.Pod, aeroCluster *asdbv1beta1.AerospikeCluster, + networkType asdbv1beta1.AerospikeNetworkType, isTLS bool, actual []string, configuredIP, customNetIP string, + customPort int32, ) error { podIP, hostInternalIP, hostExternalIP, _ := getIPs(ctx, pod) endpoint := actual[0] @@ -445,28 +1066,30 @@ func validatePodEndpoint( if podIP != host { return fmt.Errorf("expected podIP %v got %v", podIP, host) } - case asdbv1beta1.AerospikeNetworkTypeHostInternal: if hostInternalIP != host { return fmt.Errorf( "expected host internal IP %v got %v", hostInternalIP, host, ) } - case asdbv1beta1.AerospikeNetworkTypeHostExternal: if hostExternalIP != host { return fmt.Errorf( "expected host external IP %v got %v", hostExternalIP, host, ) } - case asdbv1beta1.AerospikeNetworkTypeConfigured: if configuredIP != host { return fmt.Errorf( "expected host configured IP %v got %v", configuredIP, host, ) } - + case asdbv1beta1.AerospikeNetworkTypeCustomInterface: + if customNetIP != host { + return fmt.Errorf( + "expected custom network IP %v got %v", customNetIP, host, + ) + } case asdbv1beta1.AerospikeNetworkTypeUnspecified: return fmt.Errorf( "network type cannot be unspecified", @@ -476,11 +1099,15 @@ func validatePodEndpoint( return fmt.Errorf("unknowk network type %v", networkType) } - // Validate port. - expectedPort, _ := getExpectedServicePortForPod( - aeroCluster, pod, networkType, isTLS, - ) + expectedPort := customPort + if customPort == 0 { + expectedPort, _ = getExpectedServicePortForPod( + aeroCluster, pod, networkType, isTLS, + ) + } + + // Validate port. if portStr != fmt.Sprintf("%v", expectedPort) { return fmt.Errorf( "incorrect port expected: %v actual: %v", expectedPort, portStr, @@ -496,7 +1123,8 @@ func getExpectedServicePortForPod( ) (int32, error) { var port int32 - if networkType != asdbv1beta1.AerospikeNetworkTypePod && aeroCluster.Spec.PodSpec.MultiPodPerHost { + if (networkType != asdbv1beta1.AerospikeNetworkTypePod && + networkType != asdbv1beta1.AerospikeNetworkTypeCustomInterface) && aeroCluster.Spec.PodSpec.MultiPodPerHost { svc, err := getServiceForPod(pod, k8sClient) if err != nil { return 0, fmt.Errorf("error getting service port: %v", err) @@ -560,7 +1188,7 @@ func getIPs(ctx goctx.Context, pod *corev1.Pod) ( // getAerospikeClusterSpecWithNetworkPolicy create a spec with input network policy. func getAerospikeClusterSpecWithNetworkPolicy( clusterNamespacedName types.NamespacedName, - networkPolicy asdbv1beta1.AerospikeNetworkPolicy, multiPodPerHost bool, + networkPolicy *asdbv1beta1.AerospikeNetworkPolicy, multiPodPerHost bool, enableTLS bool, ) *asdbv1beta1.AerospikeCluster { cascadeDelete := true @@ -674,7 +1302,7 @@ func getAerospikeClusterSpecWithNetworkPolicy( }, }, }, - AerospikeNetworkPolicy: networkPolicy, + AerospikeNetworkPolicy: *networkPolicy, }, } } @@ -686,12 +1314,20 @@ func setNodeLabels(ctx goctx.Context, labels map[string]string) error { } for idx := range nodeList.Items { - node := &nodeList.Items[idx] - for key, val := range labels { - node.Labels[key] = val - } + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + node := &nodeList.Items[idx] + + if err := k8sClient.Get( + ctx, types.NamespacedName{Name: node.Name}, node); err != nil { + return err + } - if err := k8sClient.Update(ctx, node); err != nil { + for key, val := range labels { + node.Labels[key] = val + } + + return k8sClient.Update(ctx, node) + }); err != nil { return err } } @@ -706,12 +1342,20 @@ func deleteNodeLabels(ctx goctx.Context, keys []string) error { } for idx := range nodeList.Items { - node := &nodeList.Items[idx] - for _, key := range keys { - delete(node.Labels, key) - } + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + node := &nodeList.Items[idx] + + if err := k8sClient.Get( + ctx, types.NamespacedName{Name: node.Name}, node); err != nil { + return err + } + + for _, key := range keys { + delete(node.Labels, key) + } - if err := k8sClient.Update(ctx, node); err != nil { + return k8sClient.Update(ctx, node) + }); err != nil { return err } }