Endpoint reflection with ExternalCIDR

Makefile

@@ -79,6 +79,28 @@ vet:
 generate: controller-gen
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
+# Generate gRPC files
+grpc: protoc
+	$(PROTOC) --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative pkg/liqonet/ipam.proto
+
+protoc:
+ifeq (, $(shell which protoc))
+	@{ \
+	PB_REL="https://github.com/protocolbuffers/protobuf/releases" ;\
+	version=3.15.5 ;\
+	arch=x86_64 ;\
+	curl -LO $${PB_REL}/download/v$${version}/protoc-$${version}-linux-$${arch}.zip ;\
+	unzip protoc-$${version}-linux-$${arch}.zip -d $${HOME}/.local ;\
+	rm protoc-$${version}-linux-$${arch}.zip ;\
+	PROTOC_TMP_DIR=$$(mktemp -d) ;\
+	cd $$PROTOC_TMP_DIR ;\
+	go mod init tmp ;\
+	go get google.golang.org/protobuf/cmd/protoc-gen-go ;\
+	go get google.golang.org/grpc/cmd/protoc-gen-go-grpc ;\
+	rm -rf $$PROTOC_TMP_DIR ;\
+	}
+endif
+PROTOC=$(shell which protoc)	
 
 # find or download controller-gen
 # download controller-gen if necessary

apis/net/v1alpha1/ipamstorage_types.go

@@ -23,12 +23,27 @@ import (
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
+// Subnets type contains relevant networks related to a remote cluster.
 type Subnets struct {
-	PodCIDR string `json:"podCIDR"`
-	// Network used in remote cluster for local service endpoints.
-	RemoteExternalCIDR string `json:"remoteExternalCIDR"`
+	// Network used in the remote cluster for local Pods. Default is "None": this means remote cluster uses local cluster PodCIDR.
+	LocalNATPodCIDR string `json:"localNATPodCIDR"`
+	// Network used for Pods in the remote cluster.
+	RemotePodCIDR string `json:"remotePodCIDR"`
+	// Network used in remote cluster for local service endpoints. Default is "None": this means remote cluster uses local cluster ExternalCIDR.
+	LocalNATExternalCIDR string `json:"localNATExternalCIDR"`
 	// Network used in local cluster for remote service endpoints.
-	LocalExternalCIDR string `json:"localExternalCIDR"`
+	RemoteExternalCIDR string `json:"remoteExternalCIDR"`
+}
+
+// ClusterMapping is an empty struct.
+type ClusterMapping struct{}
+
+// EndpointMapping describes a relation between an enpoint IP and an IP belonging to ExternalCIDR.
+type EndpointMapping struct {
+	// IP belonging to cluster ExtenalCIDR assigned to this endpoint.
+	IP string `json:"ip"`
+	// Set of clusters to which this endpoint has been reflected. Only the key, which is the ClusterID, is useful.
+	ClusterMappings map[string]ClusterMapping `json:"clusterMappings"`
 }
 
 // IpamSpec defines the desired state of Ipam.
@@ -44,6 +59,12 @@ type IpamSpec struct {
 	ClusterSubnets map[string]Subnets `json:"clusterSubnets"`
 	// Cluster ExternalCIDR
 	ExternalCIDR string `json:"externalCIDR"`
+	// Endpoint IP mappings. Key is the IP address of the local endpoint, value is the IP of the remote endpoint, so it belongs to an ExternalCIDR
+	EndpointMappings map[string]EndpointMapping `json:"endpointMappings"`
+	// Cluster PodCIDR
+	PodCIDR string `json:"podCIDR"`
+	// ServiceCIDR
+	ServiceCIDR string `json:"serviceCIDR"`
 }
 
 // +kubebuilder:object:root=true

cmd/liqonet/main.go

@@ -34,6 +34,7 @@ import (
 	route_operator "github.com/liqotech/liqo/internal/liqonet/route-operator"
 	tunnel_operator "github.com/liqotech/liqo/internal/liqonet/tunnel-operator"
 	"github.com/liqotech/liqo/internal/liqonet/tunnelEndpointCreator"
+	liqoconst "github.com/liqotech/liqo/pkg/consts"
 	liqonetOperator "github.com/liqotech/liqo/pkg/liqonet"
 	"github.com/liqotech/liqo/pkg/liqonet/wireguard"
 	"github.com/liqotech/liqo/pkg/mapperUtils"
@@ -132,9 +133,9 @@ func main() {
 	case "tunnelEndpointCreator-operator":
 		dynClient := dynamic.NewForConfigOrDie(mgr.GetConfig())
 		ipam := liqonetOperator.NewIPAM()
-		err = ipam.Init(liqonetOperator.Pools, dynClient)
+		err = ipam.Init(liqonetOperator.Pools, dynClient, liqoconst.NetworkManagerIpamPort)
 		if err != nil {
-			klog.Errorf("cannot init IPAM:%s", err.Error())
+			klog.Errorf("cannot init IPAM:%w", err)
 		}
 		r := &tunnelEndpointCreator.TunnelEndpointCreator{
 			Client:                     mgr.GetClient(),

deployments/liqo/crds/net.liqo.io_ipamstorages.yaml

@@ -38,29 +38,66 @@ spec:
             properties:
               clusterSubnets:
                 additionalProperties:
+                  description: Subnets type contains relevant networks related to
+                    a remote cluster.
                   properties:
-                    localExternalCIDR:
-                      description: Network used in local cluster for remote service
-                        endpoints.
+                    localNATExternalCIDR:
+                      description: 'Network used in remote cluster for local service
+                        endpoints. Default is "None": this means remote cluster uses
+                        local cluster ExternalCIDR.'
                       type: string
-                    podCIDR:
+                    localNATPodCIDR:
+                      description: 'Network used in the remote cluster for local Pods.
+                        Default is "None": this means remote cluster uses local cluster
+                        PodCIDR.'
                       type: string
                     remoteExternalCIDR:
-                      description: Network used in remote cluster for local service
+                      description: Network used in local cluster for remote service
                         endpoints.
                       type: string
+                    remotePodCIDR:
+                      description: Network used for Pods in the remote cluster.
+                      type: string
                   required:
-                  - localExternalCIDR
-                  - podCIDR
+                  - localNATExternalCIDR
+                  - localNATPodCIDR
                   - remoteExternalCIDR
+                  - remotePodCIDR
                   type: object
                 description: Map used to keep track of networks assigned to clusters.
                   Key is the remote cluster ID, value is a the set of networks used
                   by the remote cluster.
                 type: object
+              endpointMappings:
+                additionalProperties:
+                  description: EndpointMapping describes a relation between an enpoint
+                    IP and an IP belonging to ExternalCIDR.
+                  properties:
+                    clusterMappings:
+                      additionalProperties:
+                        description: ClusterMapping is an empty struct.
+                        type: object
+                      description: Set of clusters to which this endpoint has been
+                        reflected. Only the key, which is the ClusterID, is useful.
+                      type: object
+                    ip:
+                      description: IP belonging to cluster ExtenalCIDR assigned to
+                        this endpoint.
+                      type: string
+                  required:
+                  - clusterMappings
+                  - ip
+                  type: object
+                description: Endpoint IP mappings. Key is the IP address of the local
+                  endpoint, value is the IP of the remote endpoint, so it belongs
+                  to an ExternalCIDR
+                type: object
               externalCIDR:
                 description: Cluster ExternalCIDR
                 type: string
+              podCIDR:
+                description: Cluster PodCIDR
+                type: string
               pools:
                 description: Network pools.
                 items:
@@ -74,11 +111,17 @@ spec:
                   Important: Run "make" to regenerate code after modifying this file
                   Map consumed by go-ipam module. Key is prefic cidr, value is a Prefix.'
                 type: object
+              serviceCIDR:
+                description: ServiceCIDR
+                type: string
             required:
             - clusterSubnets
+            - endpointMappings
             - externalCIDR
+            - podCIDR
             - pools
             - prefixes
+            - serviceCIDR
             type: object
         type: object
     served: true

deployments/liqo/templates/liqo-network-manager-deployment.yaml

@@ -29,6 +29,9 @@ spec:
           imagePullPolicy: {{ .Values.pullPolicy }}
           name: {{ $netManagerConfig.name }}
           command: ["/usr/bin/liqonet"]
+          ports:
+            - name: ipam-api
+              containerPort: 6000
           args:
             - "-run-as=tunnelEndpointCreator-operator"
           resources:

deployments/liqo/templates/liqo-network-manager-service.yaml

@@ -0,0 +1,18 @@
+---
+{{- $netManagerConfig := (merge (dict "name" "network-manager" "module" "networking") .) -}}
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "liqo.prefixedName" $netManagerConfig }}
+  labels:
+    {{- include "liqo.labels" $netManagerConfig | nindent 4 }}
+spec:
+  # This service is made to be consumed within the cluster, in particular by the virtual kubelet
+  type: ClusterIP
+  ports:
+    - name: ipam-api
+      port: 6000
+      protocol: TCP
+  selector:
+    {{- include "liqo.selectorLabels" $netManagerConfig | nindent 4 }}

go.mod

@@ -30,6 +30,9 @@ require (
 	golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e
 	golang.org/x/tools v0.1.0
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20200609130330-bd2cb7843e1b
+	google.golang.org/grpc v1.33.2
+	google.golang.org/protobuf v1.25.0
+	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
 	gotest.tools v2.2.0+incompatible
 	inet.af/netaddr v0.0.0-20210313195008-843b4240e319
 	k8s.io/api v0.21.0

internal/liqonet/tunnelEndpointCreator/tunnelEndpointCreator-config.go

@@ -19,15 +19,21 @@ import (
 func (tec *TunnelEndpointCreator) setNetParameters(config *configv1alpha1.ClusterConfig) {
 	podCIDR := config.Spec.LiqonetConfig.PodCIDR
 	serviceCIDR := config.Spec.LiqonetConfig.ServiceCIDR
-	externalCIDR, err := tec.IPManager.GetClusterExternalCIDR(liqonet.GetMask(podCIDR))
+	externalCIDR, err := tec.IPManager.GetExternalCIDR(liqonet.GetMask(podCIDR))
 	if err != nil {
 		klog.Error(err)
 	}
 	if tec.PodCIDR != podCIDR {
+		if err := tec.IPManager.SetPodCIDR(podCIDR); err != nil {
+			klog.Error(err)
+		}
 		klog.Infof("PodCIDR set to %s", podCIDR)
 		tec.PodCIDR = podCIDR
 	}
 	if tec.ServiceCIDR != serviceCIDR {
+		if err := tec.IPManager.SetServiceCIDR(serviceCIDR); err != nil {
+			klog.Error(err)
+		}
 		klog.Infof("ServiceCIDR set to %s", serviceCIDR)
 		tec.ServiceCIDR = serviceCIDR
 	}
@@ -156,7 +162,6 @@ func (tec *TunnelEndpointCreator) updatePools(additionalPools []string) error {
 func (tec *TunnelEndpointCreator) getReservedSubnets(config *configv1alpha1.ClusterConfig) []string {
 	reservedSubnets := make([]string, 0)
 	liqonetConfig := config.Spec.LiqonetConfig
-	reservedSubnets = append(reservedSubnets, liqonetConfig.PodCIDR, liqonetConfig.ServiceCIDR)
 	// Cast CIDR to normal string and append
 	for _, cidr := range liqonetConfig.ReservedSubnets {
 		reservedSubnets = append(reservedSubnets, string(cidr))
@@ -189,6 +194,7 @@ func (tec *TunnelEndpointCreator) WatchConfiguration(config *rest.Config, gv *sc
 	go utils.WatchConfiguration(func(configuration *configv1alpha1.ClusterConfig) {
 		reservedSubnets := tec.getReservedSubnets(configuration)
 		additionalPools := tec.getAdditionalPools(configuration)
+		tec.setNetParameters(configuration)
 		err = tec.updateReservedSubnets(reservedSubnets)
 		if err != nil {
 			klog.Error(err)
@@ -199,7 +205,6 @@ func (tec *TunnelEndpointCreator) WatchConfiguration(config *rest.Config, gv *sc
 			klog.Error(err)
 			return
 		}
-		tec.setNetParameters(configuration)
 		if !tec.cfgConfigured {
 			tec.WaitConfig.Done()
 			klog.Infof("called done on waitgroup")

internal/liqonet/tunnelEndpointCreator/tunnelEndpointCreator-operator.go

@@ -191,6 +191,9 @@ func (tec *TunnelEndpointCreator) Reconcile(ctx context.Context, req ctrl.Reques
 		if err := tec.IPManager.FreeSubnetsPerCluster(netConfig.Spec.ClusterID); err != nil {
 			klog.Errorf("cannot free networks assigned to cluster %s", netConfig.Spec.ClusterID)
 		}
+		if err := tec.IPManager.RemoveLocalSubnetsPerCluster(netConfig.Spec.ClusterID); err != nil {
+			klog.Errorf("cannot delete local subnets assigned to cluster %s", netConfig.Spec.ClusterID)
+		}
 		return result, nil
 	}
 
@@ -222,6 +225,8 @@ func (tec *TunnelEndpointCreator) SetupSignalHandlerForTunEndCreator() context.C
 	go func(r *TunnelEndpointCreator) {
 		sig := <-c
 		klog.Infof("received signal: %s", sig.String())
+		// Stop IPAM gRPC server.
+		tec.IPManager.StopGRPCServer()
 		// closing shared informers
 		close(r.ForeignClusterStopWatcher)
 		done()
@@ -472,10 +477,19 @@ func (tec *TunnelEndpointCreator) processLocalNetConfig(netConfig *netv1alpha1.N
 	if !netConfigList.Items[0].Status.Processed {
 		return nil
 	}
-	// Store ExternalCIDR used in remote cluster
-	if err := tec.IPManager.AddExternalCIDRPerCluster(netConfig.Status.ExternalCIDRNAT,
-		netConfig.Spec.ClusterID); err != nil {
-		klog.Error(err)
+
+	retryError := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		// Store subnets used in remote cluster
+		if err := tec.IPManager.AddLocalSubnetsPerCluster(netConfig.Status.PodCIDRNAT,
+			netConfig.Status.ExternalCIDRNAT,
+			netConfig.Spec.ClusterID); err != nil {
+			return err
+		}
+		return nil
+	})
+	if retryError != nil {
+		klog.Error(retryError)
+		return retryError
 	}
 	// at this point we have all the necessary parameters to create the tunnelEndpoint resource
 	remoteNetConf := netConfigList.Items[0]

pkg/consts/liqonet.go

@@ -0,0 +1,12 @@
+package consts
+
+const (
+	// NetworkManagerIpamPort is the port used by IPAM gRPCs.
+	NetworkManagerIpamPort = 6000
+	// NetworkManagerServiceName is the service name for IPAM gRPCs.
+	NetworkManagerServiceName = "liqo-network-manager"
+	// DefaultCIDRValue is the default value for a string that contains a CIDR.
+	DefaultCIDRValue = "None"
+	// TepReady is the ready state of TunnelEndpoint resource.
+	TepReady = "Ready"
+)