diff --git a/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler.go b/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler.go index 6801a31d4c..1b17cc7a01 100644 --- a/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler.go +++ b/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler.go @@ -20,9 +20,12 @@ import ( ) const ( - NCStateInitialized = "Initialized" // NC has been initialized by DNC. - NCStateSucceeded = "Succeeded" // NC has been persisted by CNS - NCStateTerminated = "Terminated" // NC has been cleaned up from CNS + // NCStateInitialized indicates the NC has been initialized by DNC. + NCStateInitialized = "Initialized" + // NCStateSucceeded indicates the NC has been persisted by CNS. + NCStateSucceeded = "Succeeded" + // NCStateTerminated indicates the NC has been terminated by CNS. + NCStateTerminated = "Terminated" ) // multiTenantCrdReconciler reconciles multi-tenant network containers. @@ -35,20 +38,23 @@ type multiTenantCrdReconciler struct { // Reconcile is called on multi-tenant CRD status changes. func (r *multiTenantCrdReconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { ctx := context.Background() - var nc ncapi.MultiTenantNetworkContainer + logger.Printf("Reconcling MultiTenantNetworkContainer %v", request.NamespacedName.String()) + var nc ncapi.MultiTenantNetworkContainer if err := r.KubeClient.Get(ctx, request.NamespacedName, &nc); err != nil { if apierrors.IsNotFound(err) { + logger.Printf("MultiTenantNetworkContainer %s not found, skip reconciling", request.NamespacedName.String()) return ctrl.Result{}, nil } - logger.Errorf("Failed to fetch network container: %v", err) + logger.Errorf("Failed to fetch network container %s: %v", request.NamespacedName.String(), err) return ctrl.Result{}, err } if !nc.ObjectMeta.DeletionTimestamp.IsZero() { // Do nothing if the NC has already in Terminated state. if nc.Status.State == NCStateTerminated { + logger.Printf("MultiTenantNetworkContainer %s already terminated, skip reconciling", request.NamespacedName.String()) return ctrl.Result{}, nil } @@ -57,38 +63,27 @@ func (r *multiTenantCrdReconciler) Reconcile(request reconcile.Request) (reconci NetworkContainerid: nc.Spec.UUID, }) if err != nil { - logger.Errorf("Failed to delete NC %s from CNS: %v", nc.Spec.UUID, err) + logger.Errorf("Failed to delete NC %s (UUID: %s) from CNS: %v", request.NamespacedName.String(), nc.Spec.UUID, err) return ctrl.Result{}, err } // Update NC state to Terminated. nc.Status.State = NCStateTerminated if err := r.KubeClient.Status().Update(ctx, &nc); err != nil { - logger.Errorf("Failed to update network container state for %s: %v", nc.Spec.UUID, err) + logger.Errorf("Failed to update network container state for %s (UUID: %s): %v", request.NamespacedName.String(), nc.Spec.UUID, err) return ctrl.Result{}, err } - logger.Printf("NC has been terminated for %s", nc.Spec.UUID) + logger.Printf("NC has been terminated for %s (UUID: %s)", request.NamespacedName.String(), nc.Spec.UUID) return ctrl.Result{}, nil } // Do nothing if the network container hasn't been initialized yet from control plane. if nc.Status.State != NCStateInitialized { + logger.Printf("MultiTenantNetworkContainer %s hasn't initialized yet, skip reconciling", request.NamespacedName.String()) return ctrl.Result{}, nil } - // Check CNS NC states. - _, err := r.CNSClient.GetNC(cns.GetNetworkContainerRequest{ - NetworkContainerid: nc.Spec.UUID, - }) - if err == nil { - logger.Printf("NC %s has already been created in CNS", nc.Spec.UUID) - return ctrl.Result{}, nil - } else if err.Error() != "NotFound" { - logger.Errorf("Failed to fetch NC from CNS: %v", err) - return ctrl.Result{}, err - } - // Parse KubernetesPodInfo as orchestratorContext. podInfo := cns.KubernetesPodInfo{ PodName: nc.Name, @@ -100,6 +95,19 @@ func (r *multiTenantCrdReconciler) Reconcile(request reconcile.Request) (reconci return ctrl.Result{}, err } + // Check CNS NC states. + _, err = r.CNSClient.GetNC(cns.GetNetworkContainerRequest{ + NetworkContainerid: nc.Spec.UUID, + OrchestratorContext: orchestratorContext, + }) + if err == nil { + logger.Printf("NC %s (UUID: %s) has already been created in CNS", request.NamespacedName.String(), nc.Spec.UUID) + return ctrl.Result{}, nil + } else if err.Error() != "NotFound" { + logger.Errorf("Failed to fetch NC %s (UUID: %s) from CNS: %v", request.NamespacedName.String(), nc.Spec.UUID, err) + return ctrl.Result{}, err + } + // Persist NC states into CNS. _, ipNet, err := net.ParseCIDR(nc.Status.IPSubnet) if err != nil { @@ -108,8 +116,10 @@ func (r *multiTenantCrdReconciler) Reconcile(request reconcile.Request) (reconci } prefixLength, _ := ipNet.Mask.Size() networkContainerRequest := cns.CreateNetworkContainerRequest{ - NetworkContainerid: nc.Spec.UUID, - OrchestratorContext: orchestratorContext, + NetworkContainerid: nc.Spec.UUID, + OrchestratorContext: orchestratorContext, + NetworkContainerType: cns.Kubernetes, + Version: "0", IPConfiguration: cns.IPConfiguration{ IPSubnet: cns.IPSubnet{ IPAddress: nc.Status.IP, @@ -117,20 +127,22 @@ func (r *multiTenantCrdReconciler) Reconcile(request reconcile.Request) (reconci }, GatewayIPAddress: nc.Status.Gateway, }, + PrimaryInterfaceIdentifier: nc.Status.PrimaryInterfaceIdentifier, } + logger.Printf("CreateOrUpdateNC with networkContainerRequest: %#v", networkContainerRequest) if err = r.CNSClient.CreateOrUpdateNC(networkContainerRequest); err != nil { - logger.Errorf("Failed to persist state for NC %s to CNS: %v", nc.Spec.UUID, err) + logger.Errorf("Failed to persist state for NC %s (UUID: %s) to CNS: %v", request.NamespacedName.String(), nc.Spec.UUID, err) return ctrl.Result{}, err } // Update NC state to Succeeded. nc.Status.State = NCStateSucceeded if err := r.KubeClient.Status().Update(ctx, &nc); err != nil { - logger.Errorf("Failed to update network container state for %s: %v", nc.Spec.UUID, err) + logger.Errorf("Failed to update network container state for %s (UUID: %s): %v", request.NamespacedName.String(), nc.Spec.UUID, err) return ctrl.Result{}, err } - logger.Printf("Reconciled NC %s", nc.Spec.UUID) + logger.Printf("Reconciled NC %s (UUID: %s)", request.NamespacedName.String(), nc.Spec.UUID) return reconcile.Result{}, nil } diff --git a/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler_test.go b/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler_test.go index 928d7b05ed..8b0250d758 100644 --- a/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler_test.go +++ b/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler_test.go @@ -27,6 +27,10 @@ var _ = Describe("multiTenantCrdReconciler", func() { Namespace: "test", Name: "test", } + var podInfo = cns.KubernetesPodInfo{ + PodName: namespacedName.Name, + PodNamespace: namespacedName.Namespace, + } BeforeEach(func() { logger.InitLogger("multiTenantCrdReconciler", 0, 0, "") @@ -101,6 +105,10 @@ var _ = Describe("multiTenantCrdReconciler", func() { It("Should succeed when the NC is in Initialized state and it has already been persisted in CNS", func() { var uuid = "uuid" var nc ncapi.MultiTenantNetworkContainer = ncapi.MultiTenantNetworkContainer{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + }, Spec: ncapi.MultiTenantNetworkContainerSpec{ UUID: uuid, }, @@ -108,9 +116,16 @@ var _ = Describe("multiTenantCrdReconciler", func() { State: "Initialized", }, } + + orchestratorContext, err := json.Marshal(podInfo) + Expect(err).To(BeNil()) + kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc) - cnsClient.EXPECT().GetNC(cns.GetNetworkContainerRequest{NetworkContainerid: uuid}).Return(cns.GetNetworkContainerResponse{}, nil) - _, err := reconciler.Reconcile(reconcile.Request{ + cnsClient.EXPECT().GetNC(cns.GetNetworkContainerRequest{ + NetworkContainerid: uuid, + OrchestratorContext: orchestratorContext, + }).Return(cns.GetNetworkContainerResponse{}, nil) + _, err = reconciler.Reconcile(reconcile.Request{ NamespacedName: namespacedName, }) Expect(err).To(BeNil()) @@ -119,6 +134,10 @@ var _ = Describe("multiTenantCrdReconciler", func() { It("Should fail when the NC subnet isn't in correct format", func() { var uuid = "uuid" var nc ncapi.MultiTenantNetworkContainer = ncapi.MultiTenantNetworkContainer{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + }, Spec: ncapi.MultiTenantNetworkContainerSpec{ UUID: uuid, }, @@ -127,9 +146,16 @@ var _ = Describe("multiTenantCrdReconciler", func() { IPSubnet: "1.2.3.4.5", }, } + + orchestratorContext, err := json.Marshal(podInfo) + Expect(err).To(BeNil()) + kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc) - cnsClient.EXPECT().GetNC(cns.GetNetworkContainerRequest{NetworkContainerid: uuid}).Return(cns.GetNetworkContainerResponse{}, fmt.Errorf("NotFound")) - _, err := reconciler.Reconcile(reconcile.Request{ + cnsClient.EXPECT().GetNC(cns.GetNetworkContainerRequest{ + NetworkContainerid: uuid, + OrchestratorContext: orchestratorContext, + }).Return(cns.GetNetworkContainerResponse{}, fmt.Errorf("NotFound")) + _, err = reconciler.Reconcile(reconcile.Request{ NamespacedName: namespacedName, }) Expect(err).NotTo(BeNil()) @@ -158,8 +184,10 @@ var _ = Describe("multiTenantCrdReconciler", func() { }) Expect(err).To(BeNil()) var networkContainerRequest = cns.CreateNetworkContainerRequest{ - NetworkContainerid: nc.Spec.UUID, - OrchestratorContext: orchestratorContext, + NetworkContainerid: nc.Spec.UUID, + NetworkContainerType: cns.Kubernetes, + OrchestratorContext: orchestratorContext, + Version: "0", IPConfiguration: cns.IPConfiguration{ IPSubnet: cns.IPSubnet{ IPAddress: nc.Status.IP, @@ -173,7 +201,10 @@ var _ = Describe("multiTenantCrdReconciler", func() { statusWriter := mockclients.NewMockStatusWriter(mockCtl) statusWriter.EXPECT().Update(gomock.Any(), gomock.Any()).Return(nil) kubeClient.EXPECT().Status().Return(statusWriter) - cnsClient.EXPECT().GetNC(cns.GetNetworkContainerRequest{NetworkContainerid: uuid}).Return(cns.GetNetworkContainerResponse{}, fmt.Errorf("NotFound")) + cnsClient.EXPECT().GetNC(cns.GetNetworkContainerRequest{ + NetworkContainerid: uuid, + OrchestratorContext: orchestratorContext, + }).Return(cns.GetNetworkContainerResponse{}, fmt.Errorf("NotFound")) cnsClient.EXPECT().CreateOrUpdateNC(networkContainerRequest).Return(nil) _, err = reconciler.Reconcile(reconcile.Request{ NamespacedName: namespacedName, diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go index 8cc6e0ca97..5761cc304d 100644 --- a/cns/restserver/internalapi.go +++ b/cns/restserver/internalapi.go @@ -304,7 +304,7 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.C // For now only RequestController uses this API which will be initialized only for AKS scenario. // Validate ContainerType is set as Docker - if service.state.OrchestratorType != cns.KubernetesCRD { + if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { logger.Errorf("[Azure CNS] Error. Unsupported OrchestratorType: %s", service.state.OrchestratorType) return UnsupportedOrchestratorType } diff --git a/cns/restserver/util.go b/cns/restserver/util.go index aea915cd3d..eb9ab95eea 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -146,6 +146,8 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw fallthrough case cns.Docker: fallthrough + case cns.Kubernetes: + fallthrough case cns.Basic: fallthrough case cns.JobObject: @@ -664,7 +666,7 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ctx context.Context, } func (service *HTTPRestService) validateIpConfigRequest(ipConfigRequest cns.IPConfigRequest) (cns.PodInfo, int, string) { - if service.state.OrchestratorType != cns.KubernetesCRD { + if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { return nil, UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator" } diff --git a/crds/multitenantnetworkcontainer/api/v1alpha1/multitenantnetworkcontainer_types.go b/crds/multitenantnetworkcontainer/api/v1alpha1/multitenantnetworkcontainer_types.go index 5095575542..67bd75c8d9 100644 --- a/crds/multitenantnetworkcontainer/api/v1alpha1/multitenantnetworkcontainer_types.go +++ b/crds/multitenantnetworkcontainer/api/v1alpha1/multitenantnetworkcontainer_types.go @@ -48,6 +48,8 @@ type MultiTenantNetworkContainerStatus struct { State string `json:"state,omitempty"` // The subnet CIDR IPSubnet string `json:"ipSubnet,omitempty"` + // The primary interface identifier + PrimaryInterfaceIdentifier string `json:"primaryInterfaceIdentifier,omitempty"` } // +kubebuilder:object:root=true diff --git a/crds/multitenantnetworkcontainer/manifests/networking.azure.com_multitenantnetworkcontainers.yaml b/crds/multitenantnetworkcontainer/manifests/networking.azure.com_multitenantnetworkcontainers.yaml index 9f47b2479e..a6ec51433b 100644 --- a/crds/multitenantnetworkcontainer/manifests/networking.azure.com_multitenantnetworkcontainers.yaml +++ b/crds/multitenantnetworkcontainer/manifests/networking.azure.com_multitenantnetworkcontainers.yaml @@ -70,6 +70,9 @@ spec: ipSubnet: description: The subnet CIDR type: string + primaryInterfaceIdentifier: + description: The primary interface identifier + type: string state: description: The state of network container type: string