diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index c2cbe60f00..093dbb1f90 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -61,6 +61,13 @@ const ( PendingRelease = "PendingRelease" ) +// ChannelMode :- CNS channel modes +const ( + Direct = "Direct" + Managed = "Managed" + CRD = "CRD" +) + // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary. type CreateNetworkContainerRequest struct { Version string @@ -318,3 +325,9 @@ func (networkContainerRequestPolicy *NetworkContainerRequestPolicies) Validate() } return nil } + +// NodeInfoResponse - Struct to hold the node info response. +type NodeInfoResponse struct { + NetworkContainers []CreateNetworkContainerRequest + GetNCVersionURLFmt string +} diff --git a/cns/common/service.go b/cns/common/service.go index 1d09a06e51..e17e936b58 100644 --- a/cns/common/service.go +++ b/cns/common/service.go @@ -13,11 +13,12 @@ import ( // Service implements behavior common to all services. type Service struct { - Name string - Version string - Options map[string]interface{} - ErrChan chan error - Store store.KeyValueStore + Name string + Version string + Options map[string]interface{} + ErrChan chan error + Store store.KeyValueStore + ChannelMode string } // ServiceAPI defines base interface. @@ -30,25 +31,27 @@ type ServiceAPI interface { // ServiceConfig specifies common configuration. type ServiceConfig struct { - Name string - Version string - Listener *acn.Listener - ErrChan chan error - Store store.KeyValueStore + Name string + Version string + Listener *acn.Listener + ErrChan chan error + Store store.KeyValueStore + ChannelMode string } // NewService creates a new Service object. -func NewService(name, version string, store store.KeyValueStore) (*Service, error) { +func NewService(name, version, channelMode string, store store.KeyValueStore) (*Service, error) { logger.Debugf("[Azure CNS] Going to create a service object with name: %v. version: %v.", name, version) svc := &Service{ - Name: name, - Version: version, - Options: make(map[string]interface{}), - Store: store, + Name: name, + Version: version, + ChannelMode: channelMode, + Options: make(map[string]interface{}), + Store: store, } - logger.Debugf("[Azure CNS] Finished creating service object with name: %v. version: %v.", name, version) + logger.Debugf("[Azure CNS] Finished creating service object with name: %v. version: %v. managed: %s", name, version, channelMode) return svc, nil } @@ -65,6 +68,7 @@ func (service *Service) Initialize(config *ServiceConfig) error { service.ErrChan = config.ErrChan service.Store = config.Store service.Version = config.Version + service.ChannelMode = config.ChannelMode logger.Debugf("[Azure CNS] nitialized service: %+v with config: %+v.", service, config) diff --git a/cns/configuration/cns_config.json b/cns/configuration/cns_config.json index 44e134e297..dc4ef96233 100644 --- a/cns/configuration/cns_config.json +++ b/cns/configuration/cns_config.json @@ -7,5 +7,12 @@ "HeartBeatIntervalInMins": 30, "DebugMode": false, "SnapshotIntervalInMins": 60 - } + }, + "ManagedSettings": { + "PrivateEndpoint": "", + "InfrastructureNetworkID": "", + "NodeID": "", + "NodeSyncIntervalInSeconds": 30 + }, + "ChannelMode": "Direct" } diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index edaf70a18d..7ca927c9e9 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/common" ) @@ -17,6 +18,8 @@ const ( type CNSConfig struct { TelemetrySettings TelemetrySettings + ManagedSettings ManagedSettings + ChannelMode string } type TelemetrySettings struct { @@ -44,6 +47,13 @@ type TelemetrySettings struct { SnapshotIntervalInMins int } +type ManagedSettings struct { + PrivateEndpoint string + InfrastructureNetworkID string + NodeID string + NodeSyncIntervalInSeconds int +} + // This functions reads cns config file and save it in a structure func ReadConfig() (CNSConfig, error) { var cnsConfig CNSConfig @@ -99,7 +109,18 @@ func setTelemetrySettingDefaults(telemetrySettings *TelemetrySettings) { } } +// set managed setting defaults +func setManagedSettingDefaults(managedSettings *ManagedSettings) { + if managedSettings.NodeSyncIntervalInSeconds == 0 { + managedSettings.NodeSyncIntervalInSeconds = 30 + } +} + // Set Default values of CNS config if not specified func SetCNSConfigDefaults(config *CNSConfig) { setTelemetrySettingDefaults(&config.TelemetrySettings) + setManagedSettingDefaults(&config.ManagedSettings) + if config.ChannelMode == "" { + config.ChannelMode = cns.Direct + } } diff --git a/cns/dockerclient/dockerclient.go b/cns/dockerclient/dockerclient.go index 7fb36e92f9..cd67ed0c17 100644 --- a/cns/dockerclient/dockerclient.go +++ b/cns/dockerclient/dockerclient.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/cns/imdsclient" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/platform" ) @@ -117,7 +118,7 @@ func (dockerClient *DockerClient) CreateNetwork(networkName string, nicInfo *imd res, err := http.Post( dockerClient.connectionURL+createNetworkPath, - "application/json; charset=utf-8", + common.JsonContent, netConfigJSON) if err != nil { @@ -160,7 +161,7 @@ func (dockerClient *DockerClient) DeleteNetwork(networkName string) error { return err } - req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set(common.ContentType, common.JsonContent) client := &http.Client{} res, err := client.Do(req) if err != nil { diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go index c1a0c2758a..4b0d2d7790 100644 --- a/cns/nmagentclient/nmagentclient.go +++ b/cns/nmagentclient/nmagentclient.go @@ -9,6 +9,17 @@ import ( "github.com/Azure/azure-container-networking/common" ) +const ( + WireserverIP = "168.63.129.16" +) + +// NMANetworkContainerResponse - NMAgent response. +type NMANetworkContainerResponse struct { + ResponseCode string `json:"httpStatusCode"` + NetworkContainerID string `json:"networkContainerId"` + Version string `json:"version"` +} + // JoinNetwork joins the given network func JoinNetwork( networkID string, @@ -62,3 +73,16 @@ func UnpublishNetworkContainer( return response, err } + +// GetNetworkContainerVersion :- Retrieves NC version from NMAgent +func GetNetworkContainerVersion( + networkContainerID, + getNetworkContainerVersionURL string) (*http.Response, error) { + logger.Printf("[NMAgentClient] GetNetworkContainerVersion NC: %s", networkContainerID) + + response, err := common.GetHttpClient().Get(getNetworkContainerVersionURL) + + logger.Printf("[NMAgentClient][Response] GetNetworkContainerVersion NC: %s. Response: %+v. Error: %v", + networkContainerID, response, err) + return response, err +} diff --git a/cns/restserver/api.go b/cns/restserver/api.go index 6fb4ecbfe5..9a3ea73587 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -824,7 +824,7 @@ func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWr err = service.Listener.Encode(w, &reserveResp) // If the NC was created successfully, log NC snapshot. - if returnCode == 0 { + if returnCode == Success { logNCSnapshot(req) } @@ -972,8 +972,6 @@ func (service *HTTPRestService) getNetworkContainerStatus(w http.ResponseWriter, containerInfo := service.state.ContainerStatus if containerInfo != nil { containerDetails, ok = containerInfo[req.NetworkContainerid] - } else { - ok = false } var hostVersion string diff --git a/cns/restserver/api_test.go b/cns/restserver/api_test.go index 29eb4013ce..aa4aecdc29 100644 --- a/cns/restserver/api_test.go +++ b/cns/restserver/api_test.go @@ -79,13 +79,13 @@ const ( ) func getInterfaceInfo(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/xml") + w.Header().Set(acncommon.ContentType, "application/xml") output, _ := xml.Marshal(hostQueryResponse) w.Write(output) } func nmagentHandler(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.Header().Set(acncommon.ContentType, acncommon.JsonContent) w.WriteHeader(http.StatusOK) if strings.Contains(r.RequestURI, "networkContainers") { @@ -248,7 +248,7 @@ func TestGetNetworkContainerByOrchestratorContext(t *testing.T) { setEnv(t) setOrchestratorType(t, cns.Kubernetes) - err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", "AzureContainerInstance") + err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", cns.AzureContainerInstance) if err != nil { t.Errorf("creatOrUpdateNetworkContainerWithName failed Err:%+v", err) t.Fatal(err) @@ -283,7 +283,7 @@ func TestGetNetworkContainerStatus(t *testing.T) { setEnv(t) setOrchestratorType(t, cns.Kubernetes) - err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", "WebApps") + err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", cns.AzureContainerInstance) if err != nil { t.Errorf("creatOrUpdateWebAppContainerWithName failed Err:%+v", err) t.Fatal(err) @@ -597,7 +597,7 @@ func getNetworkContainerStatus(t *testing.T, name string) error { var resp cns.GetNetworkContainerStatusResponse getReq := &cns.GetNetworkContainerStatusRequest{ - NetworkContainerid: "ethWebApp", + NetworkContainerid: name, } json.NewEncoder(&body).Encode(getReq) @@ -624,7 +624,7 @@ func getInterfaceForContainer(t *testing.T, name string) error { var resp cns.GetInterfaceForContainerResponse getReq := &cns.GetInterfaceForContainerRequest{ - NetworkContainerID: "ethWebApp", + NetworkContainerID: name, } json.NewEncoder(&body).Encode(getReq) diff --git a/cns/restserver/const.go b/cns/restserver/const.go index 518a83cafd..fd1b354cb6 100644 --- a/cns/restserver/const.go +++ b/cns/restserver/const.go @@ -5,33 +5,34 @@ package restserver // Container Network Service remote API Contract. const ( - Success = 0 - UnsupportedNetworkType = 1 - InvalidParameter = 2 - UnsupportedEnvironment = 3 - UnreachableHost = 4 - ReservationNotFound = 5 - MalformedSubnet = 8 - UnreachableDockerDaemon = 9 - UnspecifiedNetworkName = 10 - NotFound = 14 - AddressUnavailable = 15 - NetworkContainerNotSpecified = 16 - CallToHostFailed = 17 - UnknownContainerID = 18 - UnsupportedOrchestratorType = 19 - DockerContainerNotSpecified = 20 - UnsupportedVerb = 21 - UnsupportedNetworkContainerType = 22 - InvalidRequest = 23 - NetworkJoinFailed = 24 - NetworkContainerPublishFailed = 25 - NetworkContainerUnpublishFailed = 26 - InvalidPrimaryIPConfig = 27 - PrimaryCANotSame = 28 - InconsistentIPConfigState = 29 - InvalidSecondaryIPConfig = 30 - UnexpectedError = 99 + Success = 0 + UnsupportedNetworkType = 1 + InvalidParameter = 2 + UnsupportedEnvironment = 3 + UnreachableHost = 4 + ReservationNotFound = 5 + MalformedSubnet = 8 + UnreachableDockerDaemon = 9 + UnspecifiedNetworkName = 10 + NotFound = 14 + AddressUnavailable = 15 + NetworkContainerNotSpecified = 16 + CallToHostFailed = 17 + UnknownContainerID = 18 + UnsupportedOrchestratorType = 19 + DockerContainerNotSpecified = 20 + UnsupportedVerb = 21 + UnsupportedNetworkContainerType = 22 + InvalidRequest = 23 + NetworkJoinFailed = 24 + NetworkContainerPublishFailed = 25 + NetworkContainerUnpublishFailed = 26 + InvalidPrimaryIPConfig = 27 + PrimaryCANotSame = 28 + InconsistentIPConfigState = 29 + InvalidSecondaryIPConfig = 30 + NetworkContainerPendingStatePropagation = 31 + UnexpectedError = 99 ) const ( @@ -42,4 +43,5 @@ const ( detach = "Detach" // Rest service state identifier for named lock stateJoinedNetworks = "JoinedNetworks" + dncApiVersion = "?api-version=2018-03-01" ) diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go index 293ad3a4ff..68b6e1c109 100644 --- a/cns/restserver/internalapi.go +++ b/cns/restserver/internalapi.go @@ -4,10 +4,17 @@ package restserver import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" "reflect" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/nmagentclient" + "github.com/Azure/azure-container-networking/common" ) // This file contains the internal functions called by either HTTP APIs (api.go) or @@ -22,6 +29,126 @@ func (service *HTTPRestService) GetPartitionKey() (dncPartitionKey string) { return } +// SetNodeOrchestrator :- Set node orchestrator after registering with mDNC +func (service *HTTPRestService) SetNodeOrchestrator(r *cns.SetOrchestratorTypeRequest) { + body, _ := json.Marshal(r) + req, _ := http.NewRequest(http.MethodPost, "", bytes.NewBuffer(body)) + req.Header.Set(common.ContentType, common.JsonContent) + service.setOrchestratorType(httptest.NewRecorder(), req) +} + +// SyncNodeStatus :- Retrieve the latest node state from DNC & returns the first occurence of returnCode and error with respect to contextFromCNI +func (service *HTTPRestService) SyncNodeStatus(dncEP, infraVnet, nodeID string, contextFromCNI json.RawMessage) (returnCode int, errStr string) { + logger.Printf("[Azure CNS] SyncNodeStatus") + var ( + response *http.Response + err error + nodeInfoResponse cns.NodeInfoResponse + req *http.Request + body []byte + httpc = common.GetHttpClient() + ) + + // try to retrieve NodeInfoResponse from mDNC + response, err = httpc.Get(fmt.Sprintf(common.SyncNodeNetworkContainersURLFmt, dncEP, infraVnet, nodeID, dncApiVersion)) + if err == nil { + if response.StatusCode == http.StatusOK { + err = json.NewDecoder(response.Body).Decode(&nodeInfoResponse) + } else { + err = fmt.Errorf("%d", response.StatusCode) + } + + response.Body.Close() + } + + if err != nil { + returnCode = UnexpectedError + errStr = fmt.Sprintf("[Azure-CNS] Failed to sync node with error: %+v", err) + logger.Errorf(errStr) + return + } + + var ( + ncsToBeAdded = make(map[string]cns.CreateNetworkContainerRequest) + ncsToBeDeleted = make(map[string]bool) + ) + + // determine new NCs and NCs to be deleted + service.RLock() + for ncid := range service.state.ContainerStatus { + ncsToBeDeleted[ncid] = true + } + + for _, nc := range nodeInfoResponse.NetworkContainers { + ncid := cns.SwiftPrefix + nc.NetworkContainerid + delete(ncsToBeDeleted, ncid) + if savedNc, exists := service.state.ContainerStatus[ncid]; !exists || savedNc.CreateNetworkContainerRequest.Version < nc.Version { + ncsToBeAdded[ncid] = nc + } + } + service.RUnlock() + + // check if the version is valid and save it to service state + for ncid, nc := range ncsToBeAdded { + var ( + versionURL = fmt.Sprintf(nodeInfoResponse.GetNCVersionURLFmt, + nmagentclient.WireserverIP, + nc.PrimaryInterfaceIdentifier, + nc.NetworkContainerid, + nc.AuthorizationToken) + w = httptest.NewRecorder() + ) + + ncVersionURLs.Store(nc.NetworkContainerid, versionURL) + waitingForUpdate, tmpReturnCode, tmpErrStr := isNCWaitingForUpdate(nc.Version, nc.NetworkContainerid) + if tmpReturnCode != Success && bytes.Compare(nc.OrchestratorContext, contextFromCNI) == 0 { + returnCode = tmpReturnCode + errStr = tmpErrStr + } + + if tmpReturnCode == UnexpectedError { + continue + } + + body, _ = json.Marshal(nc) + req, _ = http.NewRequest(http.MethodPost, "", bytes.NewBuffer(body)) + req.Header.Set(common.ContentType, common.JsonContent) + service.createOrUpdateNetworkContainer(w, req) + if w.Result().StatusCode == http.StatusOK { + var resp cns.CreateNetworkContainerResponse + if err = json.Unmarshal(w.Body.Bytes(), &resp); err == nil && resp.Response.ReturnCode == Success { + service.Lock() + ncstatus, _ := service.state.ContainerStatus[ncid] + ncstatus.WaitingForUpdate = waitingForUpdate + service.state.ContainerStatus[ncid] = ncstatus + service.Unlock() + } + } + } + + service.Lock() + service.saveState() + service.Unlock() + + // delete dangling NCs + for nc := range ncsToBeDeleted { + var body bytes.Buffer + json.NewEncoder(&body).Encode(&cns.DeleteNetworkContainerRequest{NetworkContainerid: nc}) + + req, err = http.NewRequest(http.MethodPost, "", &body) + if err == nil { + req.Header.Set(common.JsonContent, common.JsonContent) + service.deleteNetworkContainer(httptest.NewRecorder(), req) + } else { + logger.Errorf("[Azure-CNS] Failed to delete NC request to sync state: %s", err.Error()) + } + + ncVersionURLs.Delete(nc) + } + + return +} + // This API will be called by CNS RequestController on CRD update. func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.CreateNetworkContainerRequest) int { if req.NetworkContainerid == "" { diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 0807cb4b1e..b1070ed188 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -29,6 +29,8 @@ import ( var ( // Named Lock for accessing different states in httpRestServiceState namedLock = acn.InitNamedLock() + // map of NC to their respective NMA getVersion URLs + ncVersionURLs sync.Map ) // HTTPRestService represents http listener for CNS - Container Networking Service. @@ -68,6 +70,7 @@ type containerstatus struct { VMVersion string HostVersion string CreateNetworkContainerRequest cns.CreateNetworkContainerRequest + WaitingForUpdate bool // True when NC is waiting for NMA to sync versions/rules } // httpRestServiceState contains the state we would like to persist. @@ -94,11 +97,13 @@ type networkInfo struct { type HTTPService interface { common.ServiceAPI SendNCSnapShotPeriodically(int, chan bool) + SetNodeOrchestrator(*cns.SetOrchestratorTypeRequest) + SyncNodeStatus(string, string, string, json.RawMessage) (int, string) } // NewHTTPRestService creates a new HTTP Service object. func NewHTTPRestService(config *common.ServiceConfig) (HTTPService, error) { - service, err := cns.NewService(config.Name, config.Version, config.Store) + service, err := cns.NewService(config.Name, config.Version, config.ChannelMode, config.Store) if err != nil { return nil, err } diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 16beba3c58..0e5a97f84d 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -6,7 +6,9 @@ package restserver import ( "encoding/json" "fmt" + "io/ioutil" "net/http" + "strconv" "time" "github.com/Azure/azure-container-networking/aitelemetry" @@ -103,6 +105,10 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw service.Lock() defer service.Unlock() + if service.state.ContainerStatus == nil { + service.state.ContainerStatus = make(map[string]containerstatus) + } + existingNCStatus, ok := service.state.ContainerStatus[req.NetworkContainerid] var hostVersion string var existingSecondaryIPConfigs map[string]cns.SecondaryIPConfig //uuid is key @@ -111,9 +117,12 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw existingSecondaryIPConfigs = existingNCStatus.CreateNetworkContainerRequest.SecondaryIPConfigs } - if service.state.ContainerStatus == nil { - service.state.ContainerStatus = make(map[string]containerstatus) - } + service.state.ContainerStatus[req.NetworkContainerid] = + containerstatus{ + ID: req.NetworkContainerid, + VMVersion: req.Version, + CreateNetworkContainerRequest: req, + HostVersion: hostVersion} switch req.NetworkContainerType { case cns.AzureContainerInstance: @@ -288,8 +297,11 @@ func (service *HTTPRestService) removeToBeDeletedIpsStateUntransacted(ipId strin } func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkContainerRequest) cns.GetNetworkContainerResponse { - var containerID string - var getNetworkContainerResponse cns.GetNetworkContainerResponse + var ( + containerID string + getNetworkContainerResponse cns.GetNetworkContainerResponse + exists bool + ) service.RLock() defer service.RUnlock() @@ -313,7 +325,33 @@ func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkCo } logger.Printf("pod info %+v", podInfo) - containerID = service.state.ContainerIDByOrchestratorContext[podInfo.PodName+podInfo.PodNamespace] + + context := podInfo.PodName + podInfo.PodNamespace + containerID, exists = service.state.ContainerIDByOrchestratorContext[context] + if service.ChannelMode == cns.Managed { + if exists { + _, getNetworkContainerResponse.Response.ReturnCode, getNetworkContainerResponse.Response.Message = isNCWaitingForUpdate(service.state.ContainerStatus[containerID].CreateNetworkContainerRequest.Version, containerID) + if getNetworkContainerResponse.Response.ReturnCode == Success { + return getNetworkContainerResponse + } + } else { + var ( + dncEP = service.GetOption(acn.OptPrivateEndpoint).(string) + infraVnet = service.GetOption(acn.OptInfrastructureNetworkID).(string) + nodeID = service.GetOption(acn.OptNodeID).(string) + ) + + service.RUnlock() + getNetworkContainerResponse.Response.ReturnCode, getNetworkContainerResponse.Response.Message = service.SyncNodeStatus(dncEP, infraVnet, nodeID, req.OrchestratorContext) + service.RLock() + if getNetworkContainerResponse.Response.ReturnCode == NotFound { + return getNetworkContainerResponse + } + + containerID = service.state.ContainerIDByOrchestratorContext[context] + } + } + logger.Printf("containerid %v", containerID) break @@ -412,8 +450,33 @@ func (service *HTTPRestService) attachOrDetachHelper(req cns.ConfigureContainerN } existing, ok := service.getNetworkContainerDetails(cns.SwiftPrefix + req.NetworkContainerid) + if service.ChannelMode == cns.Managed && operation == attach { + if ok { + if existing.WaitingForUpdate { + _, returnCode, message := isNCWaitingForUpdate(existing.CreateNetworkContainerRequest.Version, req.NetworkContainerid) + if returnCode != Success { + return cns.Response{ + ReturnCode: returnCode, + Message: message} + } + } + } else { + var ( + dncEP = service.GetOption(acn.OptPrivateEndpoint).(string) + infraVnet = service.GetOption(acn.OptInfrastructureNetworkID).(string) + nodeID = service.GetOption(acn.OptNodeID).(string) + ) + + returnCode, msg := service.SyncNodeStatus(dncEP, infraVnet, nodeID, json.RawMessage{}) + if returnCode != Success { + return cns.Response{ + ReturnCode: returnCode, + Message: msg} + } - if !ok { + existing, _ = service.getNetworkContainerDetails(cns.SwiftPrefix + req.NetworkContainerid) + } + } else if !ok { return cns.Response{ ReturnCode: NotFound, Message: fmt.Sprintf("[Azure CNS] Error. Network Container %s does not exist.", req.NetworkContainerid)} @@ -555,6 +618,45 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ncSnapshotIntervalInM } } +// isNCWaitingForUpdate :- Determine whether NC version on NMA matches programmed version +func isNCWaitingForUpdate(ncVersion, ncid string) (waitingForUpdate bool, returnCode int, message string) { + getNCVersionURL, ok := ncVersionURLs.Load(ncid) + if !ok { + returnCode = NotFound + message = fmt.Sprintf("[Azure-CNS] Network container %s not found", ncid) + return + } + + response, err := nmagentclient.GetNetworkContainerVersion(ncid, getNCVersionURL.(string)) + if err == nil { + if response.StatusCode == http.StatusOK { + var versionResponse nmagentclient.NMANetworkContainerResponse + rBytes, _ := ioutil.ReadAll(response.Body) + json.Unmarshal(rBytes, &versionResponse) + if versionResponse.ResponseCode == "200" { + programmedVersion, _ := strconv.Atoi(ncVersion) + nmaVersion, _ := strconv.Atoi(versionResponse.Version) + if programmedVersion > nmaVersion { + waitingForUpdate = true + returnCode = NetworkContainerPendingStatePropagation + message = fmt.Sprintf("[Azure-CNS] Network container %s v%d had not propagated to respective NMA w/ v%d", ncid, programmedVersion, nmaVersion) + } + } else { + returnCode = UnexpectedError + message = fmt.Sprintf("[Azure-CNS] Failed to get NC version from response %s for NC %s", rBytes, ncid) + } + } else { + returnCode = UnexpectedError + message = fmt.Sprintf("[Azure-CNS] Failed to get NC version with http status %d", response.StatusCode) + } + } else { + returnCode = UnexpectedError + message = fmt.Sprintf("[Azure-CNS] Failed to get NC version from NMA with error: %+v", err) + } + + return +} + // ReturnCodeToString - Converts an error code to appropriate string. func ReturnCodeToString(returnCode int) (s string) { switch returnCode { @@ -590,6 +692,8 @@ func ReturnCodeToString(returnCode int) (s string) { s = "UnexpectedError" case DockerContainerNotSpecified: s = "DockerContainerNotSpecified" + case NetworkContainerPendingStatePropagation: + s = "NetworkContainerPendingStatePropagation" default: s = "UnknownError" } diff --git a/cns/service.go b/cns/service.go index c6d8b74c36..15420b1e5b 100644 --- a/cns/service.go +++ b/cns/service.go @@ -27,8 +27,8 @@ type Service struct { } // NewService creates a new Service object. -func NewService(name, version string, store store.KeyValueStore) (*Service, error) { - service, err := common.NewService(name, version, store) +func NewService(name, version, channelMode string, store store.KeyValueStore) (*Service, error) { + service, err := common.NewService(name, version, channelMode, store) if err != nil { return nil, err diff --git a/cns/service/main.go b/cns/service/main.go index f5df2f8ed9..4caaec20ae 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -4,15 +4,22 @@ package main import ( + "bytes" + "encoding/json" "fmt" + "net/http" "os" "os/signal" + "runtime" + "strconv" "strings" "syscall" + "time" "github.com/Azure/azure-container-networking/aitelemetry" "github.com/Azure/azure-container-networking/cnm/ipam" "github.com/Azure/azure-container-networking/cnm/network" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/hnsclient" @@ -30,6 +37,7 @@ const ( pluginName = "azure-vnet" defaultCNINetworkConfigFileName = "10-azure.conflist" configFileName = "config.json" + dncApiVersion = "?api-version=2018-03-01" ) // Version is populated by make during build. @@ -179,6 +187,34 @@ var args = acn.ArgumentList{ Type: "string", DefaultValue: platform.CNMRuntimePath, }, + { + Name: acn.OptPrivateEndpoint, + Shorthand: acn.OptPrivateEndpointAlias, + Description: "Set private endpoint", + Type: "string", + DefaultValue: "", + }, + { + Name: acn.OptInfrastructureNetworkID, + Shorthand: acn.OptInfrastructureNetworkIDAlias, + Description: "Set infrastructure network ID", + Type: "string", + DefaultValue: "", + }, + { + Name: acn.OptNodeID, + Shorthand: acn.OptNodeIDAlias, + Description: "Set node name/ID", + Type: "string", + DefaultValue: "", + }, + { + Name: acn.OptManaged, + Shorthand: acn.OptManagedAlias, + Description: "Set to true to enable managed mode. This is deprecated in favor of cns_config.json", + Type: "bool", + DefaultValue: false, + }, } // Prints description and version information. @@ -187,6 +223,45 @@ func printVersion() { fmt.Printf("Version %v\n", version) } +// Try to register node with DNC when CNS is started in managed DNC mode +func registerNode(httpRestService restserver.HTTPService, dncEP, infraVnet, nodeID string) { + logger.Printf("[Azure CNS] Registering node %s with Infrastructure Network: %s PrivateEndpoint: %s", nodeID, infraVnet, dncEP) + + var ( + numCPU = runtime.NumCPU() + url = fmt.Sprintf(acn.RegisterNodeURLFmt, dncEP, infraVnet, nodeID, numCPU, dncApiVersion) + response *http.Response + err = fmt.Errorf("") + body bytes.Buffer + httpc = acn.GetHttpClient() + ) + + for sleep := true; err != nil; sleep = true { + response, err = httpc.Post(url, "application/json", &body) + if err == nil { + if response.StatusCode == http.StatusCreated { + var req cns.SetOrchestratorTypeRequest + json.NewDecoder(response.Body).Decode(&req) + httpRestService.SetNodeOrchestrator(&req) + sleep = false + } else { + err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode)) + logger.Errorf(err.Error()) + } + + response.Body.Close() + } else { + logger.Errorf("[Azure CNS] Failed to register node with err: %+v", err) + } + + if sleep { + time.Sleep(acn.FiveSeconds) + } + } + + logger.Printf("[Azure CNS] Node Registered") +} + // Main is the entry point for CNS. func main() { // Initialize and parse command line arguments. @@ -200,8 +275,8 @@ func main() { logLevel := acn.GetArg(acn.OptLogLevel).(int) logTarget := acn.GetArg(acn.OptLogTarget).(int) logDirectory := acn.GetArg(acn.OptLogLocation).(string) - ipamQueryUrl, _ := acn.GetArg(acn.OptIpamQueryUrl).(string) - ipamQueryInterval, _ := acn.GetArg(acn.OptIpamQueryInterval).(int) + ipamQueryUrl := acn.GetArg(acn.OptIpamQueryUrl).(string) + ipamQueryInterval := acn.GetArg(acn.OptIpamQueryInterval).(int) startCNM := acn.GetArg(acn.OptStartAzureCNM).(bool) vers := acn.GetArg(acn.OptVersion).(bool) createDefaultExtNetworkType := acn.GetArg(acn.OptCreateDefaultExtNetworkType).(string) @@ -209,6 +284,9 @@ func main() { httpConnectionTimeout := acn.GetArg(acn.OptHttpConnectionTimeout).(int) httpResponseHeaderTimeout := acn.GetArg(acn.OptHttpResponseHeaderTimeout).(int) storeFileLocation := acn.GetArg(acn.OptStoreFileLocation).(string) + privateEndpoint := acn.GetArg(acn.OptPrivateEndpoint).(string) + infravnet := acn.GetArg(acn.OptInfrastructureNetworkID).(string) + nodeID := acn.GetArg(acn.OptNodeID).(string) if vers { printVersion() @@ -241,8 +319,15 @@ func main() { configuration.SetCNSConfigDefaults(&cnsconfig) logger.Printf("[Azure CNS] Read config :%+v", cnsconfig) - disableTelemetry := cnsconfig.TelemetrySettings.DisableAll + if cnsconfig.ChannelMode == cns.Managed { + privateEndpoint = cnsconfig.ManagedSettings.PrivateEndpoint + infravnet = cnsconfig.ManagedSettings.InfrastructureNetworkID + nodeID = cnsconfig.ManagedSettings.NodeID + } else if acn.GetArg(acn.OptManaged).(bool) { + config.ChannelMode = cns.Managed + } + disableTelemetry := cnsconfig.TelemetrySettings.DisableAll if !disableTelemetry { ts := cnsconfig.TelemetrySettings aiConfig := aitelemetry.AIConfig{ @@ -316,6 +401,30 @@ func main() { go httpRestService.SendNCSnapShotPeriodically(cnsconfig.TelemetrySettings.SnapshotIntervalInMins, stopSnapshots) } + // If CNS is running on managed DNC mode + if config.ChannelMode == cns.Managed { + if privateEndpoint == "" || infravnet == "" || nodeID == "" { + logger.Errorf("[Azure CNS] Missing required values to run in managed mode: PrivateEndpoint: %s InfrastructureNetworkID: %s NodeID: %s", + privateEndpoint, + infravnet, + nodeID) + return + } + + httpRestService.SetOption(acn.OptPrivateEndpoint, privateEndpoint) + httpRestService.SetOption(acn.OptInfrastructureNetworkID, infravnet) + httpRestService.SetOption(acn.OptNodeID, nodeID) + + registerNode(httpRestService, privateEndpoint, infravnet, nodeID) + go func(ep, vnet, node string) { + // Periodically poll DNC for node updates + for { + <-time.NewTicker(time.Duration(cnsconfig.ManagedSettings.NodeSyncIntervalInSeconds) * time.Second).C + httpRestService.SyncNodeStatus(ep, vnet, node, json.RawMessage{}) + } + }(privateEndpoint, infravnet, nodeID) + } + var netPlugin network.NetPlugin var ipamPlugin ipam.IpamPlugin diff --git a/common/config.go b/common/config.go index aefa0c1b6f..1239a04e96 100644 --- a/common/config.go +++ b/common/config.go @@ -98,4 +98,20 @@ const ( // Store file location OptStoreFileLocation = "store-file-path" OptStoreFileLocationAlias = "storefilepath" + + // Private Endpoint + OptPrivateEndpoint = "private-endpoint" + OptPrivateEndpointAlias = "pe" + + // Infrastructure Network + OptInfrastructureNetworkID = "infra-vnet" + OptInfrastructureNetworkIDAlias = "iv" + + // Node ID/Name + OptNodeID = "node-id" + OptNodeIDAlias = "n" + + // Managed mode + OptManaged = "managed" + OptManagedAlias = "m" ) diff --git a/common/utils.go b/common/utils.go index fd0832af2b..a1e77df6d4 100644 --- a/common/utils.go +++ b/common/utils.go @@ -22,10 +22,15 @@ import ( ) const ( - metadataURL = "http://169.254.169.254/metadata/instance?api-version=2017-08-01&format=json" - azCloudUrl = "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2018-10-01&format=text" - httpConnectionTimeout = 7 - headerTimeout = 7 + metadataURL = "http://169.254.169.254/metadata/instance?api-version=2017-08-01&format=json" + azCloudUrl = "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2018-10-01&format=text" + httpConnectionTimeout = 7 + headerTimeout = 7 + RegisterNodeURLFmt = "%s/%s/node/%s/cores/%d%s" + SyncNodeNetworkContainersURLFmt = "%s/%s/node/%s%s" + FiveSeconds = 5 * time.Second + JsonContent = "application/json; charset=UTF-8" + ContentType = "Content-Type" ) // XmlDocument - Azure host agent XML document format.