From 7df26c15403bb826321330f3320bbad0ee38be9d Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 2 Apr 2020 22:03:58 +0000 Subject: [PATCH 01/17] initial changes for CNS->DNC support --- cns/service/main.go | 9 +++++++++ common/config.go | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/cns/service/main.go b/cns/service/main.go index f5df2f8ed9..9a33371026 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -209,6 +209,8 @@ func main() { httpConnectionTimeout := acn.GetArg(acn.OptHttpConnectionTimeout).(int) httpResponseHeaderTimeout := acn.GetArg(acn.OptHttpResponseHeaderTimeout).(int) storeFileLocation := acn.GetArg(acn.OptStoreFileLocation).(string) + privateEndpoint := acn.GetArg(acn.OptPrivateEndpoint).(string) + infravnet := acn.GetArg(acn.OptInfrastructureNetwork).(string) if vers { printVersion() @@ -316,6 +318,13 @@ func main() { go httpRestService.SendNCSnapShotPeriodically(cnsconfig.TelemetrySettings.SnapshotIntervalInMins, stopSnapshots) } + // If running on managed DNC mode + if privateEndpoint != "" && infravnet != "" { + // parse cnsurl for primary identifier + + go httpRestService.RetrieveNodeStatus() + } + var netPlugin network.NetPlugin var ipamPlugin ipam.IpamPlugin diff --git a/common/config.go b/common/config.go index aefa0c1b6f..677895083a 100644 --- a/common/config.go +++ b/common/config.go @@ -98,4 +98,12 @@ const ( // Store file location OptStoreFileLocation = "store-file-path" OptStoreFileLocationAlias = "storefilepath" + + // Private Endpoint + OptPrivateEndpoint = "private-endpoint" + OptPrivateEndpointAlias = "pe" + + // Infrastructure Network + OptInfrastructureNetwork = "infra-vnet" + OptInfrastructureNetworkAlias = "iv" ) From 4abaabc7f5812441c36c110498454b3d8b96ff62 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Tue, 7 Apr 2020 18:28:49 +0000 Subject: [PATCH 02/17] Adding changes for CNS to be compatible with managed DNC (reverse communication channel) --- cns/NetworkContainerContract.go | 11 +++++ cns/restserver/restserver.go | 29 +++++++++++- cns/service/main.go | 79 ++++++++++++++++++++++++++++++--- common/config.go | 4 ++ 4 files changed, 115 insertions(+), 8 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 6c0aaa7d75..17e0add9e3 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -322,3 +322,14 @@ func (networkContainerRequestPolicy *NetworkContainerRequestPolicies) Validate() } return nil } + +// NodeInfoResponse - Struct to hold the node info response. +type NodeInfoResponse struct { + NetworkContainers []CreateNetworkContainerRequest +} + +// JsonErr - Struct to hold err info +type JsonErr struct { + Text string `json:"Text"` + HTTPStatus int `json:"HttpStatusCode"` +} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 8b25e9a9dc..53e3c6bcaa 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -4,6 +4,18 @@ package restserver import ( +<<<<<<< HEAD +======= + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/http/httptest" + "runtime" + "strings" +>>>>>>> Adding changes for CNS to be compatible with managed DNC (reverse communication channel) "sync" "time" @@ -29,6 +41,20 @@ var ( namedLock = acn.InitNamedLock() ) +<<<<<<< HEAD +======= +const ( + // Key against which CNS state is persisted. + storeKey = "ContainerNetworkService" + swiftAPIVersion = "1" + attach = "Attach" + detach = "Detach" + // Rest service state identifier for named lock + stateJoinedNetworks = "JoinedNetworks" + dncApiVersion = "?api-version=2018-03-01" +) + +>>>>>>> Adding changes for CNS to be compatible with managed DNC (reverse communication channel) // HTTPRestService represents http listener for CNS - Container Networking Service. type HTTPRestService struct { *cns.Service @@ -82,6 +108,7 @@ type networkInfo struct { type HTTPService interface { common.ServiceAPI SendNCSnapShotPeriodically(int, chan bool) + SyncNodeStatus(string, string, string) } // NewHTTPRestService creates a new HTTP Service object. @@ -216,4 +243,4 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { func (service *HTTPRestService) Stop() { service.Uninitialize() logger.Printf("[Azure CNS] Service stopped.") -} +} \ No newline at end of file diff --git a/cns/service/main.go b/cns/service/main.go index 9a33371026..5340d65000 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -4,11 +4,16 @@ package main import ( + "bytes" "fmt" + "net/http" "os" "os/signal" + "runtime" + "strconv" "strings" "syscall" + "time" "github.com/Azure/azure-container-networking/aitelemetry" "github.com/Azure/azure-container-networking/cnm/ipam" @@ -30,6 +35,7 @@ const ( pluginName = "azure-vnet" defaultCNINetworkConfigFileName = "10-azure.conflist" configFileName = "config.json" + dncApiVersion = "?api-version=2018-03-01" ) // Version is populated by make during build. @@ -179,6 +185,27 @@ var args = acn.ArgumentList{ Type: "string", DefaultValue: platform.CNMRuntimePath, }, + { + Name: acn.OptPrivateEndpoint, + Shorthand: acn.OptPrivateEndpointAlias, + Description: "Set private endpoint", + Type: "string", + DefaultValue: "", + }, + { + Name: acn.OptInfrastructureNetwork, + Shorthand: acn.OptInfrastructureNetworkAlias, + Description: "Set infrastructure network ID", + Type: "string", + DefaultValue: "", + }, + { + Name: acn.OptNodeID, + Shorthand: acn.OptNodeIDAlias, + Description: "Set node name/ID", + Type: "string", + DefaultValue: "", + }, } // Prints description and version information. @@ -187,6 +214,34 @@ func printVersion() { fmt.Printf("Version %v\n", version) } +// Try to register node with DNC when CNS is started in managed DNC mode +func registerNode(dncEP, infraVnet, nodeID string) { + logger.Printf("[Azure CNS] SyncNodeStatus") + var ( + numCPU = runtime.NumCPU() + response *http.Response + err = fmt.Errorf("") + body bytes.Buffer + httpc = &http.Client{Timeout: time.Second * 30} + ) + + for err != nil { + response, err = httpc.Post(fmt.Sprintf("%s/%s/node/%s/nclimit/%d%s", dncEP, infraVnet, nodeID, numCPU, dncApiVersion), "application/json", &body) + if err == nil { + break + } + time.Sleep(time.Second * 5) + } + + response.Body.Close() + + if response.StatusCode != http.StatusOK { + panic("[Azure CNS] Failed to register node with managed DNC with http status code: " + strconv.Itoa(response.StatusCode)) + } + + logger.Printf("[Azure CNS] Node Registered") +} + // Main is the entry point for CNS. func main() { // Initialize and parse command line arguments. @@ -200,8 +255,8 @@ func main() { logLevel := acn.GetArg(acn.OptLogLevel).(int) logTarget := acn.GetArg(acn.OptLogTarget).(int) logDirectory := acn.GetArg(acn.OptLogLocation).(string) - ipamQueryUrl, _ := acn.GetArg(acn.OptIpamQueryUrl).(string) - ipamQueryInterval, _ := acn.GetArg(acn.OptIpamQueryInterval).(int) + ipamQueryUrl := acn.GetArg(acn.OptIpamQueryUrl).(string) + ipamQueryInterval := acn.GetArg(acn.OptIpamQueryInterval).(int) startCNM := acn.GetArg(acn.OptStartAzureCNM).(bool) vers := acn.GetArg(acn.OptVersion).(bool) createDefaultExtNetworkType := acn.GetArg(acn.OptCreateDefaultExtNetworkType).(string) @@ -211,6 +266,7 @@ func main() { storeFileLocation := acn.GetArg(acn.OptStoreFileLocation).(string) privateEndpoint := acn.GetArg(acn.OptPrivateEndpoint).(string) infravnet := acn.GetArg(acn.OptInfrastructureNetwork).(string) + nodeID := acn.GetArg(acn.OptNodeID).(string) if vers { printVersion() @@ -318,11 +374,20 @@ func main() { go httpRestService.SendNCSnapShotPeriodically(cnsconfig.TelemetrySettings.SnapshotIntervalInMins, stopSnapshots) } - // If running on managed DNC mode - if privateEndpoint != "" && infravnet != "" { - // parse cnsurl for primary identifier - - go httpRestService.RetrieveNodeStatus() + // If CNS is running on managed DNC mode + if privateEndpoint != "" && infravnet != "" && nodeID != "" { + httpRestService.SetOption(acn.OptPrivateEndpoint, privateEndpoint) + httpRestService.SetOption(acn.OptInfrastructureNetwork, infravnet) + httpRestService.SetOption(acn.OptNodeID, nodeID) + + registerNode(privateEndpoint, infravnet, nodeID) + go func(ep, vnet, node string) { + // Periodically poll (30s) DNC for node updates + for { + <-time.NewTicker(time.Second * 30).C + httpRestService.SyncNodeStatus(ep, vnet, node) + } + }(privateEndpoint, infravnet, nodeID) } var netPlugin network.NetPlugin diff --git a/common/config.go b/common/config.go index 677895083a..279ebeb06e 100644 --- a/common/config.go +++ b/common/config.go @@ -106,4 +106,8 @@ const ( // Infrastructure Network OptInfrastructureNetwork = "infra-vnet" OptInfrastructureNetworkAlias = "iv" + + // Node ID/Name + OptNodeID = "node-id" + OptNodeIDAlias = "n" ) From c0d8137eb36da48461c54dba4580cdadf9877fb6 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 18 Jun 2020 12:02:18 -0700 Subject: [PATCH 03/17] adding NC version validation with respective NMA --- cns/NetworkContainerContract.go | 4 +++- cns/nmagentclient/nmagentclient.go | 20 ++++++++++++++++++++ cns/restserver/restserver.go | 3 ++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 17e0add9e3..ed1f7e9426 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -65,6 +65,7 @@ type CreateNetworkContainerRequest struct { NetworkContainerType string NetworkContainerid string // Mandatory input. PrimaryInterfaceIdentifier string // Primary CA. + InterfaceIP string AuthorizationToken string LocalIPConfiguration IPConfiguration OrchestratorContext json.RawMessage @@ -325,7 +326,8 @@ func (networkContainerRequestPolicy *NetworkContainerRequestPolicies) Validate() // NodeInfoResponse - Struct to hold the node info response. type NodeInfoResponse struct { - NetworkContainers []CreateNetworkContainerRequest + NetworkContainers []CreateNetworkContainerRequest + GetNCVersionURLFmt string } // JsonErr - Struct to hold err info diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go index c1a0c2758a..f1b1ba3222 100644 --- a/cns/nmagentclient/nmagentclient.go +++ b/cns/nmagentclient/nmagentclient.go @@ -9,6 +9,13 @@ import ( "github.com/Azure/azure-container-networking/common" ) +// NetworkContainerResponse - NMAgent response. +type NetworkContainerResponse struct { + ResponseCode string `json:"httpStatusCode"` + NetworkContainerID string `json:"networkContainerId"` + Version string `json:"version"` +} + // JoinNetwork joins the given network func JoinNetwork( networkID string, @@ -62,3 +69,16 @@ func UnpublishNetworkContainer( return response, err } + +// GetNetworkContainerVersion :- Retrieves NC version from NMAgent +func GetNetworkContainerVersion( + networkContainerID, + getNetworkContainerVersionURL string) (*http.Response, error) { + logger.Printf("[NMAgentClient] GetNetworkContainerVersion NC: %s", networkContainerID) + + response, err := common.GetHttpClient().Get(getNetworkContainerVersionURL) + + logger.Printf("[NMAgentClient][Response] GetNetworkContainerVersion NC: %s. Response: %+v. Error: %v", + networkContainerID, response, err) + return response, err +} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 53e3c6bcaa..c381a5a165 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -14,6 +14,7 @@ import ( "net/http" "net/http/httptest" "runtime" + "strconv" "strings" >>>>>>> Adding changes for CNS to be compatible with managed DNC (reverse communication channel) "sync" @@ -243,4 +244,4 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { func (service *HTTPRestService) Stop() { service.Uninitialize() logger.Printf("[Azure CNS] Service stopped.") -} \ No newline at end of file +} From 083a0eeebce0ba8ad0ac034e70b6ce7313e36b4e Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 18 Jun 2020 14:41:19 -0700 Subject: [PATCH 04/17] return errors for respective NC based on orchestrator context from CNI --- cns/NetworkContainerContract.go | 1 - cns/restserver/restserver.go | 17 +---------------- cns/service/main.go | 3 ++- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index ed1f7e9426..726fdbab29 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -65,7 +65,6 @@ type CreateNetworkContainerRequest struct { NetworkContainerType string NetworkContainerid string // Mandatory input. PrimaryInterfaceIdentifier string // Primary CA. - InterfaceIP string AuthorizationToken string LocalIPConfiguration IPConfiguration OrchestratorContext json.RawMessage diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index c381a5a165..0fe7ac6f89 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -4,19 +4,7 @@ package restserver import ( -<<<<<<< HEAD -======= - "bytes" "encoding/json" - "fmt" - "io/ioutil" - "net" - "net/http" - "net/http/httptest" - "runtime" - "strconv" - "strings" ->>>>>>> Adding changes for CNS to be compatible with managed DNC (reverse communication channel) "sync" "time" @@ -42,8 +30,6 @@ var ( namedLock = acn.InitNamedLock() ) -<<<<<<< HEAD -======= const ( // Key against which CNS state is persisted. storeKey = "ContainerNetworkService" @@ -55,7 +41,6 @@ const ( dncApiVersion = "?api-version=2018-03-01" ) ->>>>>>> Adding changes for CNS to be compatible with managed DNC (reverse communication channel) // HTTPRestService represents http listener for CNS - Container Networking Service. type HTTPRestService struct { *cns.Service @@ -109,7 +94,7 @@ type networkInfo struct { type HTTPService interface { common.ServiceAPI SendNCSnapShotPeriodically(int, chan bool) - SyncNodeStatus(string, string, string) + SyncNodeStatus(string, string, string, json.RawMessage) (int, string) } // NewHTTPRestService creates a new HTTP Service object. diff --git a/cns/service/main.go b/cns/service/main.go index 5340d65000..82a8c49833 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -5,6 +5,7 @@ package main import ( "bytes" + "encoding/json" "fmt" "net/http" "os" @@ -385,7 +386,7 @@ func main() { // Periodically poll (30s) DNC for node updates for { <-time.NewTicker(time.Second * 30).C - httpRestService.SyncNodeStatus(ep, vnet, node) + httpRestService.SyncNodeStatus(ep, vnet, node, json.RawMessage{}) } }(privateEndpoint, infravnet, nodeID) } From 1d70dbd75da205a311d23be1d0c50228b7f7e725 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 9 Jul 2020 17:15:06 -0700 Subject: [PATCH 05/17] add nc version check via NMA --- cns/nmagentclient/nmagentclient.go | 4 ++++ cns/restserver/restserver.go | 1 + cns/service/main.go | 24 +++++++++++++----------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go index f1b1ba3222..f591eee0a3 100644 --- a/cns/nmagentclient/nmagentclient.go +++ b/cns/nmagentclient/nmagentclient.go @@ -9,6 +9,10 @@ import ( "github.com/Azure/azure-container-networking/common" ) +const ( + WireserverIP = "168.63.129.16" +) + // NetworkContainerResponse - NMAgent response. type NetworkContainerResponse struct { ResponseCode string `json:"httpStatusCode"` diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 0fe7ac6f89..e9719e5f57 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -94,6 +94,7 @@ type networkInfo struct { type HTTPService interface { common.ServiceAPI SendNCSnapShotPeriodically(int, chan bool) + SetNodeOrchestrator(*cns.SetOrchestratorTypeRequest) SyncNodeStatus(string, string, string, json.RawMessage) (int, string) } diff --git a/cns/service/main.go b/cns/service/main.go index 82a8c49833..6470446840 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -19,6 +19,7 @@ import ( "github.com/Azure/azure-container-networking/aitelemetry" "github.com/Azure/azure-container-networking/cnm/ipam" "github.com/Azure/azure-container-networking/cnm/network" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/hnsclient" @@ -216,8 +217,7 @@ func printVersion() { } // Try to register node with DNC when CNS is started in managed DNC mode -func registerNode(dncEP, infraVnet, nodeID string) { - logger.Printf("[Azure CNS] SyncNodeStatus") +func registerNode(httpRestService restserver.HTTPService, dncEP, infraVnet, nodeID string) { var ( numCPU = runtime.NumCPU() response *http.Response @@ -227,19 +227,21 @@ func registerNode(dncEP, infraVnet, nodeID string) { ) for err != nil { - response, err = httpc.Post(fmt.Sprintf("%s/%s/node/%s/nclimit/%d%s", dncEP, infraVnet, nodeID, numCPU, dncApiVersion), "application/json", &body) + response, err = httpc.Post(fmt.Sprintf("%s/%s/node/%s/cores/%d%s", dncEP, infraVnet, nodeID, numCPU, dncApiVersion), "application/json", &body) if err == nil { - break + if response.StatusCode == http.StatusCreated { + var req cns.SetOrchestratorTypeRequest + json.NewDecoder(response.Body).Decode(&req) + httpRestService.SetNodeOrchestrator(&req) + } else { + logger.Errorf("[Azure CNS] Failed to register node with managed DNC with http status code: " + strconv.Itoa(response.StatusCode)) + } + + response.Body.Close() } time.Sleep(time.Second * 5) } - response.Body.Close() - - if response.StatusCode != http.StatusOK { - panic("[Azure CNS] Failed to register node with managed DNC with http status code: " + strconv.Itoa(response.StatusCode)) - } - logger.Printf("[Azure CNS] Node Registered") } @@ -381,7 +383,7 @@ func main() { httpRestService.SetOption(acn.OptInfrastructureNetwork, infravnet) httpRestService.SetOption(acn.OptNodeID, nodeID) - registerNode(privateEndpoint, infravnet, nodeID) + registerNode(httpRestService, privateEndpoint, infravnet, nodeID) go func(ep, vnet, node string) { // Periodically poll (30s) DNC for node updates for { From ef4c62d4988c5bf2de04537a8c53ef2c7be3a44a Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Tue, 14 Jul 2020 14:01:28 -0700 Subject: [PATCH 06/17] adding logic to SyncNodeStatus and check if NCWaitingForUpdate for CniADD and CnsAttach calls --- cns/NetworkContainerContract.go | 1 + cns/restserver/restserver.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 726fdbab29..4afbac89ce 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -76,6 +76,7 @@ type CreateNetworkContainerRequest struct { AllowHostToNCCommunication bool AllowNCToHostCommunication bool EndpointPolicies []NetworkContainerRequestPolicies + WaitingForUpdate bool // True when NC is waiting for NMA to sync versions/rules } // NetworkContainerRequestPolicies - specifies policies associated with create network request diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index e9719e5f57..7c6f6e5e88 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -28,6 +28,8 @@ import ( var ( // Named Lock for accessing different states in httpRestServiceState namedLock = acn.InitNamedLock() + // map of NC to their respective NMA getVersion URLs + versionURLs sync.Map ) const ( From 1da8183f3dea301882b5470bd02db55bc4cdb613 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 15 Jul 2020 15:44:50 -0700 Subject: [PATCH 07/17] addressing most of ashvin's comments --- cns/NetworkContainerContract.go | 6 ------ cns/restserver/restserver.go | 2 +- cns/service/main.go | 20 +++++++++++++++++--- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 4afbac89ce..b5dc594fb2 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -329,9 +329,3 @@ type NodeInfoResponse struct { NetworkContainers []CreateNetworkContainerRequest GetNCVersionURLFmt string } - -// JsonErr - Struct to hold err info -type JsonErr struct { - Text string `json:"Text"` - HTTPStatus int `json:"HttpStatusCode"` -} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 7c6f6e5e88..d5b4dc35b1 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -29,7 +29,7 @@ var ( // Named Lock for accessing different states in httpRestServiceState namedLock = acn.InitNamedLock() // map of NC to their respective NMA getVersion URLs - versionURLs sync.Map + ncVersionURLs sync.Map ) const ( diff --git a/cns/service/main.go b/cns/service/main.go index 6470446840..5078a2b808 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -223,7 +223,7 @@ func registerNode(httpRestService restserver.HTTPService, dncEP, infraVnet, node response *http.Response err = fmt.Errorf("") body bytes.Buffer - httpc = &http.Client{Timeout: time.Second * 30} + httpc = acn.GetHttpClient() ) for err != nil { @@ -234,12 +234,15 @@ func registerNode(httpRestService restserver.HTTPService, dncEP, infraVnet, node json.NewDecoder(response.Body).Decode(&req) httpRestService.SetNodeOrchestrator(&req) } else { - logger.Errorf("[Azure CNS] Failed to register node with managed DNC with http status code: " + strconv.Itoa(response.StatusCode)) + logger.Errorf("[Azure CNS] Failed to register node %s/%s with managed DNC with http status code %s", infraVnet, nodeID, strconv.Itoa(response.StatusCode)) } response.Body.Close() + time.Sleep(time.Second * 5) + } else { + logger.Errorf("[Azure CNS] Failed to register node %s/%s with err: %+v", infraVnet, nodeID, err) + return } - time.Sleep(time.Second * 5) } logger.Printf("[Azure CNS] Node Registered") @@ -391,6 +394,17 @@ func main() { httpRestService.SyncNodeStatus(ep, vnet, node, json.RawMessage{}) } }(privateEndpoint, infravnet, nodeID) + } else if !(privateEndpoint == "" && infravnet == "" && nodeID == "") { + if privateEndpoint == "" { + logger.Errorf("Failed to start CNS in managed mode since %s is not set", acn.OptPrivateEndpoint) + } + if infravnet == "" { + logger.Errorf("Failed to start CNS in managed mode since %s is not set", acn.OptInfrastructureNetwork) + } + if nodeID == "" { + logger.Errorf("Failed to start CNS in managed mode since %s is not set", acn.OptNodeID) + } + return } var netPlugin network.NetPlugin From 1af39e51ac03473b2a0e4aba9e2665ff8e3ec0d4 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 16 Jul 2020 17:48:17 -0700 Subject: [PATCH 08/17] adding managed config --- cns/common/service.go | 8 +++-- cns/configuration/cns_config.json | 6 ++++ cns/configuration/configuration.go | 16 +++++++++ cns/restserver/restserver.go | 2 +- cns/service.go | 4 +-- cns/service/main.go | 52 ++++++++++++++++++------------ common/config.go | 4 +++ common/utils.go | 11 ++++--- 8 files changed, 74 insertions(+), 29 deletions(-) diff --git a/cns/common/service.go b/cns/common/service.go index 1d09a06e51..f1700d0266 100644 --- a/cns/common/service.go +++ b/cns/common/service.go @@ -18,6 +18,7 @@ type Service struct { Options map[string]interface{} ErrChan chan error Store store.KeyValueStore + Managed bool } // ServiceAPI defines base interface. @@ -35,20 +36,22 @@ type ServiceConfig struct { Listener *acn.Listener ErrChan chan error Store store.KeyValueStore + Managed bool } // NewService creates a new Service object. -func NewService(name, version string, store store.KeyValueStore) (*Service, error) { +func NewService(name, version string, store store.KeyValueStore, managed bool) (*Service, error) { logger.Debugf("[Azure CNS] Going to create a service object with name: %v. version: %v.", name, version) svc := &Service{ Name: name, Version: version, + Managed: managed, Options: make(map[string]interface{}), Store: store, } - logger.Debugf("[Azure CNS] Finished creating service object with name: %v. version: %v.", name, version) + logger.Debugf("[Azure CNS] Finished creating service object with name: %v. version: %v. managed: %t", name, version, managed) return svc, nil } @@ -65,6 +68,7 @@ func (service *Service) Initialize(config *ServiceConfig) error { service.ErrChan = config.ErrChan service.Store = config.Store service.Version = config.Version + service.Managed = config.Managed logger.Debugf("[Azure CNS] nitialized service: %+v with config: %+v.", service, config) diff --git a/cns/configuration/cns_config.json b/cns/configuration/cns_config.json index 44e134e297..0a50870736 100644 --- a/cns/configuration/cns_config.json +++ b/cns/configuration/cns_config.json @@ -7,5 +7,11 @@ "HeartBeatIntervalInMins": 30, "DebugMode": false, "SnapshotIntervalInMins": 60 + }, + "ManagedSettings": { + "PrivateEndpoint": "", + "InfrastructureNetwork": "", + "NodeID": "", + "NodeSyncIntervalInSeconds": 30 } } diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index edaf70a18d..6fcb1ba597 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -17,6 +17,7 @@ const ( type CNSConfig struct { TelemetrySettings TelemetrySettings + ManagedSettings ManagedSettings } type TelemetrySettings struct { @@ -44,6 +45,13 @@ type TelemetrySettings struct { SnapshotIntervalInMins int } +type ManagedSettings struct { + PrivateEndpoint string + InfrastructureNetwork string + NodeID string + NodeSyncIntervalInSeconds int +} + // This functions reads cns config file and save it in a structure func ReadConfig() (CNSConfig, error) { var cnsConfig CNSConfig @@ -99,7 +107,15 @@ func setTelemetrySettingDefaults(telemetrySettings *TelemetrySettings) { } } +// set managed setting defaults +func setManagedSettingDefaults(managedSettings *ManagedSettings) { + if managedSettings.NodeSyncIntervalInSeconds == 0 { + managedSettings.NodeSyncIntervalInSeconds = 30 + } +} + // Set Default values of CNS config if not specified func SetCNSConfigDefaults(config *CNSConfig) { setTelemetrySettingDefaults(&config.TelemetrySettings) + setManagedSettingDefaults(&config.ManagedSettings) } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index d5b4dc35b1..ecb6676dfa 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -102,7 +102,7 @@ type HTTPService interface { // NewHTTPRestService creates a new HTTP Service object. func NewHTTPRestService(config *common.ServiceConfig) (HTTPService, error) { - service, err := cns.NewService(config.Name, config.Version, config.Store) + service, err := cns.NewService(config.Name, config.Version, config.Store, config.Managed) if err != nil { return nil, err } diff --git a/cns/service.go b/cns/service.go index c6d8b74c36..846c278e0f 100644 --- a/cns/service.go +++ b/cns/service.go @@ -27,8 +27,8 @@ type Service struct { } // NewService creates a new Service object. -func NewService(name, version string, store store.KeyValueStore) (*Service, error) { - service, err := common.NewService(name, version, store) +func NewService(name, version string, store store.KeyValueStore, managed bool) (*Service, error) { + service, err := common.NewService(name, version, store, managed) if err != nil { return nil, err diff --git a/cns/service/main.go b/cns/service/main.go index 5078a2b808..36cf195041 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -208,6 +208,13 @@ var args = acn.ArgumentList{ Type: "string", DefaultValue: "", }, + { + Name: acn.OptManaged, + Shorthand: acn.OptManagedAlias, + Description: "Set to true to enable managed mode. This is deprecated in favor of cns_config.json", + Type: "bool", + DefaultValue: false, + }, } // Prints description and version information. @@ -218,30 +225,36 @@ func printVersion() { // Try to register node with DNC when CNS is started in managed DNC mode func registerNode(httpRestService restserver.HTTPService, dncEP, infraVnet, nodeID string) { + logger.Printf("[Azure CNS] Registering node %s with Infrastructure Network: %s PrivateEndpoint: %s", nodeID, infraVnet, dncEP) + var ( numCPU = runtime.NumCPU() + url = fmt.Sprintf(acn.RegisterNodeURLFmt, dncEP, infraVnet, nodeID, numCPU, dncApiVersion) response *http.Response err = fmt.Errorf("") body bytes.Buffer httpc = acn.GetHttpClient() ) - for err != nil { - response, err = httpc.Post(fmt.Sprintf("%s/%s/node/%s/cores/%d%s", dncEP, infraVnet, nodeID, numCPU, dncApiVersion), "application/json", &body) + for sleep := true; err != nil; sleep = true { + response, err = httpc.Post(url, "application/json", &body) if err == nil { if response.StatusCode == http.StatusCreated { var req cns.SetOrchestratorTypeRequest json.NewDecoder(response.Body).Decode(&req) httpRestService.SetNodeOrchestrator(&req) + sleep = false } else { - logger.Errorf("[Azure CNS] Failed to register node %s/%s with managed DNC with http status code %s", infraVnet, nodeID, strconv.Itoa(response.StatusCode)) + logger.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode)) } response.Body.Close() - time.Sleep(time.Second * 5) } else { - logger.Errorf("[Azure CNS] Failed to register node %s/%s with err: %+v", infraVnet, nodeID, err) - return + logger.Errorf("[Azure CNS] Failed to register node with err: %+v", err) + } + + if sleep { + time.Sleep(acn.FiveSeconds) } } @@ -306,6 +319,7 @@ func main() { logger.Printf("[Azure CNS] Read config :%+v", cnsconfig) disableTelemetry := cnsconfig.TelemetrySettings.DisableAll + config.Managed = acn.GetArg(acn.OptManaged).(bool) || cnsconfig.ManagedSettings != configuration.ManagedSettings{NodeSyncIntervalInSeconds: 30} if !disableTelemetry { ts := cnsconfig.TelemetrySettings @@ -354,6 +368,7 @@ func main() { httpRestService.SetOption(acn.OptCreateDefaultExtNetworkType, createDefaultExtNetworkType) httpRestService.SetOption(acn.OptHttpConnectionTimeout, httpConnectionTimeout) httpRestService.SetOption(acn.OptHttpResponseHeaderTimeout, httpResponseHeaderTimeout) + httpRestService.SetOption(acn.OptManaged, config.Managed) // Create default ext network if commandline option is set if len(strings.TrimSpace(createDefaultExtNetworkType)) > 0 { @@ -381,30 +396,27 @@ func main() { } // If CNS is running on managed DNC mode - if privateEndpoint != "" && infravnet != "" && nodeID != "" { + if config.Managed { + if privateEndpoint == "" || infravnet == "" || nodeID == "" { + logger.Errorf("[Azure CNS] Missing required values to run in managed mode: PrivateEndpoint: %s InfrastructureNetwork: %s NodeID: %s", + privateEndpoint, + infravnet, + nodeID) + return + } + httpRestService.SetOption(acn.OptPrivateEndpoint, privateEndpoint) httpRestService.SetOption(acn.OptInfrastructureNetwork, infravnet) httpRestService.SetOption(acn.OptNodeID, nodeID) registerNode(httpRestService, privateEndpoint, infravnet, nodeID) go func(ep, vnet, node string) { - // Periodically poll (30s) DNC for node updates + // Periodically poll DNC for node updates for { - <-time.NewTicker(time.Second * 30).C + <-time.NewTicker(time.Duration(cnsconfig.ManagedSettings.NodeSyncIntervalInSeconds) * time.Second).C httpRestService.SyncNodeStatus(ep, vnet, node, json.RawMessage{}) } }(privateEndpoint, infravnet, nodeID) - } else if !(privateEndpoint == "" && infravnet == "" && nodeID == "") { - if privateEndpoint == "" { - logger.Errorf("Failed to start CNS in managed mode since %s is not set", acn.OptPrivateEndpoint) - } - if infravnet == "" { - logger.Errorf("Failed to start CNS in managed mode since %s is not set", acn.OptInfrastructureNetwork) - } - if nodeID == "" { - logger.Errorf("Failed to start CNS in managed mode since %s is not set", acn.OptNodeID) - } - return } var netPlugin network.NetPlugin diff --git a/common/config.go b/common/config.go index 279ebeb06e..aaa8de28fe 100644 --- a/common/config.go +++ b/common/config.go @@ -110,4 +110,8 @@ const ( // Node ID/Name OptNodeID = "node-id" OptNodeIDAlias = "n" + + // Managed mode + OptManaged = "managed" + OptManagedAlias = "m" ) diff --git a/common/utils.go b/common/utils.go index fd0832af2b..1e4b315dfb 100644 --- a/common/utils.go +++ b/common/utils.go @@ -22,10 +22,13 @@ import ( ) const ( - metadataURL = "http://169.254.169.254/metadata/instance?api-version=2017-08-01&format=json" - azCloudUrl = "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2018-10-01&format=text" - httpConnectionTimeout = 7 - headerTimeout = 7 + metadataURL = "http://169.254.169.254/metadata/instance?api-version=2017-08-01&format=json" + azCloudUrl = "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2018-10-01&format=text" + httpConnectionTimeout = 7 + headerTimeout = 7 + RegisterNodeURLFmt = "%s/%s/node/%s/cores/%d%s" + SyncNodeNetworkContainersURLFmt = "%s/%s/node/%s%s" + FiveSeconds = 5 * time.Second ) // XmlDocument - Azure host agent XML document format. From 1a9372e14a393e1cfb18a3ce5ff1a91b14570dab Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 16 Jul 2020 18:18:38 -0700 Subject: [PATCH 09/17] fat rebase --- cns/restserver/api.go | 108 +++++++++++++++++++++++++++++++++++ cns/restserver/const.go | 48 ++++++++-------- cns/restserver/restserver.go | 11 ---- cns/restserver/util.go | 107 ++++++++++++++++++++++++++++++++-- 4 files changed, 236 insertions(+), 38 deletions(-) diff --git a/cns/restserver/api.go b/cns/restserver/api.go index 7c8692c373..9bec3f58fd 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -4,10 +4,13 @@ package restserver import ( + "bytes" + "encoding/json" "fmt" "io/ioutil" "net" "net/http" + "net/http/httptest" "runtime" "strings" @@ -15,6 +18,7 @@ import ( "github.com/Azure/azure-container-networking/cns/hnsclient" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/nmagentclient" + "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/platform" ) @@ -1415,3 +1419,107 @@ func (service *HTTPRestService) deleteHostNCApipaEndpoint(w http.ResponseWriter, err = service.Listener.Encode(w, &response) logger.Response(service.Name, response, response.Response.ReturnCode, ReturnCodeToString(response.Response.ReturnCode), err) } + +// SetNodeOrchestrator :- Set node orchestrator after registering with mDNC +func (service *HTTPRestService) SetNodeOrchestrator(r *cns.SetOrchestratorTypeRequest) { + body, _ := json.Marshal(r) + req, _ := http.NewRequest(http.MethodPost, "", bytes.NewBuffer(body)) + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + service.setOrchestratorType(httptest.NewRecorder(), req) +} + +// SyncNodeStatus :- Retrieve the latest node state from DNC & returns the first occurence of returnCode and error with respect to contextFromCNI +func (service *HTTPRestService) SyncNodeStatus(dncEP, infraVnet, nodeID string, contextFromCNI json.RawMessage) (returnCode int, errStr string) { + logger.Printf("[Azure CNS] SyncNodeStatus") + var ( + response *http.Response + err error + nodeInfoResponse cns.NodeInfoResponse + req *http.Request + body []byte + httpc = common.GetHttpClient() + ) + + // try to retrieve NodeInfoResponse from mDNC + response, err = httpc.Get(fmt.Sprintf(common.SyncNodeNetworkContainersURLFmt, dncEP, infraVnet, nodeID, dncApiVersion)) + if err == nil { + if response.StatusCode == http.StatusOK { + err = json.NewDecoder(response.Body).Decode(&nodeInfoResponse) + } else { + err = fmt.Errorf("%d", response.StatusCode) + } + + response.Body.Close() + } + + if err != nil { + returnCode = UnexpectedError + errStr = fmt.Sprintf("[Azure-CNS] Failed to sync node with error: %+v", err) + logger.Errorf(errStr) + return + } + + var ( + ncsToBeAdded = make(map[string]cns.CreateNetworkContainerRequest) + ncsToBeDeleted = make(map[string]bool) + ) + + // determine new NCs and NCs to be deleted + service.RLock() + for ncid := range service.state.ContainerStatus { + ncsToBeDeleted[cns.SwiftPrefix+ncid] = true + } + + for _, nc := range nodeInfoResponse.NetworkContainers { + ncid := cns.SwiftPrefix + nc.NetworkContainerid + delete(ncsToBeDeleted, ncid) + if savedNc, exists := service.state.ContainerStatus[ncid]; !exists || savedNc.CreateNetworkContainerRequest.Version < nc.Version { + ncsToBeAdded[ncid] = nc + } + } + service.RUnlock() + + // check if the version is valid and save it to service state + for _, nc := range ncsToBeAdded { + versionURL := fmt.Sprintf(nodeInfoResponse.GetNCVersionURLFmt, + nmagentclient.WireserverIP, + nc.PrimaryInterfaceIdentifier, + nc.NetworkContainerid, + nc.AuthorizationToken) + ncVersionURLs.Store(nc.NetworkContainerid, versionURL) + waitingForUpdate, tmpReturnCode, tmpErrStr := isNCWaitingForUpdate(nc.NetworkContainerid, nc.Version) + if tmpReturnCode != Success && bytes.Compare(nc.OrchestratorContext, contextFromCNI) == 0 { + returnCode = tmpReturnCode + errStr = tmpErrStr + logger.Errorf(tmpErrStr) + } + + if tmpReturnCode == UnexpectedError { + continue + } + + nc.WaitingForUpdate = waitingForUpdate + body, _ = json.Marshal(nc) + req, _ = http.NewRequest(http.MethodPost, "", bytes.NewBuffer(body)) + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + service.createOrUpdateNetworkContainer(httptest.NewRecorder(), req) + } + + // delete dangling NCs + for nc := range ncsToBeDeleted { + var body bytes.Buffer + json.NewEncoder(&body).Encode(&cns.DeleteNetworkContainerRequest{NetworkContainerid: nc}) + + req, err = http.NewRequest(http.MethodPost, "", &body) + if err == nil { + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + service.deleteNetworkContainer(httptest.NewRecorder(), req) + } else { + logger.Errorf("[Azure-CNS] Failed to delete NC request to sync state: %s", err.Error()) + } + + ncVersionURLs.Delete(nc) + } + + return +} diff --git a/cns/restserver/const.go b/cns/restserver/const.go index 3534758072..fc32226f3f 100644 --- a/cns/restserver/const.go +++ b/cns/restserver/const.go @@ -5,29 +5,30 @@ package restserver // Container Network Service remote API Contract. const ( - Success = 0 - UnsupportedNetworkType = 1 - InvalidParameter = 2 - UnsupportedEnvironment = 3 - UnreachableHost = 4 - ReservationNotFound = 5 - MalformedSubnet = 8 - UnreachableDockerDaemon = 9 - UnspecifiedNetworkName = 10 - NotFound = 14 - AddressUnavailable = 15 - NetworkContainerNotSpecified = 16 - CallToHostFailed = 17 - UnknownContainerID = 18 - UnsupportedOrchestratorType = 19 - DockerContainerNotSpecified = 20 - UnsupportedVerb = 21 - UnsupportedNetworkContainerType = 22 - InvalidRequest = 23 - NetworkJoinFailed = 24 - NetworkContainerPublishFailed = 25 - NetworkContainerUnpublishFailed = 26 - UnexpectedError = 99 + Success = 0 + UnsupportedNetworkType = 1 + InvalidParameter = 2 + UnsupportedEnvironment = 3 + UnreachableHost = 4 + ReservationNotFound = 5 + MalformedSubnet = 8 + UnreachableDockerDaemon = 9 + UnspecifiedNetworkName = 10 + NotFound = 14 + AddressUnavailable = 15 + NetworkContainerNotSpecified = 16 + CallToHostFailed = 17 + UnknownContainerID = 18 + UnsupportedOrchestratorType = 19 + DockerContainerNotSpecified = 20 + UnsupportedVerb = 21 + UnsupportedNetworkContainerType = 22 + InvalidRequest = 23 + NetworkJoinFailed = 24 + NetworkContainerPublishFailed = 25 + NetworkContainerUnpublishFailed = 26 + NetworkContainerPendingStatePropagation = 27 + UnexpectedError = 99 ) const ( @@ -38,4 +39,5 @@ const ( detach = "Detach" // Rest service state identifier for named lock stateJoinedNetworks = "JoinedNetworks" + dncApiVersion = "?api-version=2018-03-01" ) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index ecb6676dfa..f065f56412 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -32,17 +32,6 @@ var ( ncVersionURLs sync.Map ) -const ( - // Key against which CNS state is persisted. - storeKey = "ContainerNetworkService" - swiftAPIVersion = "1" - attach = "Attach" - detach = "Detach" - // Rest service state identifier for named lock - stateJoinedNetworks = "JoinedNetworks" - dncApiVersion = "?api-version=2018-03-01" -) - // HTTPRestService represents http listener for CNS - Container Networking Service. type HTTPRestService struct { *cns.Service diff --git a/cns/restserver/util.go b/cns/restserver/util.go index af39cdaa28..9781c18777 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -6,7 +6,9 @@ package restserver import ( "encoding/json" "fmt" + "io/ioutil" "net/http" + "strconv" "time" "github.com/Azure/azure-container-networking/aitelemetry" @@ -176,8 +178,11 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw } func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkContainerRequest) cns.GetNetworkContainerResponse { - var containerID string - var getNetworkContainerResponse cns.GetNetworkContainerResponse + var ( + containerID string + getNetworkContainerResponse cns.GetNetworkContainerResponse + exists bool + ) service.RLock() defer service.RUnlock() @@ -201,7 +206,34 @@ func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkCo } logger.Printf("pod info %+v", podInfo) - containerID = service.state.ContainerIDByOrchestratorContext[podInfo.PodName+podInfo.PodNamespace] + + var ( + context = podInfo.PodName + podInfo.PodNamespace + dncEP = service.GetOption(acn.OptPrivateEndpoint).(string) + infraVnet = service.GetOption(acn.OptInfrastructureNetwork).(string) + nodeID = service.GetOption(acn.OptNodeID).(string) + managed = service.GetOption(acn.OptManaged).(bool) + ) + + containerID, exists = service.state.ContainerIDByOrchestratorContext[context] + if managed { + if exists { + _, getNetworkContainerResponse.Response.ReturnCode, getNetworkContainerResponse.Response.Message = isNCWaitingForUpdate(service.state.ContainerStatus[containerID].CreateNetworkContainerRequest.Version, containerID) + if getNetworkContainerResponse.Response.ReturnCode == Success { + return getNetworkContainerResponse + } + } else { + service.RUnlock() + getNetworkContainerResponse.Response.ReturnCode, getNetworkContainerResponse.Response.Message = service.SyncNodeStatus(dncEP, infraVnet, nodeID, req.OrchestratorContext) + service.RLock() + if getNetworkContainerResponse.Response.ReturnCode == NotFound { + return getNetworkContainerResponse + } + + containerID = service.state.ContainerIDByOrchestratorContext[context] + } + } + logger.Printf("containerid %v", containerID) break @@ -299,9 +331,35 @@ func (service *HTTPRestService) attachOrDetachHelper(req cns.ConfigureContainerN Message: "[Azure CNS] Error. NetworkContainerid is empty"} } + var ( + dncEP = service.GetOption(acn.OptPrivateEndpoint).(string) + infraVnet = service.GetOption(acn.OptInfrastructureNetwork).(string) + nodeID = service.GetOption(acn.OptNodeID).(string) + isManagedDnc = service.GetOption(acn.OptManaged).(bool) + ) + existing, ok := service.getNetworkContainerDetails(cns.SwiftPrefix + req.NetworkContainerid) + if isManagedDnc && operation == attach { + if ok { + if existing.CreateNetworkContainerRequest.WaitingForUpdate { + _, returnCode, message := isNCWaitingForUpdate(existing.CreateNetworkContainerRequest.Version, req.NetworkContainerid) + if returnCode != Success { + return cns.Response{ + ReturnCode: returnCode, + Message: message} + } + } + } else { + returnCode, msg := service.SyncNodeStatus(dncEP, infraVnet, nodeID, json.RawMessage{}) + if returnCode != Success { + return cns.Response{ + ReturnCode: returnCode, + Message: msg} + } - if !ok { + existing, _ = service.getNetworkContainerDetails(cns.SwiftPrefix + req.NetworkContainerid) + } + } else if !ok { return cns.Response{ ReturnCode: NotFound, Message: fmt.Sprintf("[Azure CNS] Error. Network Container %s does not exist.", req.NetworkContainerid)} @@ -441,6 +499,45 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ncSnapshotIntervalInM } } +// isNCWaitingForUpdate :- Determine whether NC version on NMA matches programmed version +func isNCWaitingForUpdate(ncVersion, ncid string) (waitingForUpdate bool, returnCode int, message string) { + getNCVersionURL, ok := ncVersionURLs.Load(ncid) + if !ok { + returnCode = NotFound + message = fmt.Sprintf("[Azure-CNS] Network container %s not found", ncid) + return + } + + response, err := nmagentclient.GetNetworkContainerVersion(ncid, getNCVersionURL.(string)) + if err == nil { + if response.StatusCode == http.StatusOK { + var versionResponse nmagentclient.NetworkContainerResponse + rBytes, _ := ioutil.ReadAll(response.Body) + json.Unmarshal(rBytes, &versionResponse) + if versionResponse.ResponseCode == "200" { + programmedVersion, _ := strconv.Atoi(ncVersion) + nmaVersion, _ := strconv.Atoi(versionResponse.Version) + if programmedVersion > nmaVersion { + waitingForUpdate = true + returnCode = NetworkContainerPendingStatePropagation + message = fmt.Sprintf("[Azure-CNS] Network container %s v%d had not propagated to respective NMA w/ v%d", ncid, programmedVersion, nmaVersion) + } + } else { + returnCode = UnexpectedError + message = fmt.Sprintf("[Azure-CNS] Failed to get NC version from response %s for NC %s", rBytes, ncid) + } + } else { + returnCode = UnexpectedError + message = fmt.Sprintf("[Azure-CNS] Failed to get NC version with http status %d", response.StatusCode) + } + } else { + returnCode = UnexpectedError + message = fmt.Sprintf("[Azure-CNS] Failed to get NC version from NMA with error: %+v", err) + } + + return +} + // ReturnCodeToString - Converts an error code to appropriate string. func ReturnCodeToString(returnCode int) (s string) { switch returnCode { @@ -476,6 +573,8 @@ func ReturnCodeToString(returnCode int) (s string) { s = "UnexpectedError" case DockerContainerNotSpecified: s = "DockerContainerNotSpecified" + case NetworkContainerPendingStatePropagation: + s = "NetworkContainerPendingStatePropagation" default: s = "UnknownError" } From 5084325a2c7bff9a537ea38dbb7d90a57f8c5811 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 17 Jul 2020 13:37:43 -0700 Subject: [PATCH 10/17] addressing some comments --- cns/NetworkContainerContract.go | 1 - cns/restserver/api.go | 114 ++------------------------- cns/restserver/internalapi.go | 133 ++++++++++++++++++++++++++++++++ cns/restserver/restserver.go | 1 + cns/restserver/util.go | 2 +- 5 files changed, 140 insertions(+), 111 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index b5dc594fb2..6b2495a09e 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -76,7 +76,6 @@ type CreateNetworkContainerRequest struct { AllowHostToNCCommunication bool AllowNCToHostCommunication bool EndpointPolicies []NetworkContainerRequestPolicies - WaitingForUpdate bool // True when NC is waiting for NMA to sync versions/rules } // NetworkContainerRequestPolicies - specifies policies associated with create network request diff --git a/cns/restserver/api.go b/cns/restserver/api.go index 9bec3f58fd..11c4d43860 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -4,13 +4,10 @@ package restserver import ( - "bytes" - "encoding/json" "fmt" "io/ioutil" "net" "net/http" - "net/http/httptest" "runtime" "strings" @@ -18,7 +15,6 @@ import ( "github.com/Azure/azure-container-networking/cns/hnsclient" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/nmagentclient" - "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/platform" ) @@ -779,6 +775,10 @@ func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWr returnMessage = fmt.Sprintf("[Azure CNS] Error. NetworkContainerid is empty") } + if !strings.Contains(req.NetworkContainerid, cns.SwiftPrefix) { + req.NetworkContainerid = cns.SwiftPrefix + req.NetworkContainerid + } + switch r.Method { case "POST": if req.NetworkContainerType == cns.WebApps { @@ -826,7 +826,7 @@ func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWr err = service.Listener.Encode(w, &reserveResp) // If the NC was created successfully, log NC snapshot. - if returnCode == 0 { + if returnCode == Success { logNCSnapshot(req) } @@ -1419,107 +1419,3 @@ func (service *HTTPRestService) deleteHostNCApipaEndpoint(w http.ResponseWriter, err = service.Listener.Encode(w, &response) logger.Response(service.Name, response, response.Response.ReturnCode, ReturnCodeToString(response.Response.ReturnCode), err) } - -// SetNodeOrchestrator :- Set node orchestrator after registering with mDNC -func (service *HTTPRestService) SetNodeOrchestrator(r *cns.SetOrchestratorTypeRequest) { - body, _ := json.Marshal(r) - req, _ := http.NewRequest(http.MethodPost, "", bytes.NewBuffer(body)) - req.Header.Set("Content-Type", "application/json; charset=UTF-8") - service.setOrchestratorType(httptest.NewRecorder(), req) -} - -// SyncNodeStatus :- Retrieve the latest node state from DNC & returns the first occurence of returnCode and error with respect to contextFromCNI -func (service *HTTPRestService) SyncNodeStatus(dncEP, infraVnet, nodeID string, contextFromCNI json.RawMessage) (returnCode int, errStr string) { - logger.Printf("[Azure CNS] SyncNodeStatus") - var ( - response *http.Response - err error - nodeInfoResponse cns.NodeInfoResponse - req *http.Request - body []byte - httpc = common.GetHttpClient() - ) - - // try to retrieve NodeInfoResponse from mDNC - response, err = httpc.Get(fmt.Sprintf(common.SyncNodeNetworkContainersURLFmt, dncEP, infraVnet, nodeID, dncApiVersion)) - if err == nil { - if response.StatusCode == http.StatusOK { - err = json.NewDecoder(response.Body).Decode(&nodeInfoResponse) - } else { - err = fmt.Errorf("%d", response.StatusCode) - } - - response.Body.Close() - } - - if err != nil { - returnCode = UnexpectedError - errStr = fmt.Sprintf("[Azure-CNS] Failed to sync node with error: %+v", err) - logger.Errorf(errStr) - return - } - - var ( - ncsToBeAdded = make(map[string]cns.CreateNetworkContainerRequest) - ncsToBeDeleted = make(map[string]bool) - ) - - // determine new NCs and NCs to be deleted - service.RLock() - for ncid := range service.state.ContainerStatus { - ncsToBeDeleted[cns.SwiftPrefix+ncid] = true - } - - for _, nc := range nodeInfoResponse.NetworkContainers { - ncid := cns.SwiftPrefix + nc.NetworkContainerid - delete(ncsToBeDeleted, ncid) - if savedNc, exists := service.state.ContainerStatus[ncid]; !exists || savedNc.CreateNetworkContainerRequest.Version < nc.Version { - ncsToBeAdded[ncid] = nc - } - } - service.RUnlock() - - // check if the version is valid and save it to service state - for _, nc := range ncsToBeAdded { - versionURL := fmt.Sprintf(nodeInfoResponse.GetNCVersionURLFmt, - nmagentclient.WireserverIP, - nc.PrimaryInterfaceIdentifier, - nc.NetworkContainerid, - nc.AuthorizationToken) - ncVersionURLs.Store(nc.NetworkContainerid, versionURL) - waitingForUpdate, tmpReturnCode, tmpErrStr := isNCWaitingForUpdate(nc.NetworkContainerid, nc.Version) - if tmpReturnCode != Success && bytes.Compare(nc.OrchestratorContext, contextFromCNI) == 0 { - returnCode = tmpReturnCode - errStr = tmpErrStr - logger.Errorf(tmpErrStr) - } - - if tmpReturnCode == UnexpectedError { - continue - } - - nc.WaitingForUpdate = waitingForUpdate - body, _ = json.Marshal(nc) - req, _ = http.NewRequest(http.MethodPost, "", bytes.NewBuffer(body)) - req.Header.Set("Content-Type", "application/json; charset=UTF-8") - service.createOrUpdateNetworkContainer(httptest.NewRecorder(), req) - } - - // delete dangling NCs - for nc := range ncsToBeDeleted { - var body bytes.Buffer - json.NewEncoder(&body).Encode(&cns.DeleteNetworkContainerRequest{NetworkContainerid: nc}) - - req, err = http.NewRequest(http.MethodPost, "", &body) - if err == nil { - req.Header.Set("Content-Type", "application/json; charset=UTF-8") - service.deleteNetworkContainer(httptest.NewRecorder(), req) - } else { - logger.Errorf("[Azure-CNS] Failed to delete NC request to sync state: %s", err.Error()) - } - - ncVersionURLs.Delete(nc) - } - - return -} diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go index fab353c370..f910d2e476 100644 --- a/cns/restserver/internalapi.go +++ b/cns/restserver/internalapi.go @@ -3,6 +3,19 @@ package restserver +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/nmagentclient" + "github.com/Azure/azure-container-networking/common" +) + // This file contains the internal functions called by either HTTP APIs (api.go) or // internal APIs (definde in internalapi.go). // This will be used internally (say by RequestController in case of AKS) @@ -14,3 +27,123 @@ func (service *HTTPRestService) GetPartitionKey() (dncPartitionKey string) { service.RUnlock() return } + +// SetNodeOrchestrator :- Set node orchestrator after registering with mDNC +func (service *HTTPRestService) SetNodeOrchestrator(r *cns.SetOrchestratorTypeRequest) { + body, _ := json.Marshal(r) + req, _ := http.NewRequest(http.MethodPost, "", bytes.NewBuffer(body)) + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + service.setOrchestratorType(httptest.NewRecorder(), req) +} + +// SyncNodeStatus :- Retrieve the latest node state from DNC & returns the first occurence of returnCode and error with respect to contextFromCNI +func (service *HTTPRestService) SyncNodeStatus(dncEP, infraVnet, nodeID string, contextFromCNI json.RawMessage) (returnCode int, errStr string) { + logger.Printf("[Azure CNS] SyncNodeStatus") + var ( + response *http.Response + err error + nodeInfoResponse cns.NodeInfoResponse + req *http.Request + body []byte + httpc = common.GetHttpClient() + ) + + // try to retrieve NodeInfoResponse from mDNC + response, err = httpc.Get(fmt.Sprintf(common.SyncNodeNetworkContainersURLFmt, dncEP, infraVnet, nodeID, dncApiVersion)) + if err == nil { + if response.StatusCode == http.StatusOK { + err = json.NewDecoder(response.Body).Decode(&nodeInfoResponse) + } else { + err = fmt.Errorf("%d", response.StatusCode) + } + + response.Body.Close() + } + + if err != nil { + returnCode = UnexpectedError + errStr = fmt.Sprintf("[Azure-CNS] Failed to sync node with error: %+v", err) + logger.Errorf(errStr) + return + } + + var ( + ncsToBeAdded = make(map[string]cns.CreateNetworkContainerRequest) + ncsToBeDeleted = make(map[string]bool) + ) + + // determine new NCs and NCs to be deleted + service.RLock() + for ncid := range service.state.ContainerStatus { + ncsToBeDeleted[ncid] = true + } + + for _, nc := range nodeInfoResponse.NetworkContainers { + ncid := cns.SwiftPrefix + nc.NetworkContainerid + delete(ncsToBeDeleted, ncid) + if savedNc, exists := service.state.ContainerStatus[ncid]; !exists || savedNc.CreateNetworkContainerRequest.Version < nc.Version { + ncsToBeAdded[ncid] = nc + } + } + service.RUnlock() + + // check if the version is valid and save it to service state + for ncid, nc := range ncsToBeAdded { + var ( + versionURL = fmt.Sprintf(nodeInfoResponse.GetNCVersionURLFmt, + nmagentclient.WireserverIP, + nc.PrimaryInterfaceIdentifier, + nc.NetworkContainerid, + nc.AuthorizationToken) + w = httptest.NewRecorder() + ) + + ncVersionURLs.Store(nc.NetworkContainerid, versionURL) + waitingForUpdate, tmpReturnCode, tmpErrStr := isNCWaitingForUpdate(nc.Version, nc.NetworkContainerid) + if tmpReturnCode != Success && bytes.Compare(nc.OrchestratorContext, contextFromCNI) == 0 { + returnCode = tmpReturnCode + errStr = tmpErrStr + } + + if tmpReturnCode == UnexpectedError { + continue + } + + body, _ = json.Marshal(nc) + req, _ = http.NewRequest(http.MethodPost, "", bytes.NewBuffer(body)) + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + service.createOrUpdateNetworkContainer(w, req) + if w.Result().StatusCode == http.StatusOK { + var resp cns.CreateNetworkContainerResponse + if err = json.Unmarshal(w.Body.Bytes(), &resp); err == nil && resp.Response.ReturnCode == Success { + service.Lock() + ncstatus, _ := service.state.ContainerStatus[ncid] + ncstatus.WaitingForUpdate = waitingForUpdate + service.state.ContainerStatus[ncid] = ncstatus + service.Unlock() + } + } + } + + service.Lock() + service.saveState() + service.Unlock() + + // delete dangling NCs + for nc := range ncsToBeDeleted { + var body bytes.Buffer + json.NewEncoder(&body).Encode(&cns.DeleteNetworkContainerRequest{NetworkContainerid: nc}) + + req, err = http.NewRequest(http.MethodPost, "", &body) + if err == nil { + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + service.deleteNetworkContainer(httptest.NewRecorder(), req) + } else { + logger.Errorf("[Azure-CNS] Failed to delete NC request to sync state: %s", err.Error()) + } + + ncVersionURLs.Delete(nc) + } + + return +} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index f065f56412..9a8e6f8064 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -59,6 +59,7 @@ type containerstatus struct { VMVersion string HostVersion string CreateNetworkContainerRequest cns.CreateNetworkContainerRequest + WaitingForUpdate bool // True when NC is waiting for NMA to sync versions/rules } // httpRestServiceState contains the state we would like to persist. diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 9781c18777..21e3487aaa 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -341,7 +341,7 @@ func (service *HTTPRestService) attachOrDetachHelper(req cns.ConfigureContainerN existing, ok := service.getNetworkContainerDetails(cns.SwiftPrefix + req.NetworkContainerid) if isManagedDnc && operation == attach { if ok { - if existing.CreateNetworkContainerRequest.WaitingForUpdate { + if existing.WaitingForUpdate { _, returnCode, message := isNCWaitingForUpdate(existing.CreateNetworkContainerRequest.Version, req.NetworkContainerid) if returnCode != Success { return cns.Response{ From 048c6bd5329bf95d6d4b8117a70b3fbd5f7b3e3f Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 20 Jul 2020 10:43:43 -0700 Subject: [PATCH 11/17] slight optimizations... --- cns/NetworkContainerContract.go | 6 ++++++ cns/restserver/util.go | 32 +++++++++++++++----------------- cns/service/main.go | 11 +++++++++-- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 6b2495a09e..a279d92f30 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -59,6 +59,12 @@ const ( PendingRelease = "PendingRelease" ) +// CNS Modes +const ( + Direct = "Direct" + PrivateLink = "PrivateLink" +) + // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary. type CreateNetworkContainerRequest struct { Version string diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 21e3487aaa..8a85de1f86 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -207,22 +207,21 @@ func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkCo logger.Printf("pod info %+v", podInfo) - var ( - context = podInfo.PodName + podInfo.PodNamespace - dncEP = service.GetOption(acn.OptPrivateEndpoint).(string) - infraVnet = service.GetOption(acn.OptInfrastructureNetwork).(string) - nodeID = service.GetOption(acn.OptNodeID).(string) - managed = service.GetOption(acn.OptManaged).(bool) - ) - + context := podInfo.PodName + podInfo.PodNamespace containerID, exists = service.state.ContainerIDByOrchestratorContext[context] - if managed { + if service.Managed { if exists { _, getNetworkContainerResponse.Response.ReturnCode, getNetworkContainerResponse.Response.Message = isNCWaitingForUpdate(service.state.ContainerStatus[containerID].CreateNetworkContainerRequest.Version, containerID) if getNetworkContainerResponse.Response.ReturnCode == Success { return getNetworkContainerResponse } } else { + var ( + dncEP = service.GetOption(acn.OptPrivateEndpoint).(string) + infraVnet = service.GetOption(acn.OptInfrastructureNetwork).(string) + nodeID = service.GetOption(acn.OptNodeID).(string) + ) + service.RUnlock() getNetworkContainerResponse.Response.ReturnCode, getNetworkContainerResponse.Response.Message = service.SyncNodeStatus(dncEP, infraVnet, nodeID, req.OrchestratorContext) service.RLock() @@ -331,15 +330,8 @@ func (service *HTTPRestService) attachOrDetachHelper(req cns.ConfigureContainerN Message: "[Azure CNS] Error. NetworkContainerid is empty"} } - var ( - dncEP = service.GetOption(acn.OptPrivateEndpoint).(string) - infraVnet = service.GetOption(acn.OptInfrastructureNetwork).(string) - nodeID = service.GetOption(acn.OptNodeID).(string) - isManagedDnc = service.GetOption(acn.OptManaged).(bool) - ) - existing, ok := service.getNetworkContainerDetails(cns.SwiftPrefix + req.NetworkContainerid) - if isManagedDnc && operation == attach { + if service.Managed && operation == attach { if ok { if existing.WaitingForUpdate { _, returnCode, message := isNCWaitingForUpdate(existing.CreateNetworkContainerRequest.Version, req.NetworkContainerid) @@ -350,6 +342,12 @@ func (service *HTTPRestService) attachOrDetachHelper(req cns.ConfigureContainerN } } } else { + var ( + dncEP = service.GetOption(acn.OptPrivateEndpoint).(string) + infraVnet = service.GetOption(acn.OptInfrastructureNetwork).(string) + nodeID = service.GetOption(acn.OptNodeID).(string) + ) + returnCode, msg := service.SyncNodeStatus(dncEP, infraVnet, nodeID, json.RawMessage{}) if returnCode != Success { return cns.Response{ diff --git a/cns/service/main.go b/cns/service/main.go index 36cf195041..8b8fd230dc 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -318,9 +318,16 @@ func main() { configuration.SetCNSConfigDefaults(&cnsconfig) logger.Printf("[Azure CNS] Read config :%+v", cnsconfig) - disableTelemetry := cnsconfig.TelemetrySettings.DisableAll - config.Managed = acn.GetArg(acn.OptManaged).(bool) || cnsconfig.ManagedSettings != configuration.ManagedSettings{NodeSyncIntervalInSeconds: 30} + if cnsconfig.ManagedSettings != (configuration.ManagedSettings{NodeSyncIntervalInSeconds: 30}) { + config.Managed = true + privateEndpoint = cnsconfig.ManagedSettings.PrivateEndpoint + infravnet = cnsconfig.ManagedSettings.InfrastructureNetwork + nodeID = cnsconfig.ManagedSettings.NodeID + } else { + config.Managed = acn.GetArg(acn.OptManaged).(bool) + } + disableTelemetry := cnsconfig.TelemetrySettings.DisableAll if !disableTelemetry { ts := cnsconfig.TelemetrySettings aiConfig := aitelemetry.AIConfig{ From 3666fe4219ebd88147441d4e26567f2c29abf857 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 20 Jul 2020 17:26:38 -0700 Subject: [PATCH 12/17] adding channel mode instead of managed bool --- cns/NetworkContainerContract.go | 7 +++--- cns/common/service.go | 40 +++++++++++++++--------------- cns/configuration/configuration.go | 5 ++++ cns/restserver/restserver.go | 2 +- cns/restserver/util.go | 4 +-- cns/service.go | 4 +-- cns/service/main.go | 10 +++----- 7 files changed, 38 insertions(+), 34 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index a279d92f30..d15b45afe0 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -59,10 +59,11 @@ const ( PendingRelease = "PendingRelease" ) -// CNS Modes +// ChannelMode :- CNS channel modes const ( - Direct = "Direct" - PrivateLink = "PrivateLink" + Direct = "Direct" + Managed = "Managed" + CRD = "CRD" ) // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary. diff --git a/cns/common/service.go b/cns/common/service.go index f1700d0266..e17e936b58 100644 --- a/cns/common/service.go +++ b/cns/common/service.go @@ -13,12 +13,12 @@ import ( // Service implements behavior common to all services. type Service struct { - Name string - Version string - Options map[string]interface{} - ErrChan chan error - Store store.KeyValueStore - Managed bool + Name string + Version string + Options map[string]interface{} + ErrChan chan error + Store store.KeyValueStore + ChannelMode string } // ServiceAPI defines base interface. @@ -31,27 +31,27 @@ type ServiceAPI interface { // ServiceConfig specifies common configuration. type ServiceConfig struct { - Name string - Version string - Listener *acn.Listener - ErrChan chan error - Store store.KeyValueStore - Managed bool + Name string + Version string + Listener *acn.Listener + ErrChan chan error + Store store.KeyValueStore + ChannelMode string } // NewService creates a new Service object. -func NewService(name, version string, store store.KeyValueStore, managed bool) (*Service, error) { +func NewService(name, version, channelMode string, store store.KeyValueStore) (*Service, error) { logger.Debugf("[Azure CNS] Going to create a service object with name: %v. version: %v.", name, version) svc := &Service{ - Name: name, - Version: version, - Managed: managed, - Options: make(map[string]interface{}), - Store: store, + Name: name, + Version: version, + ChannelMode: channelMode, + Options: make(map[string]interface{}), + Store: store, } - logger.Debugf("[Azure CNS] Finished creating service object with name: %v. version: %v. managed: %t", name, version, managed) + logger.Debugf("[Azure CNS] Finished creating service object with name: %v. version: %v. managed: %s", name, version, channelMode) return svc, nil } @@ -68,7 +68,7 @@ func (service *Service) Initialize(config *ServiceConfig) error { service.ErrChan = config.ErrChan service.Store = config.Store service.Version = config.Version - service.Managed = config.Managed + service.ChannelMode = config.ChannelMode logger.Debugf("[Azure CNS] nitialized service: %+v with config: %+v.", service, config) diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index 6fcb1ba597..7310baa6ce 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/common" ) @@ -18,6 +19,7 @@ const ( type CNSConfig struct { TelemetrySettings TelemetrySettings ManagedSettings ManagedSettings + ChannelMode string } type TelemetrySettings struct { @@ -118,4 +120,7 @@ func setManagedSettingDefaults(managedSettings *ManagedSettings) { func SetCNSConfigDefaults(config *CNSConfig) { setTelemetrySettingDefaults(&config.TelemetrySettings) setManagedSettingDefaults(&config.ManagedSettings) + if config.ChannelMode == "" { + config.ChannelMode = cns.Direct + } } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 9a8e6f8064..942c8bad07 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -92,7 +92,7 @@ type HTTPService interface { // NewHTTPRestService creates a new HTTP Service object. func NewHTTPRestService(config *common.ServiceConfig) (HTTPService, error) { - service, err := cns.NewService(config.Name, config.Version, config.Store, config.Managed) + service, err := cns.NewService(config.Name, config.Version, config.ChannelMode, config.Store) if err != nil { return nil, err } diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 8a85de1f86..f1bcecb3bf 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -209,7 +209,7 @@ func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkCo context := podInfo.PodName + podInfo.PodNamespace containerID, exists = service.state.ContainerIDByOrchestratorContext[context] - if service.Managed { + if service.ChannelMode == cns.Managed { if exists { _, getNetworkContainerResponse.Response.ReturnCode, getNetworkContainerResponse.Response.Message = isNCWaitingForUpdate(service.state.ContainerStatus[containerID].CreateNetworkContainerRequest.Version, containerID) if getNetworkContainerResponse.Response.ReturnCode == Success { @@ -331,7 +331,7 @@ func (service *HTTPRestService) attachOrDetachHelper(req cns.ConfigureContainerN } existing, ok := service.getNetworkContainerDetails(cns.SwiftPrefix + req.NetworkContainerid) - if service.Managed && operation == attach { + if service.ChannelMode == cns.Managed && operation == attach { if ok { if existing.WaitingForUpdate { _, returnCode, message := isNCWaitingForUpdate(existing.CreateNetworkContainerRequest.Version, req.NetworkContainerid) diff --git a/cns/service.go b/cns/service.go index 846c278e0f..15420b1e5b 100644 --- a/cns/service.go +++ b/cns/service.go @@ -27,8 +27,8 @@ type Service struct { } // NewService creates a new Service object. -func NewService(name, version string, store store.KeyValueStore, managed bool) (*Service, error) { - service, err := common.NewService(name, version, store, managed) +func NewService(name, version, channelMode string, store store.KeyValueStore) (*Service, error) { + service, err := common.NewService(name, version, channelMode, store) if err != nil { return nil, err diff --git a/cns/service/main.go b/cns/service/main.go index 8b8fd230dc..5718d7c9f9 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -318,13 +318,12 @@ func main() { configuration.SetCNSConfigDefaults(&cnsconfig) logger.Printf("[Azure CNS] Read config :%+v", cnsconfig) - if cnsconfig.ManagedSettings != (configuration.ManagedSettings{NodeSyncIntervalInSeconds: 30}) { - config.Managed = true + if cnsconfig.ChannelMode == cns.Managed { privateEndpoint = cnsconfig.ManagedSettings.PrivateEndpoint infravnet = cnsconfig.ManagedSettings.InfrastructureNetwork nodeID = cnsconfig.ManagedSettings.NodeID - } else { - config.Managed = acn.GetArg(acn.OptManaged).(bool) + } else if acn.GetArg(acn.OptManaged).(bool) { + config.ChannelMode = cns.Managed } disableTelemetry := cnsconfig.TelemetrySettings.DisableAll @@ -375,7 +374,6 @@ func main() { httpRestService.SetOption(acn.OptCreateDefaultExtNetworkType, createDefaultExtNetworkType) httpRestService.SetOption(acn.OptHttpConnectionTimeout, httpConnectionTimeout) httpRestService.SetOption(acn.OptHttpResponseHeaderTimeout, httpResponseHeaderTimeout) - httpRestService.SetOption(acn.OptManaged, config.Managed) // Create default ext network if commandline option is set if len(strings.TrimSpace(createDefaultExtNetworkType)) > 0 { @@ -403,7 +401,7 @@ func main() { } // If CNS is running on managed DNC mode - if config.Managed { + if config.ChannelMode == cns.Managed { if privateEndpoint == "" || infravnet == "" || nodeID == "" { logger.Errorf("[Azure CNS] Missing required values to run in managed mode: PrivateEndpoint: %s InfrastructureNetwork: %s NodeID: %s", privateEndpoint, From 945410d2d14853275a73cb86e2b8f93bb168fedc Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Tue, 21 Jul 2020 11:29:15 -0700 Subject: [PATCH 13/17] set err in register node so that we keep looping --- cns/service/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cns/service/main.go b/cns/service/main.go index 5718d7c9f9..7d4c9e049d 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -245,7 +245,8 @@ func registerNode(httpRestService restserver.HTTPService, dncEP, infraVnet, node httpRestService.SetNodeOrchestrator(&req) sleep = false } else { - logger.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode)) + err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode)) + logger.Errorf(err.Error()) } response.Body.Close() From aba2db4ddd2b89601c1e4ffb55733d322ff5be06 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Tue, 21 Jul 2020 14:13:04 -0700 Subject: [PATCH 14/17] addressing ashvin's comments --- cns/configuration/cns_config.json | 3 ++- cns/nmagentclient/nmagentclient.go | 4 ++-- cns/restserver/util.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cns/configuration/cns_config.json b/cns/configuration/cns_config.json index 0a50870736..88d00a253f 100644 --- a/cns/configuration/cns_config.json +++ b/cns/configuration/cns_config.json @@ -13,5 +13,6 @@ "InfrastructureNetwork": "", "NodeID": "", "NodeSyncIntervalInSeconds": 30 - } + }, + "ChannelMode": "Direct" } diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go index f591eee0a3..4b0d2d7790 100644 --- a/cns/nmagentclient/nmagentclient.go +++ b/cns/nmagentclient/nmagentclient.go @@ -13,8 +13,8 @@ const ( WireserverIP = "168.63.129.16" ) -// NetworkContainerResponse - NMAgent response. -type NetworkContainerResponse struct { +// NMANetworkContainerResponse - NMAgent response. +type NMANetworkContainerResponse struct { ResponseCode string `json:"httpStatusCode"` NetworkContainerID string `json:"networkContainerId"` Version string `json:"version"` diff --git a/cns/restserver/util.go b/cns/restserver/util.go index f1bcecb3bf..1257a5a41c 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -509,7 +509,7 @@ func isNCWaitingForUpdate(ncVersion, ncid string) (waitingForUpdate bool, return response, err := nmagentclient.GetNetworkContainerVersion(ncid, getNCVersionURL.(string)) if err == nil { if response.StatusCode == http.StatusOK { - var versionResponse nmagentclient.NetworkContainerResponse + var versionResponse nmagentclient.NMANetworkContainerResponse rBytes, _ := ioutil.ReadAll(response.Body) json.Unmarshal(rBytes, &versionResponse) if versionResponse.ResponseCode == "200" { From e4beffa01d561c8f1d12b052faa232448180b099 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Tue, 21 Jul 2020 17:01:05 -0700 Subject: [PATCH 15/17] fix test --- cns/restserver/api.go | 2 -- cns/restserver/api_test.go | 20 ++++++++++---------- cns/restserver/util.go | 8 ++++---- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/cns/restserver/api.go b/cns/restserver/api.go index 11c4d43860..dba9520639 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -974,8 +974,6 @@ func (service *HTTPRestService) getNetworkContainerStatus(w http.ResponseWriter, containerInfo := service.state.ContainerStatus if containerInfo != nil { containerDetails, ok = containerInfo[req.NetworkContainerid] - } else { - ok = false } var hostVersion string diff --git a/cns/restserver/api_test.go b/cns/restserver/api_test.go index 375bc8b201..fe493be673 100644 --- a/cns/restserver/api_test.go +++ b/cns/restserver/api_test.go @@ -218,7 +218,7 @@ func TestCreateNetworkContainer(t *testing.T) { fmt.Println("Now calling DeleteNetworkContainer") - err = deleteNetworkAdapterWithName(t, "ethWebApp") + err = deleteNetworkAdapterWithName(t, cns.SwiftPrefix+"ethWebApp") if err != nil { t.Errorf("Deleting interface failed Err:%+v", err) t.Fatal(err) @@ -248,7 +248,7 @@ func TestGetNetworkContainerByOrchestratorContext(t *testing.T) { setEnv(t) setOrchestratorType(t, cns.Kubernetes) - err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", "AzureContainerInstance") + err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", cns.AzureContainerInstance) if err != nil { t.Errorf("creatOrUpdateNetworkContainerWithName failed Err:%+v", err) t.Fatal(err) @@ -263,7 +263,7 @@ func TestGetNetworkContainerByOrchestratorContext(t *testing.T) { fmt.Println("Now calling DeleteNetworkContainer") - err = deleteNetworkAdapterWithName(t, "ethWebApp") + err = deleteNetworkAdapterWithName(t, cns.SwiftPrefix+"ethWebApp") if err != nil { t.Errorf("Deleting interface failed Err:%+v", err) t.Fatal(err) @@ -283,14 +283,14 @@ func TestGetNetworkContainerStatus(t *testing.T) { setEnv(t) setOrchestratorType(t, cns.Kubernetes) - err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", "WebApps") + err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", cns.AzureContainerInstance) if err != nil { t.Errorf("creatOrUpdateWebAppContainerWithName failed Err:%+v", err) t.Fatal(err) } fmt.Println("Now calling getNetworkContainerStatus") - err = getNetworkContainerStatus(t, "ethWebApp") + err = getNetworkContainerStatus(t, cns.SwiftPrefix+"ethWebApp") if err != nil { t.Errorf("getNetworkContainerStatus failed Err:%+v", err) t.Fatal(err) @@ -298,7 +298,7 @@ func TestGetNetworkContainerStatus(t *testing.T) { fmt.Println("Now calling DeleteNetworkContainer") - err = deleteNetworkAdapterWithName(t, "ethWebApp") + err = deleteNetworkAdapterWithName(t, cns.SwiftPrefix+"ethWebApp") if err != nil { t.Errorf("Deleting interface failed Err:%+v", err) t.Fatal(err) @@ -319,7 +319,7 @@ func TestGetInterfaceForNetworkContainer(t *testing.T) { } fmt.Println("Now calling getInterfaceForContainer") - err = getInterfaceForContainer(t, "ethWebApp") + err = getInterfaceForContainer(t, cns.SwiftPrefix+"ethWebApp") if err != nil { t.Errorf("getInterfaceForContainer failed Err:%+v", err) t.Fatal(err) @@ -327,7 +327,7 @@ func TestGetInterfaceForNetworkContainer(t *testing.T) { fmt.Println("Now calling DeleteNetworkContainer") - err = deleteNetworkAdapterWithName(t, "ethWebApp") + err = deleteNetworkAdapterWithName(t, cns.SwiftPrefix+"ethWebApp") if err != nil { t.Errorf("Deleting interface failed Err:%+v", err) t.Fatal(err) @@ -597,7 +597,7 @@ func getNetworkContainerStatus(t *testing.T, name string) error { var resp cns.GetNetworkContainerStatusResponse getReq := &cns.GetNetworkContainerStatusRequest{ - NetworkContainerid: "ethWebApp", + NetworkContainerid: name, } json.NewEncoder(&body).Encode(getReq) @@ -624,7 +624,7 @@ func getInterfaceForContainer(t *testing.T, name string) error { var resp cns.GetInterfaceForContainerResponse getReq := &cns.GetInterfaceForContainerRequest{ - NetworkContainerID: "ethWebApp", + NetworkContainerID: name, } json.NewEncoder(&body).Encode(getReq) diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 1257a5a41c..fc8a39babb 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -105,16 +105,16 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw service.Lock() defer service.Unlock() + if service.state.ContainerStatus == nil { + service.state.ContainerStatus = make(map[string]containerstatus) + } + existing, ok := service.state.ContainerStatus[req.NetworkContainerid] var hostVersion string if ok { hostVersion = existing.HostVersion } - if service.state.ContainerStatus == nil { - service.state.ContainerStatus = make(map[string]containerstatus) - } - service.state.ContainerStatus[req.NetworkContainerid] = containerstatus{ ID: req.NetworkContainerid, From 1827cdec217598e53f862179fc3fcb2db2f08a79 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 22 Jul 2020 10:22:29 -0700 Subject: [PATCH 16/17] removing swift prefix mods for mdnc --- cns/restserver/api.go | 4 ---- cns/restserver/api_test.go | 12 ++++++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/cns/restserver/api.go b/cns/restserver/api.go index dba9520639..5b43295e1a 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -775,10 +775,6 @@ func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWr returnMessage = fmt.Sprintf("[Azure CNS] Error. NetworkContainerid is empty") } - if !strings.Contains(req.NetworkContainerid, cns.SwiftPrefix) { - req.NetworkContainerid = cns.SwiftPrefix + req.NetworkContainerid - } - switch r.Method { case "POST": if req.NetworkContainerType == cns.WebApps { diff --git a/cns/restserver/api_test.go b/cns/restserver/api_test.go index fe493be673..66940e00dc 100644 --- a/cns/restserver/api_test.go +++ b/cns/restserver/api_test.go @@ -218,7 +218,7 @@ func TestCreateNetworkContainer(t *testing.T) { fmt.Println("Now calling DeleteNetworkContainer") - err = deleteNetworkAdapterWithName(t, cns.SwiftPrefix+"ethWebApp") + err = deleteNetworkAdapterWithName(t, "ethWebApp") if err != nil { t.Errorf("Deleting interface failed Err:%+v", err) t.Fatal(err) @@ -263,7 +263,7 @@ func TestGetNetworkContainerByOrchestratorContext(t *testing.T) { fmt.Println("Now calling DeleteNetworkContainer") - err = deleteNetworkAdapterWithName(t, cns.SwiftPrefix+"ethWebApp") + err = deleteNetworkAdapterWithName(t, "ethWebApp") if err != nil { t.Errorf("Deleting interface failed Err:%+v", err) t.Fatal(err) @@ -290,7 +290,7 @@ func TestGetNetworkContainerStatus(t *testing.T) { } fmt.Println("Now calling getNetworkContainerStatus") - err = getNetworkContainerStatus(t, cns.SwiftPrefix+"ethWebApp") + err = getNetworkContainerStatus(t, "ethWebApp") if err != nil { t.Errorf("getNetworkContainerStatus failed Err:%+v", err) t.Fatal(err) @@ -298,7 +298,7 @@ func TestGetNetworkContainerStatus(t *testing.T) { fmt.Println("Now calling DeleteNetworkContainer") - err = deleteNetworkAdapterWithName(t, cns.SwiftPrefix+"ethWebApp") + err = deleteNetworkAdapterWithName(t, "ethWebApp") if err != nil { t.Errorf("Deleting interface failed Err:%+v", err) t.Fatal(err) @@ -319,7 +319,7 @@ func TestGetInterfaceForNetworkContainer(t *testing.T) { } fmt.Println("Now calling getInterfaceForContainer") - err = getInterfaceForContainer(t, cns.SwiftPrefix+"ethWebApp") + err = getInterfaceForContainer(t, "ethWebApp") if err != nil { t.Errorf("getInterfaceForContainer failed Err:%+v", err) t.Fatal(err) @@ -327,7 +327,7 @@ func TestGetInterfaceForNetworkContainer(t *testing.T) { fmt.Println("Now calling DeleteNetworkContainer") - err = deleteNetworkAdapterWithName(t, cns.SwiftPrefix+"ethWebApp") + err = deleteNetworkAdapterWithName(t, "ethWebApp") if err != nil { t.Errorf("Deleting interface failed Err:%+v", err) t.Fatal(err) From d3c82e815b410c64a52f87ccea85c42ee0692369 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 23 Jul 2020 11:10:11 -0700 Subject: [PATCH 17/17] addressing tamanoha's comments --- cns/configuration/cns_config.json | 2 +- cns/configuration/configuration.go | 2 +- cns/dockerclient/dockerclient.go | 5 +++-- cns/restserver/api_test.go | 4 ++-- cns/restserver/internalapi.go | 6 +++--- cns/restserver/util.go | 4 ++-- cns/service/main.go | 12 ++++++------ common/config.go | 4 ++-- common/utils.go | 2 ++ 9 files changed, 22 insertions(+), 19 deletions(-) diff --git a/cns/configuration/cns_config.json b/cns/configuration/cns_config.json index 88d00a253f..dc4ef96233 100644 --- a/cns/configuration/cns_config.json +++ b/cns/configuration/cns_config.json @@ -10,7 +10,7 @@ }, "ManagedSettings": { "PrivateEndpoint": "", - "InfrastructureNetwork": "", + "InfrastructureNetworkID": "", "NodeID": "", "NodeSyncIntervalInSeconds": 30 }, diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index 7310baa6ce..7ca927c9e9 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -49,7 +49,7 @@ type TelemetrySettings struct { type ManagedSettings struct { PrivateEndpoint string - InfrastructureNetwork string + InfrastructureNetworkID string NodeID string NodeSyncIntervalInSeconds int } diff --git a/cns/dockerclient/dockerclient.go b/cns/dockerclient/dockerclient.go index 7fb36e92f9..cd67ed0c17 100644 --- a/cns/dockerclient/dockerclient.go +++ b/cns/dockerclient/dockerclient.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/cns/imdsclient" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/platform" ) @@ -117,7 +118,7 @@ func (dockerClient *DockerClient) CreateNetwork(networkName string, nicInfo *imd res, err := http.Post( dockerClient.connectionURL+createNetworkPath, - "application/json; charset=utf-8", + common.JsonContent, netConfigJSON) if err != nil { @@ -160,7 +161,7 @@ func (dockerClient *DockerClient) DeleteNetwork(networkName string) error { return err } - req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set(common.ContentType, common.JsonContent) client := &http.Client{} res, err := client.Do(req) if err != nil { diff --git a/cns/restserver/api_test.go b/cns/restserver/api_test.go index 66940e00dc..e313fdc1a9 100644 --- a/cns/restserver/api_test.go +++ b/cns/restserver/api_test.go @@ -78,13 +78,13 @@ const ( ) func getInterfaceInfo(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/xml") + w.Header().Set(acncommon.ContentType, "application/xml") output, _ := xml.Marshal(hostQueryResponse) w.Write(output) } func nmagentHandler(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.Header().Set(acncommon.ContentType, acncommon.JsonContent) w.WriteHeader(http.StatusOK) if strings.Contains(r.RequestURI, "networkContainers") { diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go index f910d2e476..04c188dddc 100644 --- a/cns/restserver/internalapi.go +++ b/cns/restserver/internalapi.go @@ -32,7 +32,7 @@ func (service *HTTPRestService) GetPartitionKey() (dncPartitionKey string) { func (service *HTTPRestService) SetNodeOrchestrator(r *cns.SetOrchestratorTypeRequest) { body, _ := json.Marshal(r) req, _ := http.NewRequest(http.MethodPost, "", bytes.NewBuffer(body)) - req.Header.Set("Content-Type", "application/json; charset=UTF-8") + req.Header.Set(common.ContentType, common.JsonContent) service.setOrchestratorType(httptest.NewRecorder(), req) } @@ -111,7 +111,7 @@ func (service *HTTPRestService) SyncNodeStatus(dncEP, infraVnet, nodeID string, body, _ = json.Marshal(nc) req, _ = http.NewRequest(http.MethodPost, "", bytes.NewBuffer(body)) - req.Header.Set("Content-Type", "application/json; charset=UTF-8") + req.Header.Set(common.ContentType, common.JsonContent) service.createOrUpdateNetworkContainer(w, req) if w.Result().StatusCode == http.StatusOK { var resp cns.CreateNetworkContainerResponse @@ -136,7 +136,7 @@ func (service *HTTPRestService) SyncNodeStatus(dncEP, infraVnet, nodeID string, req, err = http.NewRequest(http.MethodPost, "", &body) if err == nil { - req.Header.Set("Content-Type", "application/json; charset=UTF-8") + req.Header.Set(common.JsonContent, common.JsonContent) service.deleteNetworkContainer(httptest.NewRecorder(), req) } else { logger.Errorf("[Azure-CNS] Failed to delete NC request to sync state: %s", err.Error()) diff --git a/cns/restserver/util.go b/cns/restserver/util.go index fc8a39babb..985bc3dd76 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -218,7 +218,7 @@ func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkCo } else { var ( dncEP = service.GetOption(acn.OptPrivateEndpoint).(string) - infraVnet = service.GetOption(acn.OptInfrastructureNetwork).(string) + infraVnet = service.GetOption(acn.OptInfrastructureNetworkID).(string) nodeID = service.GetOption(acn.OptNodeID).(string) ) @@ -344,7 +344,7 @@ func (service *HTTPRestService) attachOrDetachHelper(req cns.ConfigureContainerN } else { var ( dncEP = service.GetOption(acn.OptPrivateEndpoint).(string) - infraVnet = service.GetOption(acn.OptInfrastructureNetwork).(string) + infraVnet = service.GetOption(acn.OptInfrastructureNetworkID).(string) nodeID = service.GetOption(acn.OptNodeID).(string) ) diff --git a/cns/service/main.go b/cns/service/main.go index 7d4c9e049d..4caaec20ae 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -195,8 +195,8 @@ var args = acn.ArgumentList{ DefaultValue: "", }, { - Name: acn.OptInfrastructureNetwork, - Shorthand: acn.OptInfrastructureNetworkAlias, + Name: acn.OptInfrastructureNetworkID, + Shorthand: acn.OptInfrastructureNetworkIDAlias, Description: "Set infrastructure network ID", Type: "string", DefaultValue: "", @@ -285,7 +285,7 @@ func main() { httpResponseHeaderTimeout := acn.GetArg(acn.OptHttpResponseHeaderTimeout).(int) storeFileLocation := acn.GetArg(acn.OptStoreFileLocation).(string) privateEndpoint := acn.GetArg(acn.OptPrivateEndpoint).(string) - infravnet := acn.GetArg(acn.OptInfrastructureNetwork).(string) + infravnet := acn.GetArg(acn.OptInfrastructureNetworkID).(string) nodeID := acn.GetArg(acn.OptNodeID).(string) if vers { @@ -321,7 +321,7 @@ func main() { if cnsconfig.ChannelMode == cns.Managed { privateEndpoint = cnsconfig.ManagedSettings.PrivateEndpoint - infravnet = cnsconfig.ManagedSettings.InfrastructureNetwork + infravnet = cnsconfig.ManagedSettings.InfrastructureNetworkID nodeID = cnsconfig.ManagedSettings.NodeID } else if acn.GetArg(acn.OptManaged).(bool) { config.ChannelMode = cns.Managed @@ -404,7 +404,7 @@ func main() { // If CNS is running on managed DNC mode if config.ChannelMode == cns.Managed { if privateEndpoint == "" || infravnet == "" || nodeID == "" { - logger.Errorf("[Azure CNS] Missing required values to run in managed mode: PrivateEndpoint: %s InfrastructureNetwork: %s NodeID: %s", + logger.Errorf("[Azure CNS] Missing required values to run in managed mode: PrivateEndpoint: %s InfrastructureNetworkID: %s NodeID: %s", privateEndpoint, infravnet, nodeID) @@ -412,7 +412,7 @@ func main() { } httpRestService.SetOption(acn.OptPrivateEndpoint, privateEndpoint) - httpRestService.SetOption(acn.OptInfrastructureNetwork, infravnet) + httpRestService.SetOption(acn.OptInfrastructureNetworkID, infravnet) httpRestService.SetOption(acn.OptNodeID, nodeID) registerNode(httpRestService, privateEndpoint, infravnet, nodeID) diff --git a/common/config.go b/common/config.go index aaa8de28fe..1239a04e96 100644 --- a/common/config.go +++ b/common/config.go @@ -104,8 +104,8 @@ const ( OptPrivateEndpointAlias = "pe" // Infrastructure Network - OptInfrastructureNetwork = "infra-vnet" - OptInfrastructureNetworkAlias = "iv" + OptInfrastructureNetworkID = "infra-vnet" + OptInfrastructureNetworkIDAlias = "iv" // Node ID/Name OptNodeID = "node-id" diff --git a/common/utils.go b/common/utils.go index 1e4b315dfb..a1e77df6d4 100644 --- a/common/utils.go +++ b/common/utils.go @@ -29,6 +29,8 @@ const ( RegisterNodeURLFmt = "%s/%s/node/%s/cores/%d%s" SyncNodeNetworkContainersURLFmt = "%s/%s/node/%s%s" FiveSeconds = 5 * time.Second + JsonContent = "application/json; charset=UTF-8" + ContentType = "Content-Type" ) // XmlDocument - Azure host agent XML document format.