From dbab440666705212a815adf8a8018c46af82819c Mon Sep 17 00:00:00 2001 From: Jaeryn Chu Date: Mon, 10 Dec 2018 11:33:24 -0800 Subject: [PATCH 1/2] Adding telemetry capability for CNS. --- cns/NetworkContainerContract.go | 1 + cns/restserver/restserver.go | 69 +++++++++++++++----------- cns/service/main.go | 24 +++++++++ common/config.go | 4 ++ log/logger.go | 16 +++++- platform/os_linux.go | 3 ++ platform/os_windows.go | 7 ++- telemetry/cnstelemetry.go | 87 +++++++++++++++++++++++++++++++++ telemetry/telemetry.go | 14 ++++++ telemetry/telemetrybuffer.go | 5 ++ 10 files changed, 198 insertions(+), 32 deletions(-) create mode 100644 telemetry/cnstelemetry.go diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 005e2263c4..4a27bf697e 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -81,6 +81,7 @@ type Route struct { // SetOrchestratorTypeRequest specifies the orchestrator type for the node. type SetOrchestratorTypeRequest struct { OrchestratorType string + DncPartitionKey string } // CreateNetworkContainerResponse specifies response of creating a network container. diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index f517403c69..b909a1ce31 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -29,8 +29,8 @@ const ( swiftAPIVersion = "1" ) -// httpRestService represents http listener for CNS - Container Networking Service. -type httpRestService struct { +// HTTPRestService represents http listener for CNS - Container Networking Service. +type HTTPRestService struct { *cns.Service dockerClient *dockerclient.DockerClient imdsClient *imdsclient.ImdsClient @@ -40,6 +40,7 @@ type httpRestService struct { store store.KeyValueStore state *httpRestServiceState lock sync.Mutex + dncPartitionKey string } // containerstatus is used to save status of an existing container @@ -97,7 +98,7 @@ func NewHTTPRestService(config *common.ServiceConfig) (HTTPService, error) { serviceState := &httpRestServiceState{} serviceState.Networks = make(map[string]*networkInfo) - return &httpRestService{ + return &HTTPRestService{ Service: service, store: service.Service.Store, dockerClient: dc, @@ -111,7 +112,7 @@ func NewHTTPRestService(config *common.ServiceConfig) (HTTPService, error) { } // Start starts the CNS listener. -func (service *httpRestService) Start(config *common.ServiceConfig) error { +func (service *HTTPRestService) Start(config *common.ServiceConfig) error { err := service.Initialize(config) if err != nil { @@ -170,13 +171,21 @@ func (service *httpRestService) Start(config *common.ServiceConfig) error { } // Stop stops the CNS. -func (service *httpRestService) Stop() { +func (service *HTTPRestService) Stop() { service.Uninitialize() log.Printf("[Azure CNS] Service stopped.") } +// Get dnc/service partition key +func (service *HTTPRestService) GetPartitionKey() (dncPartitionKey string) { + service.lock.Lock() + dncPartitionKey = service.dncPartitionKey + service.lock.Unlock() + return +} + // Handles requests to set the environment type. -func (service *httpRestService) setEnvironment(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) setEnvironment(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] setEnvironment") var req cns.SetEnvironmentRequest @@ -204,7 +213,7 @@ func (service *httpRestService) setEnvironment(w http.ResponseWriter, r *http.Re } // Handles CreateNetwork requests. -func (service *httpRestService) createNetwork(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) createNetwork(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] createNetwork") var err error @@ -308,7 +317,7 @@ func (service *httpRestService) createNetwork(w http.ResponseWriter, r *http.Req } // Handles DeleteNetwork requests. -func (service *httpRestService) deleteNetwork(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) deleteNetwork(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] deleteNetwork") var req cns.DeleteNetworkRequest @@ -364,7 +373,7 @@ func (service *httpRestService) deleteNetwork(w http.ResponseWriter, r *http.Req } // Handles ip reservation requests. -func (service *httpRestService) reserveIPAddress(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) reserveIPAddress(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] reserveIPAddress") var req cns.ReserveIPAddressRequest @@ -442,7 +451,7 @@ func (service *httpRestService) reserveIPAddress(w http.ResponseWriter, r *http. } // Handles release ip reservation requests. -func (service *httpRestService) releaseIPAddress(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) releaseIPAddress(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] releaseIPAddress") var req cns.ReleaseIPAddressRequest @@ -508,7 +517,7 @@ func (service *httpRestService) releaseIPAddress(w http.ResponseWriter, r *http. } // Retrieves the host local ip address. Containers can talk to host using this IP address. -func (service *httpRestService) getHostLocalIP(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) getHostLocalIP(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] getHostLocalIP") log.Request(service.Name, "getHostLocalIP", nil) @@ -560,7 +569,7 @@ func (service *httpRestService) getHostLocalIP(w http.ResponseWriter, r *http.Re } // Handles ip address utilization requests. -func (service *httpRestService) getIPAddressUtilization(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) getIPAddressUtilization(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] getIPAddressUtilization") log.Request(service.Name, "getIPAddressUtilization", nil) @@ -626,7 +635,7 @@ func (service *httpRestService) getIPAddressUtilization(w http.ResponseWriter, r } // Handles retrieval of ip addresses that are available to be reserved from ipam driver. -func (service *httpRestService) getAvailableIPAddresses(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) getAvailableIPAddresses(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] getAvailableIPAddresses") log.Request(service.Name, "getAvailableIPAddresses", nil) @@ -643,7 +652,7 @@ func (service *httpRestService) getAvailableIPAddresses(w http.ResponseWriter, r } // Handles retrieval of reserved ip addresses from ipam driver. -func (service *httpRestService) getReservedIPAddresses(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) getReservedIPAddresses(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] getReservedIPAddresses") log.Request(service.Name, "getReservedIPAddresses", nil) @@ -660,7 +669,7 @@ func (service *httpRestService) getReservedIPAddresses(w http.ResponseWriter, r } // Handles retrieval of ghost ip addresses from ipam driver. -func (service *httpRestService) getUnhealthyIPAddresses(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) getUnhealthyIPAddresses(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] getUnhealthyIPAddresses") log.Request(service.Name, "getUnhealthyIPAddresses", nil) @@ -724,7 +733,7 @@ func (service *httpRestService) getUnhealthyIPAddresses(w http.ResponseWriter, r } // getAllIPAddresses retrieves all ip addresses from ipam driver. -func (service *httpRestService) getAllIPAddresses(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) getAllIPAddresses(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] getAllIPAddresses") log.Request(service.Name, "getAllIPAddresses", nil) @@ -741,7 +750,7 @@ func (service *httpRestService) getAllIPAddresses(w http.ResponseWriter, r *http } // Handles health report requests. -func (service *httpRestService) getHealthReport(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) getHealthReport(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] getHealthReport") log.Request(service.Name, "getHealthReport", nil) @@ -757,7 +766,7 @@ func (service *httpRestService) getHealthReport(w http.ResponseWriter, r *http.R } // saveState writes CNS state to persistent store. -func (service *httpRestService) saveState() error { +func (service *HTTPRestService) saveState() error { log.Printf("[Azure CNS] saveState") // Skip if a store is not provided. @@ -779,7 +788,7 @@ func (service *httpRestService) saveState() error { } // restoreState restores CNS state from persistent store. -func (service *httpRestService) restoreState() error { +func (service *HTTPRestService) restoreState() error { log.Printf("[Azure CNS] restoreState") // Skip if a store is not provided. @@ -805,7 +814,7 @@ func (service *httpRestService) restoreState() error { return nil } -func (service *httpRestService) setOrchestratorType(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) setOrchestratorType(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] setOrchestratorType") var req cns.SetOrchestratorTypeRequest @@ -819,6 +828,8 @@ func (service *httpRestService) setOrchestratorType(w http.ResponseWriter, r *ht service.lock.Lock() + service.dncPartitionKey = req.DncPartitionKey + switch req.OrchestratorType { case cns.ServiceFabric, cns.Kubernetes, cns.WebApps: service.state.OrchestratorType = req.OrchestratorType @@ -839,7 +850,7 @@ func (service *httpRestService) setOrchestratorType(w http.ResponseWriter, r *ht log.Response(service.Name, resp, err) } -func (service *httpRestService) saveNetworkContainerGoalState(req cns.CreateNetworkContainerRequest) (int, string) { +func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetworkContainerRequest) (int, string) { // we don't want to overwrite what other calls may have written service.lock.Lock() defer service.lock.Unlock() @@ -890,7 +901,7 @@ func (service *httpRestService) saveNetworkContainerGoalState(req cns.CreateNetw return 0, "" } -func (service *httpRestService) createOrUpdateNetworkContainer(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] createOrUpdateNetworkContainer") var req cns.CreateNetworkContainerRequest @@ -945,7 +956,7 @@ func (service *httpRestService) createOrUpdateNetworkContainer(w http.ResponseWr log.Response(service.Name, reserveResp, err) } -func (service *httpRestService) getNetworkContainerByID(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) getNetworkContainerByID(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] getNetworkContainerByID") var req cns.GetNetworkContainerRequest @@ -968,7 +979,7 @@ func (service *httpRestService) getNetworkContainerByID(w http.ResponseWriter, r log.Response(service.Name, reserveResp, err) } -func (service *httpRestService) getNetworkContainerResponse(req cns.GetNetworkContainerRequest) cns.GetNetworkContainerResponse { +func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkContainerRequest) cns.GetNetworkContainerResponse { var containerID string var getNetworkContainerResponse cns.GetNetworkContainerResponse @@ -1017,7 +1028,7 @@ func (service *httpRestService) getNetworkContainerResponse(req cns.GetNetworkCo return getNetworkContainerResponse } -func (service *httpRestService) getNetworkContainerByOrchestratorContext(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) getNetworkContainerByOrchestratorContext(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] getNetworkContainerByOrchestratorContext") var req cns.GetNetworkContainerRequest @@ -1034,7 +1045,7 @@ func (service *httpRestService) getNetworkContainerByOrchestratorContext(w http. log.Response(service.Name, getNetworkContainerResponse, err) } -func (service *httpRestService) deleteNetworkContainer(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) deleteNetworkContainer(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] deleteNetworkContainer") var req cns.DeleteNetworkContainerRequest @@ -1109,7 +1120,7 @@ func (service *httpRestService) deleteNetworkContainer(w http.ResponseWriter, r log.Response(service.Name, reserveResp, err) } -func (service *httpRestService) getNetworkContainerStatus(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) getNetworkContainerStatus(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] getNetworkContainerStatus") var req cns.GetNetworkContainerStatusRequest @@ -1172,7 +1183,7 @@ func (service *httpRestService) getNetworkContainerStatus(w http.ResponseWriter, log.Response(service.Name, networkContainerStatusReponse, err) } -func (service *httpRestService) getInterfaceForContainer(w http.ResponseWriter, r *http.Request) { +func (service *HTTPRestService) getInterfaceForContainer(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] getInterfaceForContainer") var req cns.GetInterfaceForContainerRequest @@ -1227,7 +1238,7 @@ func (service *httpRestService) getInterfaceForContainer(w http.ResponseWriter, } // restoreNetworkState restores Network state that existed before reboot. -func (service *httpRestService) restoreNetworkState() error { +func (service *HTTPRestService) restoreNetworkState() error { log.Printf("[Azure CNS] Enter Restoring Network State") if service.store == nil { diff --git a/cns/service/main.go b/cns/service/main.go index 0b93318082..aaa001fc03 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -9,6 +9,8 @@ import ( "os/signal" "syscall" + "github.com/Azure/azure-container-networking/telemetry" + "github.com/Azure/azure-container-networking/cnm/ipam" "github.com/Azure/azure-container-networking/cnm/network" "github.com/Azure/azure-container-networking/cns/common" @@ -28,6 +30,10 @@ const ( // Version is populated by make during build. var version string +// Reports channel +var reports = make(chan interface{}) +var telemetryStopProcessing = make(chan bool) + // Command line arguments for CNS. var args = acn.ArgumentList{ { @@ -116,6 +122,13 @@ var args = acn.ArgumentList{ Type: "bool", DefaultValue: false, }, + { + Name: acn.OptReportToHostInterval, + Shorthand: acn.OptReportToHostIntervalAlias, + Description: "Set interval in ms to report to host", + Type: "int", + DefaultValue: 60000, + }, } // Prints description and version information. @@ -140,6 +153,7 @@ func main() { ipamQueryInterval, _ := acn.GetArg(acn.OptIpamQueryInterval).(int) stopcnm = acn.GetArg(acn.OptStopAzureVnet).(bool) vers := acn.GetArg(acn.OptVersion).(bool) + reportToHostInterval := acn.GetArg(acn.OptReportToHostInterval).(int) if vers { printVersion() @@ -168,6 +182,10 @@ func main() { return } + if logger := log.GetStd(); logger != nil { + logger.SetChannel(reports) + } + // Log platform information. log.Printf("Running on %v", platform.GetOSInfo()) @@ -196,6 +214,10 @@ func main() { // Start CNS. if httpRestService != nil { + go telemetry.SendCnsTelemetry(reportToHostInterval, + reports, + httpRestService.(*restserver.HTTPRestService), + telemetryStopProcessing) err = httpRestService.Start(&config) if err != nil { log.Printf("Failed to start CNS, err:%v.\n", err) @@ -269,6 +291,8 @@ func main() { httpRestService.Stop() } + telemetryStopProcessing <- true + if !stopcnm { if netPlugin != nil { netPlugin.Stop() diff --git a/common/config.go b/common/config.go index c986cc990f..89cb15d8be 100644 --- a/common/config.go +++ b/common/config.go @@ -48,6 +48,10 @@ const ( OptStopAzureVnet = "stop-azure-cnm" OptStopAzureVnetAlias = "stopcnm" + // Interval to send reports to host + OptReportToHostInterval = "report-interval" + OptReportToHostIntervalAlias = "hostinterval" + // Version. OptVersion = "version" OptVersionAlias = "v" diff --git a/log/logger.go b/log/logger.go index e5176d6688..78e38d4e78 100644 --- a/log/logger.go +++ b/log/logger.go @@ -53,6 +53,7 @@ type Logger struct { maxFileCount int callCount int directory string + reports chan interface{} mutex *sync.Mutex } @@ -88,6 +89,11 @@ func (logger *Logger) SetLogFileLimits(maxFileSize int, maxFileCount int) { logger.maxFileCount = maxFileCount } +// SetChannel sets the channel for error message reports. +func (logger *Logger) SetChannel(reports chan interface{}) { + logger.reports = reports +} + // Close closes the log stream. func (logger *Logger) Close() { if logger.out != nil { @@ -182,8 +188,8 @@ func (logger *Logger) logf(format string, args ...interface{}) { if logger.callCount%rotationCheckFrq == 0 { logger.rotate() } - logger.callCount++ + logger.callCount++ logger.l.Printf(format, args...) } @@ -204,3 +210,11 @@ func (logger *Logger) Debugf(format string, args ...interface{}) { logger.mutex.Unlock() } } + +// Errorf logs a formatted string at info level and sends the string to TelemetryBuffer. +func (logger *Logger) Errorf(format string, args ...interface{}) { + logger.Printf(format, args...) + go func() { + logger.reports <- fmt.Sprintf(format, args...) + }() +} diff --git a/platform/os_linux.go b/platform/os_linux.go index 44f69bc7e8..ebf4239bb6 100644 --- a/platform/os_linux.go +++ b/platform/os_linux.go @@ -21,6 +21,9 @@ const ( // CNIRuntimePath is the path where CNI state files are stored. CNIRuntimePath = "/var/run/" + // CNSRuntimePath is the path where CNS state files are stored. + CNSRuntimePath = "/var/run/" + // NPMRuntimePath is the path where NPM logging files are stored. NPMRuntimePath = "/var/run/" diff --git a/platform/os_windows.go b/platform/os_windows.go index 81439257a0..00e1d97d1e 100644 --- a/platform/os_windows.go +++ b/platform/os_windows.go @@ -12,13 +12,16 @@ const ( // CNMRuntimePath is the path where CNM state files are stored. CNMRuntimePath = "" - // CNIRuntimePath is the path where CNM state files are stored. + // CNIRuntimePath is the path where CNI state files are stored. CNIRuntimePath = "" + // CNSRuntimePath is the path where CNS state files are stored. + CNSRuntimePath = "" + // NPMRuntimePath is the path where NPM state files are stored. NPMRuntimePath = "" - // DNCRuntimePath is the path where NPM state files are stored. + // DNCRuntimePath is the path where DNC state files are stored. DNCRuntimePath = "" ) diff --git a/telemetry/cnstelemetry.go b/telemetry/cnstelemetry.go new file mode 100644 index 0000000000..79780d45b7 --- /dev/null +++ b/telemetry/cnstelemetry.go @@ -0,0 +1,87 @@ +// Copyright 2018 Microsoft. All rights reserved. +// MIT License + +package telemetry + +import ( + "fmt" + "reflect" + "time" + + "github.com/Azure/azure-container-networking/cns/restserver" + "github.com/Azure/azure-container-networking/platform" + "github.com/google/uuid" +) + +const ( + // CNSTelemetryFile - telemetry file path. + CNSTelemetryFile = platform.CNSRuntimePath + "AzureCNSTelemetry.json" +) + +// SendCnsTelemetry - handles cns telemetry reports +func SendCnsTelemetry(interval int, reports chan interface{}, service *restserver.HTTPRestService, telemetryStopProcessing chan bool) { + retrieveMetadata := true + +CONNECT: + telemetryBuffer, err := NewTelemetryBuffer() + if err == nil { + go telemetryBuffer.Start(time.Duration(interval)) + + heartbeat := time.NewTicker(time.Minute * 30).C + reportMgr := ReportManager{ + ContentType: ContentType, + Report: &CNSReport{}, + } + + reportMgr.GetReportState(CNSTelemetryFile) + reportMgr.GetKernelVersion() + + for { + // Try to retrieve metadata until successful + if retrieveMetadata { + if err := reportMgr.GetHostMetadata(); err != nil { + reports <- CNSReport{EventMessage: fmt.Sprintf("Failed to retrieve host metadata with error: %s", err.Error())} + } else { + retrieveMetadata = false + } + } + + // Try to set partition key from DNC + if reportMgr.Report.(CNSReport).DncPartitionKey == "" { + reflect.ValueOf(reportMgr.Report).Elem().FieldByName("DncPartitionKey").SetString(service.GetPartitionKey()) + } + + select { + case <-heartbeat: + reflect.ValueOf(reportMgr.Report).Elem().FieldByName("EventMessage").SetString("Heartbeat") + case msg := <-reports: + reflect.ValueOf(reportMgr.Report).Elem().FieldByName("EventMessage").SetString(msg.(string)) + case <-telemetryStopProcessing: + telemetryBuffer.Cancel() + return + } + + reflect.ValueOf(reportMgr.Report).Elem().FieldByName("Timestamp").SetString(time.Now().UTC().String()) + if id, err := uuid.NewUUID(); err == nil { + reflect.ValueOf(reportMgr.Report).Elem().FieldByName("UUID").SetString(id.String()) + } + + if !reportMgr.GetReportState(CNSTelemetryFile) { + reportMgr.SetReportState(CNSTelemetryFile) + } + + report, err := reportMgr.ReportToBytes() + if err == nil { + // If write fails, try to re-establish connections as server/client + if _, err = telemetryBuffer.Write(report); err != nil { + telemetryBuffer.Cancel() + goto CONNECT + } + } + } + } else { + reports <- CNSReport{EventMessage: "Failed to establish telemetry buffer connection."} + time.Sleep(time.Minute * 1) + goto CONNECT + } +} diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index ab25059007..4bdb56f6c4 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -120,6 +120,19 @@ type CNIReport struct { Metadata Metadata `json:"compute"` } +// Azure CNS Telemetry Report structure. +type CNSReport struct { + IsNewInstance bool + CPUUsage string + MemoryUsage string + Processes string + EventMessage string + DncPartitionKey string + Timestamp string + UUID string + Metadata Metadata `json:"compute"` +} + // ClusterState contains the current kubernetes cluster state. type ClusterState struct { PodCount int @@ -476,6 +489,7 @@ func (reportMgr *ReportManager) ReportToBytes() (report []byte, err error) { case *CNIReport: case *NPMReport: case *DNCReport: + case *CNSReport: default: err = fmt.Errorf("[Telemetry] Invalid report type") } diff --git a/telemetry/telemetrybuffer.go b/telemetry/telemetrybuffer.go index d807b65d28..122485e213 100644 --- a/telemetry/telemetrybuffer.go +++ b/telemetry/telemetrybuffer.go @@ -45,6 +45,7 @@ type Payload struct { DNCReports []DNCReport CNIReports []CNIReport NPMReports []NPMReport + CNSReports []CNSReport } // NewTelemetryBuffer - create a new TelemetryBuffer @@ -82,6 +83,10 @@ func NewTelemetryBuffer() (*TelemetryBuffer, error) { var dncReport DNCReport json.Unmarshal([]byte(reportStr), &dncReport) tb.data <- dncReport + } else if _, ok := tmp["DncPartitionKey"]; ok { + var cnsReport CNSReport + json.Unmarshal([]byte(reportStr), &cnsReport) + tb.data <- cnsReport } } } From 8bee17544f1d621efd79c0590c3618e2fe84396f Mon Sep 17 00:00:00 2001 From: Jaeryn Chu Date: Tue, 11 Dec 2018 14:40:59 -0800 Subject: [PATCH 2/2] Adding CNS telemetry. --- cnm/ipam/ipam.go | 16 +++++---- cnm/network/network.go | 18 +++++----- cnm/plugin.go | 4 +-- cns/cnsclient/cnsclient.go | 10 +++--- cns/dockerclient/dockerclient.go | 4 +-- cns/restserver/api.go | 39 +++++++++++++++++++++ cns/restserver/restserver.go | 58 ++++++++++++++------------------ cns/routes/routes_windows.go | 2 +- cns/service.go | 2 +- cns/service/main.go | 20 +++++------ log/logger.go | 8 ++--- log/stdapi.go | 8 +++-- telemetry/cnstelemetry.go | 12 +++++-- telemetry/telemetry.go | 1 + telemetry/telemetrybuffer.go | 13 +++++-- 15 files changed, 136 insertions(+), 79 deletions(-) diff --git a/cnm/ipam/ipam.go b/cnm/ipam/ipam.go index 8162a9ae31..5aa7f00247 100644 --- a/cnm/ipam/ipam.go +++ b/cnm/ipam/ipam.go @@ -19,6 +19,8 @@ const ( // Plugin capabilities reported to libnetwork. requiresMACAddress = false requiresRequestReplay = false + returnCode = 0 + returnStr = "Success" ) // IpamPlugin represents a CNM (libnetwork) IPAM plugin. @@ -118,7 +120,7 @@ func (plugin *ipamPlugin) getCapabilities(w http.ResponseWriter, r *http.Request err := plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles GetDefaultAddressSpaces requests. @@ -135,7 +137,7 @@ func (plugin *ipamPlugin) getDefaultAddressSpaces(w http.ResponseWriter, r *http err := plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles RequestPool requests. @@ -163,7 +165,7 @@ func (plugin *ipamPlugin) requestPool(w http.ResponseWriter, r *http.Request) { err = plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles ReleasePool requests. @@ -195,7 +197,7 @@ func (plugin *ipamPlugin) releasePool(w http.ResponseWriter, r *http.Request) { err = plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles GetPoolInfo requests. @@ -234,7 +236,7 @@ func (plugin *ipamPlugin) getPoolInfo(w http.ResponseWriter, r *http.Request) { err = plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles RequestAddress requests. @@ -275,7 +277,7 @@ func (plugin *ipamPlugin) requestAddress(w http.ResponseWriter, r *http.Request) err = plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles ReleaseAddress requests. @@ -307,5 +309,5 @@ func (plugin *ipamPlugin) releaseAddress(w http.ResponseWriter, r *http.Request) err = plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } diff --git a/cnm/network/network.go b/cnm/network/network.go index f51c86fe3a..860551c1ad 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -23,6 +23,8 @@ const ( // Prefix for container network interface names. containerInterfacePrefix = "eth" + returnCode = 0 + returnStr = "Success" ) // NetPlugin represents a CNM (libnetwork) network plugin. @@ -121,7 +123,7 @@ func (plugin *netPlugin) getCapabilities(w http.ResponseWriter, r *http.Request) resp := getCapabilitiesResponse{Scope: plugin.scope} err := plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles CreateNetwork requests. @@ -175,7 +177,7 @@ func (plugin *netPlugin) createNetwork(w http.ResponseWriter, r *http.Request) { resp := createNetworkResponse{} err = plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles DeleteNetwork requests. @@ -200,7 +202,7 @@ func (plugin *netPlugin) deleteNetwork(w http.ResponseWriter, r *http.Request) { resp := deleteNetworkResponse{} err = plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles CreateEndpoint requests. @@ -244,7 +246,7 @@ func (plugin *netPlugin) createEndpoint(w http.ResponseWriter, r *http.Request) err = plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles DeleteEndpoint requests. @@ -269,7 +271,7 @@ func (plugin *netPlugin) deleteEndpoint(w http.ResponseWriter, r *http.Request) resp := deleteEndpointResponse{} err = plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles Join requests. @@ -303,7 +305,7 @@ func (plugin *netPlugin) join(w http.ResponseWriter, r *http.Request) { err = plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles Leave requests. @@ -328,7 +330,7 @@ func (plugin *netPlugin) leave(w http.ResponseWriter, r *http.Request) { resp := leaveResponse{} err = plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } // Handles EndpointOperInfo requests. @@ -353,5 +355,5 @@ func (plugin *netPlugin) endpointOperInfo(w http.ResponseWriter, r *http.Request resp := endpointOperInfoResponse{Value: epInfo.Data} err = plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, returnCode, returnStr, err) } diff --git a/cnm/plugin.go b/cnm/plugin.go index dd89cc7c07..7b08981062 100644 --- a/cnm/plugin.go +++ b/cnm/plugin.go @@ -124,7 +124,7 @@ func (plugin *Plugin) activate(w http.ResponseWriter, r *http.Request) { resp := ActivateResponse{Implements: plugin.Listener.GetEndpoints()} err := plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, 0, "Success", err) } // SendErrorResponse sends and logs an error response. @@ -132,5 +132,5 @@ func (plugin *Plugin) SendErrorResponse(w http.ResponseWriter, errMsg error) { resp := errorResponse{errMsg.Error()} err := plugin.Listener.Encode(w, &resp) - log.Response(plugin.Name, &resp, err) + log.Response(plugin.Name, &resp, 0, "Success", err) } diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index 2a24c50134..aef2749c6d 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -44,13 +44,13 @@ func (cnsClient *CNSClient) GetNetworkConfiguration(orchestratorContext []byte) err := json.NewEncoder(&body).Encode(payload) if err != nil { - log.Printf("encoding json failed with %v", err) + log.Errorf("encoding json failed with %v", err) return nil, err } res, err := httpc.Post(url, "application/json", &body) if err != nil { - log.Printf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) return nil, err } @@ -58,7 +58,7 @@ func (cnsClient *CNSClient) GetNetworkConfiguration(orchestratorContext []byte) if res.StatusCode != http.StatusOK { errMsg := fmt.Sprintf("[Azure CNSClient] GetNetworkConfiguration invalid http status code: %v", res.StatusCode) - log.Printf(errMsg) + log.Errorf(errMsg) return nil, fmt.Errorf(errMsg) } @@ -66,12 +66,12 @@ func (cnsClient *CNSClient) GetNetworkConfiguration(orchestratorContext []byte) err = json.NewDecoder(res.Body).Decode(&resp) if err != nil { - log.Printf("[Azure CNSClient] Error received while parsing GetNetworkConfiguration response resp:%v err:%v", res.Body, err.Error()) + log.Errorf("[Azure CNSClient] Error received while parsing GetNetworkConfiguration response resp:%v err:%v", res.Body, err.Error()) return nil, err } if resp.Response.ReturnCode != 0 { - log.Printf("[Azure CNSClient] GetNetworkConfiguration received error response :%v", resp.Response.Message) + log.Errorf("[Azure CNSClient] GetNetworkConfiguration received error response :%v", resp.Response.Message) return nil, fmt.Errorf(resp.Response.Message) } diff --git a/cns/dockerclient/dockerclient.go b/cns/dockerclient/dockerclient.go index eba45b5780..b3be960878 100644 --- a/cns/dockerclient/dockerclient.go +++ b/cns/dockerclient/dockerclient.go @@ -9,9 +9,9 @@ import ( "fmt" "net/http" - "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/cns/imdsclient" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/platform" ) const ( @@ -51,7 +51,7 @@ func (dockerClient *DockerClient) NetworkExists(networkName string) error { dockerClient.connectionURL + inspectNetworkPath + networkName) if err != nil { - log.Printf("[Azure CNS] Error received from http Post for docker network inspect %v %v", networkName, err.Error()) + log.Errorf("[Azure CNS] Error received from http Post for docker network inspect %v %v", networkName, err.Error()) return err } diff --git a/cns/restserver/api.go b/cns/restserver/api.go index 6db8f6f5f5..35b82c9534 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -22,3 +22,42 @@ const ( UnsupportedOrchestratorType = 19 UnexpectedError = 99 ) + +func ReturnCodeToString(returnCode int) (s string) { + switch returnCode { + case Success: + s = "Success" + case UnsupportedNetworkType: + s = "UnsupportedNetworkType" + case InvalidParameter: + s = "InvalidParameter" + case UnreachableHost: + s = "UnreachableHost" + case ReservationNotFound: + s = "ReservationNotFound" + case MalformedSubnet: + s = "MalformedSubnet" + case UnreachableDockerDaemon: + s = "UnreachableDockerDaemon" + case UnspecifiedNetworkName: + s = "UnspecifiedNetworkName" + case NotFound: + s = "NotFound" + case AddressUnavailable: + s = "AddressUnavailable" + case NetworkContainerNotSpecified: + s = "NetworkContainerNotSpecified" + case CallToHostFailed: + s = "CallToHostFailed" + case UnknownContainerID: + s = "UnknownContainerID" + case UnsupportedOrchestratorType: + s = "UnsupportedOrchestratorType" + case UnexpectedError: + s = "UnexpectedError" + default: + s = "UnknownError" + } + + return +} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index b909a1ce31..977b38f0c5 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -116,19 +116,19 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { err := service.Initialize(config) if err != nil { - log.Printf("[Azure CNS] Failed to initialize base service, err:%v.", err) + log.Errorf("[Azure CNS] Failed to initialize base service, err:%v.", err) return err } err = service.restoreState() if err != nil { - log.Printf("[Azure CNS] Failed to restore service state, err:%v.", err) + log.Errorf("[Azure CNS] Failed to restore service state, err:%v.", err) return err } err = service.restoreNetworkState() if err != nil { - log.Printf("[Azure CNS] Failed to restore network state, err:%v.", err) + log.Errorf("[Azure CNS] Failed to restore network state, err:%v.", err) return err } @@ -209,7 +209,7 @@ func (service *HTTPRestService) setEnvironment(w http.ResponseWriter, r *http.Re resp := &cns.Response{ReturnCode: 0} err = service.Listener.Encode(w, &resp) - log.Response(service.Name, resp, err) + log.Response(service.Name, resp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // Handles CreateNetwork requests. @@ -313,7 +313,7 @@ func (service *HTTPRestService) createNetwork(w http.ResponseWriter, r *http.Req service.saveState() } - log.Response(service.Name, resp, err) + log.Response(service.Name, resp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // Handles DeleteNetwork requests. @@ -369,7 +369,7 @@ func (service *HTTPRestService) deleteNetwork(w http.ResponseWriter, r *http.Req service.saveState() } - log.Response(service.Name, resp, err) + log.Response(service.Name, resp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // Handles ip reservation requests. @@ -444,10 +444,10 @@ func (service *HTTPRestService) reserveIPAddress(w http.ResponseWriter, r *http. ReturnCode: returnCode, Message: returnMessage, } + reserveResp := &cns.ReserveIPAddressResponse{Response: resp, IPAddress: address} err = service.Listener.Encode(w, &reserveResp) - - log.Response(service.Name, reserveResp, err) + log.Response(service.Name, reserveResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // Handles release ip reservation requests. @@ -512,8 +512,7 @@ func (service *HTTPRestService) releaseIPAddress(w http.ResponseWriter, r *http. } err = service.Listener.Encode(w, &resp) - - log.Response(service.Name, resp, err) + log.Response(service.Name, resp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // Retrieves the host local ip address. Containers can talk to host using this IP address. @@ -565,7 +564,7 @@ func (service *HTTPRestService) getHostLocalIP(w http.ResponseWriter, r *http.Re err := service.Listener.Encode(w, &hostLocalIPResponse) - log.Response(service.Name, hostLocalIPResponse, err) + log.Response(service.Name, hostLocalIPResponse, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // Handles ip address utilization requests. @@ -630,8 +629,7 @@ func (service *HTTPRestService) getIPAddressUtilization(w http.ResponseWriter, r } err := service.Listener.Encode(w, &utilResponse) - - log.Response(service.Name, utilResponse, err) + log.Response(service.Name, utilResponse, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // Handles retrieval of ip addresses that are available to be reserved from ipam driver. @@ -648,7 +646,7 @@ func (service *HTTPRestService) getAvailableIPAddresses(w http.ResponseWriter, r ipResp := &cns.GetIPAddressesResponse{Response: resp} err := service.Listener.Encode(w, &ipResp) - log.Response(service.Name, ipResp, err) + log.Response(service.Name, ipResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // Handles retrieval of reserved ip addresses from ipam driver. @@ -665,7 +663,7 @@ func (service *HTTPRestService) getReservedIPAddresses(w http.ResponseWriter, r ipResp := &cns.GetIPAddressesResponse{Response: resp} err := service.Listener.Encode(w, &ipResp) - log.Response(service.Name, ipResp, err) + log.Response(service.Name, ipResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // Handles retrieval of ghost ip addresses from ipam driver. @@ -728,8 +726,7 @@ func (service *HTTPRestService) getUnhealthyIPAddresses(w http.ResponseWriter, r } err := service.Listener.Encode(w, &ipResp) - - log.Response(service.Name, ipResp, err) + log.Response(service.Name, ipResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // getAllIPAddresses retrieves all ip addresses from ipam driver. @@ -746,7 +743,7 @@ func (service *HTTPRestService) getAllIPAddresses(w http.ResponseWriter, r *http ipResp := &cns.GetIPAddressesResponse{Response: resp} err := service.Listener.Encode(w, &ipResp) - log.Response(service.Name, ipResp, err) + log.Response(service.Name, ipResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // Handles health report requests. @@ -762,7 +759,7 @@ func (service *HTTPRestService) getHealthReport(w http.ResponseWriter, r *http.R resp := &cns.Response{ReturnCode: 0} err := service.Listener.Encode(w, &resp) - log.Response(service.Name, resp, err) + log.Response(service.Name, resp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // saveState writes CNS state to persistent store. @@ -781,7 +778,7 @@ func (service *HTTPRestService) saveState() error { if err == nil { log.Printf("[Azure CNS] State saved successfully.\n") } else { - log.Printf("[Azure CNS] Failed to save state., err:%v\n", err) + log.Errorf("[Azure CNS] Failed to save state., err:%v\n", err) } return err @@ -806,7 +803,7 @@ func (service *HTTPRestService) restoreState() error { return nil } - log.Printf("[Azure CNS] Failed to restore state, err:%v\n", err) + log.Errorf("[Azure CNS] Failed to restore state, err:%v\n", err) return err } @@ -847,7 +844,7 @@ func (service *HTTPRestService) setOrchestratorType(w http.ResponseWriter, r *ht } err = service.Listener.Encode(w, &resp) - log.Response(service.Name, resp, err) + log.Response(service.Name, resp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetworkContainerRequest) (int, string) { @@ -952,8 +949,7 @@ func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWr reserveResp := &cns.CreateNetworkContainerResponse{Response: resp} err = service.Listener.Encode(w, &reserveResp) - - log.Response(service.Name, reserveResp, err) + log.Response(service.Name, reserveResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } func (service *HTTPRestService) getNetworkContainerByID(w http.ResponseWriter, r *http.Request) { @@ -976,7 +972,7 @@ func (service *HTTPRestService) getNetworkContainerByID(w http.ResponseWriter, r reserveResp := &cns.GetNetworkContainerResponse{Response: resp} err = service.Listener.Encode(w, &reserveResp) - log.Response(service.Name, reserveResp, err) + log.Response(service.Name, reserveResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkContainerRequest) cns.GetNetworkContainerResponse { @@ -1040,9 +1036,9 @@ func (service *HTTPRestService) getNetworkContainerByOrchestratorContext(w http. } getNetworkContainerResponse := service.getNetworkContainerResponse(req) - + returnCode := getNetworkContainerResponse.Response.ReturnCode err = service.Listener.Encode(w, &getNetworkContainerResponse) - log.Response(service.Name, getNetworkContainerResponse, err) + log.Response(service.Name, getNetworkContainerResponse, returnCode, ReturnCodeToString(returnCode), err) } func (service *HTTPRestService) deleteNetworkContainer(w http.ResponseWriter, r *http.Request) { @@ -1116,8 +1112,7 @@ func (service *HTTPRestService) deleteNetworkContainer(w http.ResponseWriter, r reserveResp := &cns.DeleteNetworkContainerResponse{Response: resp} err = service.Listener.Encode(w, &reserveResp) - - log.Response(service.Name, reserveResp, err) + log.Response(service.Name, reserveResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } func (service *HTTPRestService) getNetworkContainerStatus(w http.ResponseWriter, r *http.Request) { @@ -1179,8 +1174,7 @@ func (service *HTTPRestService) getNetworkContainerStatus(w http.ResponseWriter, } err = service.Listener.Encode(w, &networkContainerStatusReponse) - - log.Response(service.Name, networkContainerStatusReponse, err) + log.Response(service.Name, networkContainerStatusReponse, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } func (service *HTTPRestService) getInterfaceForContainer(w http.ResponseWriter, r *http.Request) { @@ -1234,7 +1228,7 @@ func (service *HTTPRestService) getInterfaceForContainer(w http.ResponseWriter, err = service.Listener.Encode(w, &getInterfaceForContainerResponse) - log.Response(service.Name, getInterfaceForContainerResponse, err) + log.Response(service.Name, getInterfaceForContainerResponse, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } // restoreNetworkState restores Network state that existed before reboot. diff --git a/cns/routes/routes_windows.go b/cns/routes/routes_windows.go index 521577575b..4152169a6d 100644 --- a/cns/routes/routes_windows.go +++ b/cns/routes/routes_windows.go @@ -168,7 +168,7 @@ func putRoutes(routes []Route) error { if err == nil { log.Printf("[Azure CNS] Successfully executed add route: %v\n%v", args, string(bytes)) } else { - log.Printf("[Azure CNS] Failed to execute add route: %v\n%v", args, string(bytes)) + log.Errorf("[Azure CNS] Failed to execute add route: %v\n%v", args, string(bytes)) } } else { log.Printf("[Azure CNS] Route already exists. skipping %+v", route) diff --git a/cns/service.go b/cns/service.go index 4de0024e63..c6d8b74c36 100644 --- a/cns/service.go +++ b/cns/service.go @@ -101,5 +101,5 @@ func (service *Service) ParseOptions(options OptionMap) OptionMap { func (service *Service) SendErrorResponse(w http.ResponseWriter, errMsg error) { resp := errorResponse{errMsg.Error()} err := service.Listener.Encode(w, &resp) - log.Response(service.Name, &resp, err) + log.Errorf("[%s] %+v %s.", service.Name, &resp, err.Error()) } diff --git a/cns/service/main.go b/cns/service/main.go index aaa001fc03..5efe9ad744 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -127,7 +127,7 @@ var args = acn.ArgumentList{ Shorthand: acn.OptReportToHostIntervalAlias, Description: "Set interval in ms to report to host", Type: "int", - DefaultValue: 60000, + DefaultValue: "60000", }, } @@ -191,21 +191,21 @@ func main() { err = acn.CreateDirectory(platform.CNMRuntimePath) if err != nil { - log.Printf("Failed to create File Store directory Error:%v", err.Error()) + log.Errorf("Failed to create File Store directory Error:%v", err.Error()) return } // Create the key value store. config.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + name + ".json") if err != nil { - log.Printf("Failed to create store: %v\n", err) + log.Errorf("Failed to create store: %v\n", err) return } // Create CNS object. httpRestService, err := restserver.NewHTTPRestService(&config) if err != nil { - log.Printf("Failed to create CNS object, err:%v.\n", err) + log.Errorf("Failed to create CNS object, err:%v.\n", err) return } @@ -220,7 +220,7 @@ func main() { telemetryStopProcessing) err = httpRestService.Start(&config) if err != nil { - log.Printf("Failed to start CNS, err:%v.\n", err) + log.Errorf("Failed to start CNS, err:%v.\n", err) return } } @@ -238,21 +238,21 @@ func main() { // Create network plugin. netPlugin, err = network.NewPlugin(&pluginConfig) if err != nil { - log.Printf("Failed to create network plugin, err:%v.\n", err) + log.Errorf("Failed to create network plugin, err:%v.\n", err) return } // Create IPAM plugin. ipamPlugin, err = ipam.NewPlugin(&pluginConfig) if err != nil { - log.Printf("Failed to create IPAM plugin, err:%v.\n", err) + log.Errorf("Failed to create IPAM plugin, err:%v.\n", err) return } // Create the key value store. pluginConfig.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + pluginName + ".json") if err != nil { - log.Printf("Failed to create store: %v\n", err) + log.Errorf("Failed to create store: %v\n", err) return } @@ -260,7 +260,7 @@ func main() { netPlugin.SetOption(acn.OptAPIServerURL, url) log.Printf("Start netplugin\n") if err := netPlugin.Start(&pluginConfig); err != nil { - log.Printf("Failed to create network plugin, err:%v.\n", err) + log.Errorf("Failed to create network plugin, err:%v.\n", err) return } @@ -269,7 +269,7 @@ func main() { ipamPlugin.SetOption(acn.OptIpamQueryUrl, ipamQueryUrl) ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) if err := ipamPlugin.Start(&pluginConfig); err != nil { - log.Printf("Failed to create IPAM plugin, err:%v.\n", err) + log.Errorf("Failed to create IPAM plugin, err:%v.\n", err) return } } diff --git a/log/logger.go b/log/logger.go index 78e38d4e78..aac175686e 100644 --- a/log/logger.go +++ b/log/logger.go @@ -170,16 +170,16 @@ func (logger *Logger) Request(tag string, request interface{}, err error) { if err == nil { logger.Printf("[%s] Received %T %+v.", tag, request, request) } else { - logger.Printf("[%s] Failed to decode %T %+v %s.", tag, request, request, err.Error()) + logger.Errorf("[%s] Failed to decode %T %+v %s.", tag, request, request, err.Error()) } } // Response logs a structured response. -func (logger *Logger) Response(tag string, response interface{}, err error) { - if err == nil { +func (logger *Logger) Response(tag string, response interface{}, returnCode int, returnStr string, err error) { + if err == nil && returnCode == 0 { logger.Printf("[%s] Sent %T %+v.", tag, response, response) } else { - logger.Printf("[%s] Failed to encode %T %+v %s.", tag, response, response, err.Error()) + logger.Errorf("[%s] Code:%s, %+v %s.", tag, returnStr, response, err.Error()) } } diff --git a/log/stdapi.go b/log/stdapi.go index 721e2a84e1..21a3a5fcf4 100644 --- a/log/stdapi.go +++ b/log/stdapi.go @@ -43,8 +43,8 @@ func Request(tag string, request interface{}, err error) { stdLog.Request(tag, request, err) } -func Response(tag string, response interface{}, err error) { - stdLog.Response(tag, response, err) +func Response(tag string, response interface{}, returnCode int, returnStr string, err error) { + stdLog.Response(tag, response, returnCode, returnStr, err) } func Printf(format string, args ...interface{}) { @@ -54,3 +54,7 @@ func Printf(format string, args ...interface{}) { func Debugf(format string, args ...interface{}) { stdLog.Debugf(format, args...) } + +func Errorf(format string, args ...interface{}) { + stdLog.Errorf(format, args...) +} diff --git a/telemetry/cnstelemetry.go b/telemetry/cnstelemetry.go index 79780d45b7..6536983f5f 100644 --- a/telemetry/cnstelemetry.go +++ b/telemetry/cnstelemetry.go @@ -6,9 +6,11 @@ package telemetry import ( "fmt" "reflect" + "regexp" "time" "github.com/Azure/azure-container-networking/cns/restserver" + "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/platform" "github.com/google/uuid" ) @@ -16,6 +18,7 @@ import ( const ( // CNSTelemetryFile - telemetry file path. CNSTelemetryFile = platform.CNSRuntimePath + "AzureCNSTelemetry.json" + errorcodePrefix = 5 ) // SendCnsTelemetry - handles cns telemetry reports @@ -47,7 +50,7 @@ CONNECT: } // Try to set partition key from DNC - if reportMgr.Report.(CNSReport).DncPartitionKey == "" { + if reportMgr.Report.(*CNSReport).DncPartitionKey == "" { reflect.ValueOf(reportMgr.Report).Elem().FieldByName("DncPartitionKey").SetString(service.GetPartitionKey()) } @@ -55,6 +58,11 @@ CONNECT: case <-heartbeat: reflect.ValueOf(reportMgr.Report).Elem().FieldByName("EventMessage").SetString("Heartbeat") case msg := <-reports: + codeStr := regexp.MustCompile(`Code:(\w*)`).FindString(msg.(string)) + if len(codeStr) > errorcodePrefix { + reflect.ValueOf(reportMgr.Report).Elem().FieldByName("Errorcode").SetString(codeStr[errorcodePrefix:]) + } + reflect.ValueOf(reportMgr.Report).Elem().FieldByName("EventMessage").SetString(msg.(string)) case <-telemetryStopProcessing: telemetryBuffer.Cancel() @@ -80,7 +88,7 @@ CONNECT: } } } else { - reports <- CNSReport{EventMessage: "Failed to establish telemetry buffer connection."} + log.Printf("[Telemetry] Failed to establish telemetry buffer connection.") time.Sleep(time.Minute * 1) goto CONNECT } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 4bdb56f6c4..c5527c030d 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -130,6 +130,7 @@ type CNSReport struct { DncPartitionKey string Timestamp string UUID string + Errorcode string Metadata Metadata `json:"compute"` } diff --git a/telemetry/telemetrybuffer.go b/telemetry/telemetrybuffer.go index 122485e213..004b6f79fc 100644 --- a/telemetry/telemetrybuffer.go +++ b/telemetry/telemetrybuffer.go @@ -22,9 +22,10 @@ import ( // DefaultNpmReportsSize - default NPM report slice size // DefaultInterval - default interval for sending payload to host const ( - FdName = "azure-telemetry" - Delimiter = '\n' - HostNetAgentURL = "http://169.254.169.254/machine/plugins?comp=netagent&type=payload" + FdName = "azure-telemetry" + Delimiter = '\n' + //HostNetAgentURL = "http://169.254.169.254/machine/plugins?comp=netagent&type=payload" + HostNetAgentURL = "http://localhost:8019/hostnetagent/container/1234/payload" DefaultInterval = 1 * time.Minute ) @@ -102,6 +103,7 @@ func NewTelemetryBuffer() (*TelemetryBuffer, error) { tb.payload.DNCReports = make([]DNCReport, 0) tb.payload.CNIReports = make([]CNIReport, 0) tb.payload.NPMReports = make([]NPMReport, 0) + tb.payload.CNSReports = make([]CNSReport, 0) } else if tb.fdExists { tb.cleanup(FdName) } @@ -185,6 +187,7 @@ func (tb *TelemetryBuffer) close() { // sendToHost - send payload to host func (tb *TelemetryBuffer) sendToHost() error { + fmt.Printf("%+v\n", tb.payload) httpc := &http.Client{} var body bytes.Buffer json.NewEncoder(&body).Encode(tb.payload) @@ -211,6 +214,8 @@ func (pl *Payload) push(x interface{}) { pl.CNIReports = append(pl.CNIReports, x.(CNIReport)) case NPMReport: pl.NPMReports = append(pl.NPMReports, x.(NPMReport)) + case CNSReport: + pl.CNSReports = append(pl.CNSReports, x.(CNSReport)) } } @@ -222,4 +227,6 @@ func (pl *Payload) reset() { pl.CNIReports = make([]CNIReport, 0) pl.NPMReports = nil pl.NPMReports = make([]NPMReport, 0) + pl.CNSReports = nil + pl.CNSReports = make([]CNSReport, 0) }