From 0308cd07db333da1a521e33bc4dc99661ddca894 Mon Sep 17 00:00:00 2001 From: Sathya Singh <29132953+sathyasi@users.noreply.github.com> Date: Tue, 11 Feb 2020 18:24:40 -0800 Subject: [PATCH 1/4] Add AI Events and CNS NC Snapshots. --- aitelemetry/api.go | 10 +++++++ aitelemetry/telemetrywrapper.go | 51 +++++++++++++++++++++++++++++++++ cns/logger/log.go | 6 ++++ cns/restserver/restserver.go | 49 +++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+) diff --git a/aitelemetry/api.go b/aitelemetry/api.go index d23389edd4..eab7d70320 100644 --- a/aitelemetry/api.go +++ b/aitelemetry/api.go @@ -14,6 +14,13 @@ type Report struct { CustomDimensions map[string]string } +// Application event structure +type AiEvent struct { + EventName string + ResourceID string + Properties map[string]string +} + // Application metrics structure type Metric struct { Name string @@ -54,6 +61,9 @@ type TelemetryHandle interface { // TrackMetric function sends metric to appinsights resource. It overrides few of the existing columns with app information // and for rest it uses custom dimesion TrackMetric(metric Metric) + // TrackEvent function sends events to appinsights resource. It overrides a few of the existing columns + // with app information. + TrackEvent(aiEvent AiEvent) // Close - should be called for each NewAITelemetry call. Will release resources acquired Close(timeout int) } diff --git a/aitelemetry/telemetrywrapper.go b/aitelemetry/telemetrywrapper.go index aa18e7df84..e36f2aacb1 100644 --- a/aitelemetry/telemetrywrapper.go +++ b/aitelemetry/telemetrywrapper.go @@ -15,6 +15,7 @@ const ( resourceGroupStr = "ResourceGroup" vmSizeStr = "VMSize" osVersionStr = "OSVersion" + osStr = "OS" locationStr = "Region" appNameStr = "AppName" subscriptionIDStr = "SubscriptionID" @@ -220,6 +221,56 @@ func (th *telemetryHandle) TrackLog(report Report) { th.client.Track(trace) } +// TrackEvent function sends events to appinsights resource. It overrides a few of the existing columns +// with app information. +func (th *telemetryHandle) TrackEvent(aiEvent AiEvent) { + + // Initialize new event message + event := appinsights.NewEventTelemetry(aiEvent.EventName) + + // OperationId => resourceID (e.g.: NCID) + event.Tags.Operation().SetId(aiEvent.ResourceID) + + // Copy the properties, if supplied + if aiEvent.Properties != nil { + for key, value := range aiEvent.Properties { + event.Properties[key] = value + } + } + + // Acquire read lock to read metadata + th.rwmutex.RLock() + metadata := th.metadata + + // Add metadata + if metadata.SubscriptionID != "" { + event.Tags.User().SetAccountId(metadata.SubscriptionID) + + // AnonId => VMName + event.Tags.User().SetId(metadata.VMName) + + // SessionId => VMID + event.Tags.Session().SetId(metadata.VMID) + event.Tags.Operation().SetParentId(th.appVersion) + event.Tags.User().SetAuthUserId(runtime.GOOS) + + event.Properties[locationStr] = metadata.Location + event.Properties[resourceGroupStr] = metadata.ResourceGroupName + event.Properties[vmSizeStr] = metadata.VMSize + event.Properties[osVersionStr] = metadata.OSVersion + event.Properties[vmIDStr] = metadata.VMID + event.Properties[vmNameStr] = metadata.VMName + event.Properties[osStr] = runtime.GOOS + } + + th.rwmutex.RUnlock() + + event.Properties[appNameStr] = th.appName + event.Properties[versionStr] = th.appVersion + + th.client.Track(event) +} + // TrackMetric function sends metric to appinsights resource. It overrides few of the existing columns with app information // and for rest it uses custom dimesion func (th *telemetryHandle) TrackMetric(metric Metric) { diff --git a/cns/logger/log.go b/cns/logger/log.go index d83bc7889e..8636143fe4 100644 --- a/cns/logger/log.go +++ b/cns/logger/log.go @@ -104,6 +104,12 @@ func Debugf(format string, args ...interface{}) { sendTraceInternal(msg) } +func LogNCSnapshot(aiEvent aitelemetry.AiEvent) { + aiEvent.Properties[OrchestratorTypeStr] = Log.Orchestrator + aiEvent.Properties[NodeIDStr] = Log.NodeID + Log.th.TrackEvent(aiEvent) +} + func Errorf(format string, args ...interface{}) { Log.logger.Errorf(format, args...) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 231a09c173..67c49ed19c 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -13,6 +13,7 @@ import ( "sync" "time" + "github.com/Azure/azure-container-networking/aitelemetry" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/dockerclient" @@ -205,6 +206,10 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { logger.SetContextDetails(service.state.OrchestratorType, service.state.NodeID) logger.Printf("[Azure CNS] Listening.") + + // Setup to emit snapshot every 60 minutes. + go service.sendNCSnapShotPeriodically(60) + return nil } @@ -1174,6 +1179,9 @@ func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWr reserveResp := &cns.CreateNetworkContainerResponse{Response: resp} err = service.Listener.Encode(w, &reserveResp) + + logNCSnapshot(req) + logger.Response(service.Name, reserveResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } @@ -1961,3 +1969,44 @@ func (service *HTTPRestService) unpublishNetworkContainer(w http.ResponseWriter, err = service.Listener.Encode(w, &response) logger.Response(service.Name, response, response.Response.ReturnCode, ReturnCodeToString(response.Response.ReturnCode), err) } + +func logNCSnapshot(createNetworkContainerRequest cns.CreateNetworkContainerRequest) { + var aiEvent = aitelemetry.AiEvent{ + EventName: "CNSNCSnapshot", + Properties: make(map[string]string), + ResourceID: createNetworkContainerRequest.NetworkContainerid, + } + + aiEvent.Properties["IPConfiguration"] = fmt.Sprintf("%+v", createNetworkContainerRequest.IPConfiguration) + aiEvent.Properties["LocalIPConfiguration"] = fmt.Sprintf("%+v", createNetworkContainerRequest.LocalIPConfiguration) + aiEvent.Properties["PrimaryInterfaceIdentifier"] = createNetworkContainerRequest.PrimaryInterfaceIdentifier + aiEvent.Properties["MultiTenancyInfo"] = fmt.Sprintf("%+v", createNetworkContainerRequest.MultiTenancyInfo) + aiEvent.Properties["CnetAddressSpace"] = fmt.Sprintf("%+v", createNetworkContainerRequest.CnetAddressSpace) + aiEvent.Properties["AllowNCToHostCommunication"] = fmt.Sprintf("%t", createNetworkContainerRequest.AllowNCToHostCommunication) + aiEvent.Properties["AllowHostToNCCommunication"] = fmt.Sprintf("%t", createNetworkContainerRequest.AllowHostToNCCommunication) + aiEvent.Properties["NetworkContainerType"] = createNetworkContainerRequest.NetworkContainerType + aiEvent.Properties["OrchestratorContext"] = fmt.Sprintf("%s", createNetworkContainerRequest.OrchestratorContext) + + logger.LogNCSnapshot(aiEvent) +} + +func (service *HTTPRestService) logNCSnapshots() { + logger.Printf("[Azure CNS] Logging periodic NC snapshots.") + for _, ncStatus := range service.state.ContainerStatus { + logNCSnapshot(ncStatus.CreateNetworkContainerRequest) + } +} + +func (service *HTTPRestService) sendNCSnapShotPeriodically(ncSnapshotIntervalInMinutes int) { + + // Emit snapshot on startup and then emit it periodically. + service.logNCSnapshots() + + snapshot := time.NewTicker(time.Minute * time.Duration(ncSnapshotIntervalInMinutes)).C + for { + select { + case <-snapshot: + service.logNCSnapshots() + } + } +} From 74aa0393c63a787c318584fe5f26165481c34d38 Mon Sep 17 00:00:00 2001 From: Sathya Singh <29132953+sathyasi@users.noreply.github.com> Date: Wed, 12 Feb 2020 13:03:00 -0800 Subject: [PATCH 2/4] Addressed review comments. --- aitelemetry/telemetrywrapper.go | 7 +++---- cns/logger/log.go | 2 +- cns/restserver/restserver.go | 21 ++++++++++++++++----- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/aitelemetry/telemetrywrapper.go b/aitelemetry/telemetrywrapper.go index e36f2aacb1..0bf022abe6 100644 --- a/aitelemetry/telemetrywrapper.go +++ b/aitelemetry/telemetrywrapper.go @@ -251,20 +251,19 @@ func (th *telemetryHandle) TrackEvent(aiEvent AiEvent) { // SessionId => VMID event.Tags.Session().SetId(metadata.VMID) - event.Tags.Operation().SetParentId(th.appVersion) - event.Tags.User().SetAuthUserId(runtime.GOOS) - event.Properties[locationStr] = metadata.Location event.Properties[resourceGroupStr] = metadata.ResourceGroupName event.Properties[vmSizeStr] = metadata.VMSize event.Properties[osVersionStr] = metadata.OSVersion event.Properties[vmIDStr] = metadata.VMID event.Properties[vmNameStr] = metadata.VMName - event.Properties[osStr] = runtime.GOOS } th.rwmutex.RUnlock() + event.Tags.Operation().SetParentId(th.appVersion) + event.Tags.User().SetAuthUserId(runtime.GOOS) + event.Properties[osStr] = runtime.GOOS event.Properties[appNameStr] = th.appName event.Properties[versionStr] = th.appVersion diff --git a/cns/logger/log.go b/cns/logger/log.go index 8636143fe4..89605bf4cb 100644 --- a/cns/logger/log.go +++ b/cns/logger/log.go @@ -104,7 +104,7 @@ func Debugf(format string, args ...interface{}) { sendTraceInternal(msg) } -func LogNCSnapshot(aiEvent aitelemetry.AiEvent) { +func LogAiEvent(aiEvent aitelemetry.AiEvent) { aiEvent.Properties[OrchestratorTypeStr] = Log.Orchestrator aiEvent.Properties[NodeIDStr] = Log.NodeID Log.th.TrackEvent(aiEvent) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 67c49ed19c..2990d49e37 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -32,6 +32,9 @@ import ( var ( // Named Lock for accessing different states in httpRestServiceState namedLock = acn.InitNamedLock() + + // Network container snapshot interval in minutes. + ncSnapshotIntervalInMinutes = 60 ) const ( @@ -207,8 +210,8 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { logger.SetContextDetails(service.state.OrchestratorType, service.state.NodeID) logger.Printf("[Azure CNS] Listening.") - // Setup to emit snapshot every 60 minutes. - go service.sendNCSnapShotPeriodically(60) + // Setup to emit snapshot periodically. + go service.sendNCSnapShotPeriodically(ncSnapshotIntervalInMinutes) return nil } @@ -1180,7 +1183,10 @@ func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWr reserveResp := &cns.CreateNetworkContainerResponse{Response: resp} err = service.Listener.Encode(w, &reserveResp) - logNCSnapshot(req) + // If the NC was created successfully, log NC snapshot. + if returnCode == 0 { + logNCSnapshot(req) + } logger.Response(service.Name, reserveResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } @@ -1987,14 +1993,19 @@ func logNCSnapshot(createNetworkContainerRequest cns.CreateNetworkContainerReque aiEvent.Properties["NetworkContainerType"] = createNetworkContainerRequest.NetworkContainerType aiEvent.Properties["OrchestratorContext"] = fmt.Sprintf("%s", createNetworkContainerRequest.OrchestratorContext) - logger.LogNCSnapshot(aiEvent) + logger.LogAiEvent(aiEvent) } +// Sends network container snapshots to App Insights telemetry. func (service *HTTPRestService) logNCSnapshots() { - logger.Printf("[Azure CNS] Logging periodic NC snapshots.") + ncCount := 0 + for _, ncStatus := range service.state.ContainerStatus { logNCSnapshot(ncStatus.CreateNetworkContainerRequest) + ncCount++ } + + logger.Printf("[Azure CNS] Logging periodic NC snapshots. NC Count %d", ncCount) } func (service *HTTPRestService) sendNCSnapShotPeriodically(ncSnapshotIntervalInMinutes int) { From a4c1463ea066ee5dbe24162c604fedd94078e4ce Mon Sep 17 00:00:00 2001 From: Sathya Singh <29132953+sathyasi@users.noreply.github.com> Date: Wed, 12 Feb 2020 19:15:10 -0800 Subject: [PATCH 3/4] Addressed review comments and added UTs. --- aitelemetry/api.go | 4 +- aitelemetry/telemetrywrapper.go | 64 +++++++++++++--------------- aitelemetry/telemetrywrapper_test.go | 12 ++++++ cns/configuration/cns_config.json | 3 +- cns/configuration/configuration.go | 8 ++++ cns/logger/log.go | 17 +++++--- cns/restserver/restserver.go | 48 ++++++++++++--------- cns/service/main.go | 5 ++- 8 files changed, 98 insertions(+), 63 deletions(-) diff --git a/aitelemetry/api.go b/aitelemetry/api.go index eab7d70320..d23f055eb2 100644 --- a/aitelemetry/api.go +++ b/aitelemetry/api.go @@ -15,7 +15,7 @@ type Report struct { } // Application event structure -type AiEvent struct { +type Event struct { EventName string ResourceID string Properties map[string]string @@ -63,7 +63,7 @@ type TelemetryHandle interface { TrackMetric(metric Metric) // TrackEvent function sends events to appinsights resource. It overrides a few of the existing columns // with app information. - TrackEvent(aiEvent AiEvent) + TrackEvent(aiEvent Event) // Close - should be called for each NewAITelemetry call. Will release resources acquired Close(timeout int) } diff --git a/aitelemetry/telemetrywrapper.go b/aitelemetry/telemetrywrapper.go index 0bf022abe6..5262618ecf 100644 --- a/aitelemetry/telemetrywrapper.go +++ b/aitelemetry/telemetrywrapper.go @@ -208,13 +208,13 @@ func (th *telemetryHandle) TrackLog(report Report) { // Check if metadata is populated if metadata.SubscriptionID != "" { // copy metadata from wireserver to trace - trace.Tags.User().SetAccountId(th.metadata.SubscriptionID) - trace.Tags.User().SetId(th.metadata.VMName) - trace.Properties[locationStr] = th.metadata.Location - trace.Properties[resourceGroupStr] = th.metadata.ResourceGroupName - trace.Properties[vmSizeStr] = th.metadata.VMSize - trace.Properties[osVersionStr] = th.metadata.OSVersion - trace.Properties[vmIDStr] = th.metadata.VMID + trace.Tags.User().SetAccountId(metadata.SubscriptionID) + trace.Tags.User().SetId(metadata.VMName) + trace.Properties[locationStr] = metadata.Location + trace.Properties[resourceGroupStr] = metadata.ResourceGroupName + trace.Properties[vmSizeStr] = metadata.VMSize + trace.Properties[osVersionStr] = metadata.OSVersion + trace.Properties[vmIDStr] = metadata.VMID } // send to appinsights resource @@ -223,51 +223,45 @@ func (th *telemetryHandle) TrackLog(report Report) { // TrackEvent function sends events to appinsights resource. It overrides a few of the existing columns // with app information. -func (th *telemetryHandle) TrackEvent(aiEvent AiEvent) { - +func (th *telemetryHandle) TrackEvent(event Event) { // Initialize new event message - event := appinsights.NewEventTelemetry(aiEvent.EventName) - + aiEvent := appinsights.NewEventTelemetry(event.EventName) // OperationId => resourceID (e.g.: NCID) - event.Tags.Operation().SetId(aiEvent.ResourceID) + aiEvent.Tags.Operation().SetId(event.ResourceID) // Copy the properties, if supplied - if aiEvent.Properties != nil { - for key, value := range aiEvent.Properties { - event.Properties[key] = value + if event.Properties != nil { + for key, value := range event.Properties { + aiEvent.Properties[key] = value } } // Acquire read lock to read metadata th.rwmutex.RLock() metadata := th.metadata + th.rwmutex.RUnlock() // Add metadata if metadata.SubscriptionID != "" { - event.Tags.User().SetAccountId(metadata.SubscriptionID) - + aiEvent.Tags.User().SetAccountId(metadata.SubscriptionID) // AnonId => VMName - event.Tags.User().SetId(metadata.VMName) - + aiEvent.Tags.User().SetId(metadata.VMName) // SessionId => VMID - event.Tags.Session().SetId(metadata.VMID) - event.Properties[locationStr] = metadata.Location - event.Properties[resourceGroupStr] = metadata.ResourceGroupName - event.Properties[vmSizeStr] = metadata.VMSize - event.Properties[osVersionStr] = metadata.OSVersion - event.Properties[vmIDStr] = metadata.VMID - event.Properties[vmNameStr] = metadata.VMName + aiEvent.Tags.Session().SetId(metadata.VMID) + aiEvent.Properties[locationStr] = metadata.Location + aiEvent.Properties[resourceGroupStr] = metadata.ResourceGroupName + aiEvent.Properties[vmSizeStr] = metadata.VMSize + aiEvent.Properties[osVersionStr] = metadata.OSVersion + aiEvent.Properties[vmIDStr] = metadata.VMID + aiEvent.Properties[vmNameStr] = metadata.VMName } - th.rwmutex.RUnlock() - - event.Tags.Operation().SetParentId(th.appVersion) - event.Tags.User().SetAuthUserId(runtime.GOOS) - event.Properties[osStr] = runtime.GOOS - event.Properties[appNameStr] = th.appName - event.Properties[versionStr] = th.appVersion - - th.client.Track(event) + aiEvent.Tags.Operation().SetParentId(th.appVersion) + aiEvent.Tags.User().SetAuthUserId(runtime.GOOS) + aiEvent.Properties[osStr] = runtime.GOOS + aiEvent.Properties[appNameStr] = th.appName + aiEvent.Properties[versionStr] = th.appVersion + th.client.Track(aiEvent) } // TrackMetric function sends metric to appinsights resource. It overrides few of the existing columns with app information diff --git a/aitelemetry/telemetrywrapper_test.go b/aitelemetry/telemetrywrapper_test.go index 4897fe583d..0c1f3f70e8 100644 --- a/aitelemetry/telemetrywrapper_test.go +++ b/aitelemetry/telemetrywrapper_test.go @@ -129,6 +129,18 @@ func TestTrackLog(t *testing.T) { th.TrackLog(report) } +func TestTrackEvent(t *testing.T) { + event := Event{ + EventName: "testEvent", + ResourceID: "SomeResourceId", + Properties: make(map[string]string), + } + + event.Properties["P1"] = "V1" + event.Properties["P2"] = "V2" + th.TrackEvent(event) +} + func TestClose(t *testing.T) { th.Close(10) } diff --git a/cns/configuration/cns_config.json b/cns/configuration/cns_config.json index fb72310285..44e134e297 100644 --- a/cns/configuration/cns_config.json +++ b/cns/configuration/cns_config.json @@ -5,6 +5,7 @@ "RefreshIntervalInSecs": 15, "DisableAll": false, "HeartBeatIntervalInMins": 30, - "DebugMode": false + "DebugMode": false, + "SnapshotIntervalInMins": 60 } } diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index 18e7a4c680..edaf70a18d 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -26,6 +26,8 @@ type TelemetrySettings struct { DisableTrace bool // Flag to Disable sending metric. DisableMetric bool + // Flag to Disable sending events. + DisableEvent bool // Configure how many bytes can be sent in one call to the data collector TelemetryBatchSizeBytes int // Configure the maximum delay before sending queued telemetry in milliseconds @@ -38,6 +40,8 @@ type TelemetrySettings struct { RefreshIntervalInSecs int // Disable debug logging for telemetry messages DebugMode bool + // Interval for sending snapshot events. + SnapshotIntervalInMins int } // This functions reads cns config file and save it in a structure @@ -89,6 +93,10 @@ func setTelemetrySettingDefaults(telemetrySettings *TelemetrySettings) { // set the default Heartbeat interval to 30 minutes telemetrySettings.HeartBeatIntervalInMins = 30 } + + if telemetrySettings.SnapshotIntervalInMins == 0 { + telemetrySettings.SnapshotIntervalInMins = 60 + } } // Set Default values of CNS config if not specified diff --git a/cns/logger/log.go b/cns/logger/log.go index 89605bf4cb..3781e2f264 100644 --- a/cns/logger/log.go +++ b/cns/logger/log.go @@ -25,6 +25,7 @@ type CNSLogger struct { NodeID string DisableTraceLogging bool DisableMetricLogging bool + DisableEventLogging bool } // Initialize CNS Logger @@ -35,7 +36,7 @@ func InitLogger(fileName string, logLevel, logTarget int, logDir string) { } // Intialize CNS AI telmetry instance -func InitAI(aiConfig aitelemetry.AIConfig, disableTraceLogging, disableMetricLogging bool) { +func InitAI(aiConfig aitelemetry.AIConfig, disableTraceLogging, disableMetricLogging bool, disableEventLogging bool) { var err error Log.th, err = aitelemetry.NewAITelemetry("", aiMetadata, aiConfig) @@ -47,6 +48,7 @@ func InitAI(aiConfig aitelemetry.AIConfig, disableTraceLogging, disableMetricLog Log.logger.Printf("AI Telemetry Handle created") Log.DisableMetricLogging = disableMetricLogging Log.DisableTraceLogging = disableTraceLogging + Log.DisableEventLogging = disableEventLogging } func InitReportChannel(reports chan interface{}) { @@ -104,10 +106,15 @@ func Debugf(format string, args ...interface{}) { sendTraceInternal(msg) } -func LogAiEvent(aiEvent aitelemetry.AiEvent) { - aiEvent.Properties[OrchestratorTypeStr] = Log.Orchestrator - aiEvent.Properties[NodeIDStr] = Log.NodeID - Log.th.TrackEvent(aiEvent) +func LogEvent(event aitelemetry.Event) { + + if Log.th == nil || Log.DisableEventLogging { + return + } + + event.Properties[OrchestratorTypeStr] = Log.Orchestrator + event.Properties[NodeIDStr] = Log.NodeID + Log.th.TrackEvent(event) } func Errorf(format string, args ...interface{}) { diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 2990d49e37..6e0a1aa5c0 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -45,6 +45,17 @@ const ( detach = "Detach" // Rest service state identifier for named lock stateJoinedNetworks = "JoinedNetworks" + // CNS Snspshot properties + cnsNCSnapshotEventStr = "CNSNCSnapshot" + ipConfigurationStr = "IPConfiguration" + localIPConfigurationStr = "LocalIPConfiguration" + primaryInterfaceIdentifierStr = "PrimaryInterfaceIdentifier" + multiTenancyInfoStr = "MultiTenancyInfo" + cnetAddressSpaceStr = "CnetAddressSpace" + allowNCToHostCommunicationStr = "AllowNCToHostCommunication" + allowHostToNCCommunicationStr = "AllowHostToNCCommunication" + networkContainerTypeStr = "NetworkContainerType" + orchestratorContextStr = "OrchestratorContext" ) // HTTPRestService represents http listener for CNS - Container Networking Service. @@ -92,6 +103,7 @@ type networkInfo struct { // HTTPService describes the min API interface that every service should have. type HTTPService interface { common.ServiceAPI + SendNCSnapShotPeriodically(int, chan bool) } // NewHTTPRestService creates a new HTTP Service object. @@ -210,9 +222,6 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { logger.SetContextDetails(service.state.OrchestratorType, service.state.NodeID) logger.Printf("[Azure CNS] Listening.") - // Setup to emit snapshot periodically. - go service.sendNCSnapShotPeriodically(ncSnapshotIntervalInMinutes) - return nil } @@ -1977,38 +1986,37 @@ func (service *HTTPRestService) unpublishNetworkContainer(w http.ResponseWriter, } func logNCSnapshot(createNetworkContainerRequest cns.CreateNetworkContainerRequest) { - var aiEvent = aitelemetry.AiEvent{ - EventName: "CNSNCSnapshot", + var aiEvent = aitelemetry.Event{ + EventName: cnsNCSnapshotEventStr, Properties: make(map[string]string), ResourceID: createNetworkContainerRequest.NetworkContainerid, } - aiEvent.Properties["IPConfiguration"] = fmt.Sprintf("%+v", createNetworkContainerRequest.IPConfiguration) - aiEvent.Properties["LocalIPConfiguration"] = fmt.Sprintf("%+v", createNetworkContainerRequest.LocalIPConfiguration) - aiEvent.Properties["PrimaryInterfaceIdentifier"] = createNetworkContainerRequest.PrimaryInterfaceIdentifier - aiEvent.Properties["MultiTenancyInfo"] = fmt.Sprintf("%+v", createNetworkContainerRequest.MultiTenancyInfo) - aiEvent.Properties["CnetAddressSpace"] = fmt.Sprintf("%+v", createNetworkContainerRequest.CnetAddressSpace) - aiEvent.Properties["AllowNCToHostCommunication"] = fmt.Sprintf("%t", createNetworkContainerRequest.AllowNCToHostCommunication) - aiEvent.Properties["AllowHostToNCCommunication"] = fmt.Sprintf("%t", createNetworkContainerRequest.AllowHostToNCCommunication) - aiEvent.Properties["NetworkContainerType"] = createNetworkContainerRequest.NetworkContainerType - aiEvent.Properties["OrchestratorContext"] = fmt.Sprintf("%s", createNetworkContainerRequest.OrchestratorContext) + aiEvent.Properties[ipConfigurationStr] = fmt.Sprintf("%+v", createNetworkContainerRequest.IPConfiguration) + aiEvent.Properties[localIPConfigurationStr] = fmt.Sprintf("%+v", createNetworkContainerRequest.LocalIPConfiguration) + aiEvent.Properties[primaryInterfaceIdentifierStr] = createNetworkContainerRequest.PrimaryInterfaceIdentifier + aiEvent.Properties[multiTenancyInfoStr] = fmt.Sprintf("%+v", createNetworkContainerRequest.MultiTenancyInfo) + aiEvent.Properties[cnetAddressSpaceStr] = fmt.Sprintf("%+v", createNetworkContainerRequest.CnetAddressSpace) + aiEvent.Properties[allowNCToHostCommunicationStr] = fmt.Sprintf("%t", createNetworkContainerRequest.AllowNCToHostCommunication) + aiEvent.Properties[allowHostToNCCommunicationStr] = fmt.Sprintf("%t", createNetworkContainerRequest.AllowHostToNCCommunication) + aiEvent.Properties[networkContainerTypeStr] = createNetworkContainerRequest.NetworkContainerType + aiEvent.Properties[orchestratorContextStr] = fmt.Sprintf("%s", createNetworkContainerRequest.OrchestratorContext) - logger.LogAiEvent(aiEvent) + logger.LogEvent(aiEvent) } // Sends network container snapshots to App Insights telemetry. func (service *HTTPRestService) logNCSnapshots() { - ncCount := 0 for _, ncStatus := range service.state.ContainerStatus { logNCSnapshot(ncStatus.CreateNetworkContainerRequest) - ncCount++ } - logger.Printf("[Azure CNS] Logging periodic NC snapshots. NC Count %d", ncCount) + logger.Printf("[Azure CNS] Logging periodic NC snapshots. NC Count %d", len(service.state.ContainerStatus)) } -func (service *HTTPRestService) sendNCSnapShotPeriodically(ncSnapshotIntervalInMinutes int) { +// Sets up periodic timer for sending network container snapshots +func (service *HTTPRestService) SendNCSnapShotPeriodically(ncSnapshotIntervalInMinutes int, stopSnapshot chan bool) { // Emit snapshot on startup and then emit it periodically. service.logNCSnapshots() @@ -2018,6 +2026,8 @@ func (service *HTTPRestService) sendNCSnapShotPeriodically(ncSnapshotIntervalInM select { case <-snapshot: service.logNCSnapshots() + case <-stopSnapshot: + return } } } diff --git a/cns/service/main.go b/cns/service/main.go index 200651090e..f5df2f8ed9 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -39,6 +39,7 @@ var version string var reports = make(chan interface{}) var telemetryStopProcessing = make(chan bool) var stopheartbeat = make(chan bool) +var stopSnapshots = make(chan bool) // Command line arguments for CNS. var args = acn.ArgumentList{ @@ -254,7 +255,7 @@ func main() { DebugMode: ts.DebugMode, } - logger.InitAI(aiConfig, ts.DisableTrace, ts.DisableMetric) + logger.InitAI(aiConfig, ts.DisableTrace, ts.DisableMetric, ts.DisableEvent) logger.InitReportChannel(reports) } @@ -312,6 +313,7 @@ func main() { if !disableTelemetry { go logger.SendToTelemetryService(reports, telemetryStopProcessing) go logger.SendHeartBeat(cnsconfig.TelemetrySettings.HeartBeatIntervalInMins, stopheartbeat) + go httpRestService.SendNCSnapShotPeriodically(cnsconfig.TelemetrySettings.SnapshotIntervalInMins, stopSnapshots) } var netPlugin network.NetPlugin @@ -387,6 +389,7 @@ func main() { if !disableTelemetry { telemetryStopProcessing <- true stopheartbeat <- true + stopSnapshots <- true } // Cleanup. From ef49759d529f6aa7a1720826b55dbdf73a7055bc Mon Sep 17 00:00:00 2001 From: Sathya Singh <29132953+sathyasi@users.noreply.github.com> Date: Thu, 13 Feb 2020 13:36:36 -0800 Subject: [PATCH 4/4] Addressed review comments. --- aitelemetry/telemetrywrapper.go | 10 ++++++---- cns/logger/constants.go | 11 +++++++++++ cns/logger/log.go | 1 - cns/restserver/restserver.go | 34 ++++++++++----------------------- 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/aitelemetry/telemetrywrapper.go b/aitelemetry/telemetrywrapper.go index 5262618ecf..1047b377f2 100644 --- a/aitelemetry/telemetrywrapper.go +++ b/aitelemetry/telemetrywrapper.go @@ -215,6 +215,7 @@ func (th *telemetryHandle) TrackLog(report Report) { trace.Properties[vmSizeStr] = metadata.VMSize trace.Properties[osVersionStr] = metadata.OSVersion trace.Properties[vmIDStr] = metadata.VMID + trace.Tags.Session().SetId(metadata.VMID) } // send to appinsights resource @@ -277,12 +278,13 @@ func (th *telemetryHandle) TrackMetric(metric Metric) { // Check if metadata is populated if metadata.SubscriptionID != "" { - aimetric.Properties[locationStr] = th.metadata.Location - aimetric.Properties[subscriptionIDStr] = th.metadata.SubscriptionID - aimetric.Properties[vmNameStr] = th.metadata.VMName + aimetric.Properties[locationStr] = metadata.Location + aimetric.Properties[subscriptionIDStr] = metadata.SubscriptionID + aimetric.Properties[vmNameStr] = metadata.VMName aimetric.Properties[versionStr] = th.appVersion aimetric.Properties[resourceGroupStr] = th.metadata.ResourceGroupName - aimetric.Properties[vmIDStr] = th.metadata.VMID + aimetric.Properties[vmIDStr] = metadata.VMID + aimetric.Tags.Session().SetId(metadata.VMID) } // copy custom dimensions diff --git a/cns/logger/constants.go b/cns/logger/constants.go index 7ef87ce12a..3ece449301 100644 --- a/cns/logger/constants.go +++ b/cns/logger/constants.go @@ -8,4 +8,15 @@ const ( //Dimensions OrchestratorTypeStr = "OrchestratorType" NodeIDStr = "NodeID" + // CNS Snspshot properties + CnsNCSnapshotEventStr = "CNSNCSnapshot" + IpConfigurationStr = "IPConfiguration" + LocalIPConfigurationStr = "LocalIPConfiguration" + PrimaryInterfaceIdentifierStr = "PrimaryInterfaceIdentifier" + MultiTenancyInfoStr = "MultiTenancyInfo" + CnetAddressSpaceStr = "CnetAddressSpace" + AllowNCToHostCommunicationStr = "AllowNCToHostCommunication" + AllowHostToNCCommunicationStr = "AllowHostToNCCommunication" + NetworkContainerTypeStr = "NetworkContainerType" + OrchestratorContextStr = "OrchestratorContext" ) diff --git a/cns/logger/log.go b/cns/logger/log.go index 3781e2f264..450551e46e 100644 --- a/cns/logger/log.go +++ b/cns/logger/log.go @@ -107,7 +107,6 @@ func Debugf(format string, args ...interface{}) { } func LogEvent(event aitelemetry.Event) { - if Log.th == nil || Log.DisableEventLogging { return } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 6e0a1aa5c0..0366c0149f 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -32,9 +32,6 @@ import ( var ( // Named Lock for accessing different states in httpRestServiceState namedLock = acn.InitNamedLock() - - // Network container snapshot interval in minutes. - ncSnapshotIntervalInMinutes = 60 ) const ( @@ -45,17 +42,6 @@ const ( detach = "Detach" // Rest service state identifier for named lock stateJoinedNetworks = "JoinedNetworks" - // CNS Snspshot properties - cnsNCSnapshotEventStr = "CNSNCSnapshot" - ipConfigurationStr = "IPConfiguration" - localIPConfigurationStr = "LocalIPConfiguration" - primaryInterfaceIdentifierStr = "PrimaryInterfaceIdentifier" - multiTenancyInfoStr = "MultiTenancyInfo" - cnetAddressSpaceStr = "CnetAddressSpace" - allowNCToHostCommunicationStr = "AllowNCToHostCommunication" - allowHostToNCCommunicationStr = "AllowHostToNCCommunication" - networkContainerTypeStr = "NetworkContainerType" - orchestratorContextStr = "OrchestratorContext" ) // HTTPRestService represents http listener for CNS - Container Networking Service. @@ -1987,20 +1973,20 @@ func (service *HTTPRestService) unpublishNetworkContainer(w http.ResponseWriter, func logNCSnapshot(createNetworkContainerRequest cns.CreateNetworkContainerRequest) { var aiEvent = aitelemetry.Event{ - EventName: cnsNCSnapshotEventStr, + EventName: logger.CnsNCSnapshotEventStr, Properties: make(map[string]string), ResourceID: createNetworkContainerRequest.NetworkContainerid, } - aiEvent.Properties[ipConfigurationStr] = fmt.Sprintf("%+v", createNetworkContainerRequest.IPConfiguration) - aiEvent.Properties[localIPConfigurationStr] = fmt.Sprintf("%+v", createNetworkContainerRequest.LocalIPConfiguration) - aiEvent.Properties[primaryInterfaceIdentifierStr] = createNetworkContainerRequest.PrimaryInterfaceIdentifier - aiEvent.Properties[multiTenancyInfoStr] = fmt.Sprintf("%+v", createNetworkContainerRequest.MultiTenancyInfo) - aiEvent.Properties[cnetAddressSpaceStr] = fmt.Sprintf("%+v", createNetworkContainerRequest.CnetAddressSpace) - aiEvent.Properties[allowNCToHostCommunicationStr] = fmt.Sprintf("%t", createNetworkContainerRequest.AllowNCToHostCommunication) - aiEvent.Properties[allowHostToNCCommunicationStr] = fmt.Sprintf("%t", createNetworkContainerRequest.AllowHostToNCCommunication) - aiEvent.Properties[networkContainerTypeStr] = createNetworkContainerRequest.NetworkContainerType - aiEvent.Properties[orchestratorContextStr] = fmt.Sprintf("%s", createNetworkContainerRequest.OrchestratorContext) + aiEvent.Properties[logger.IpConfigurationStr] = fmt.Sprintf("%+v", createNetworkContainerRequest.IPConfiguration) + aiEvent.Properties[logger.LocalIPConfigurationStr] = fmt.Sprintf("%+v", createNetworkContainerRequest.LocalIPConfiguration) + aiEvent.Properties[logger.PrimaryInterfaceIdentifierStr] = createNetworkContainerRequest.PrimaryInterfaceIdentifier + aiEvent.Properties[logger.MultiTenancyInfoStr] = fmt.Sprintf("%+v", createNetworkContainerRequest.MultiTenancyInfo) + aiEvent.Properties[logger.CnetAddressSpaceStr] = fmt.Sprintf("%+v", createNetworkContainerRequest.CnetAddressSpace) + aiEvent.Properties[logger.AllowNCToHostCommunicationStr] = fmt.Sprintf("%t", createNetworkContainerRequest.AllowNCToHostCommunication) + aiEvent.Properties[logger.AllowHostToNCCommunicationStr] = fmt.Sprintf("%t", createNetworkContainerRequest.AllowHostToNCCommunication) + aiEvent.Properties[logger.NetworkContainerTypeStr] = createNetworkContainerRequest.NetworkContainerType + aiEvent.Properties[logger.OrchestratorContextStr] = fmt.Sprintf("%s", createNetworkContainerRequest.OrchestratorContext) logger.LogEvent(aiEvent) }