From 57d7c8612b35e2248eab8fbd6332fbfffc1ae264 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 12 Dec 2019 16:01:04 -0800 Subject: [PATCH 01/17] Added AITelemetry support for CNI --- Makefile | 4 +- aitelemetry/telemetrywrapper.go | 2 + cni/network/network.go | 61 ++++++++++++++++++++++++-- cni/network/plugin/main.go | 25 +++++++---- cni/telemetry/service/telemetrymain.go | 13 ++++++ telemetry/telemetry.go | 20 +++++++++ telemetry/telemetrybuffer.go | 10 ++++- 7 files changed, 122 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index bbba1ba616..876ec81759 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,8 @@ CNI_MULTITENANCY_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy CNS_BUILD_DIR = $(BUILD_DIR)/cns NPM_BUILD_DIR = $(BUILD_DIR)/npm NPM_TELEMETRY_DIR = $(NPM_BUILD_DIR)/telemetry +CNI_AI_ID = 5515a1eb-b2bc-406a-98eb-ba462e6f0411 +ACN_PACKAGE_PATH = github.com/Azure/azure-container-networking # Containerized build parameters. BUILD_CONTAINER_IMAGE = acn-build @@ -165,7 +167,7 @@ $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT): $(CNIFILES) # Build the Azure CNI telemetry plugin. $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT): $(CNIFILES) - go build -v -o $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNI_TELEMETRY_DIR)/*.go + go build -v -o $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -X $(ACN_PACKAGE_PATH)/telemetry.aiMetadata=$(CNI_AI_ID) -s -w" $(CNI_TELEMETRY_DIR)/*.go # Build the Azure CNS Service. $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT): $(CNSFILES) diff --git a/aitelemetry/telemetrywrapper.go b/aitelemetry/telemetrywrapper.go index 4d69f85c37..c8bd38b985 100644 --- a/aitelemetry/telemetrywrapper.go +++ b/aitelemetry/telemetrywrapper.go @@ -18,6 +18,7 @@ const ( appNameStr = "AppName" subscriptionIDStr = "SubscriptionID" vmNameStr = "VMName" + versionStr = "AppVersion" defaultTimeout = 10 ) @@ -167,6 +168,7 @@ func (th *telemetryHandle) TrackMetric(metric Metric) { aimetric.Properties[locationStr] = th.metadata.Location aimetric.Properties[subscriptionIDStr] = th.metadata.SubscriptionID aimetric.Properties[vmNameStr] = th.metadata.VMName + aimetric.Properties[versionStr] = th.appVersion } // copy custom dimensions diff --git a/cni/network/network.go b/cni/network/network.go index da6e72a3a4..b1b7d15892 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -10,9 +10,11 @@ import ( "net" "net/http" "os" + "runtime" "strings" "time" + "github.com/Azure/azure-container-networking/aitelemetry" "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/cnsclient" @@ -60,6 +62,7 @@ type netPlugin struct { *cni.Plugin nm network.NetworkManager report *telemetry.CNIReport + tb *telemetry.TelemetryBuffer } // snatConfiguration contains a bool that determines whether CNI enables snat on host and snat for dns @@ -90,8 +93,9 @@ func NewPlugin(name string, config *common.PluginConfig) (*netPlugin, error) { }, nil } -func (plugin *netPlugin) SetCNIReport(report *telemetry.CNIReport) { +func (plugin *netPlugin) SetCNIReport(report *telemetry.CNIReport, tb *telemetry.TelemetryBuffer) { plugin.report = report + plugin.tb = tb } // Starts the plugin. @@ -186,6 +190,23 @@ func (plugin *netPlugin) getPodInfo(args string) (string, string, error) { return k8sPodName, k8sNamespace, nil } +func setCustomDimensions(cniMetric *telemetry.CNIReport, nwCfg *cni.NetworkConfig, err error) { + if err != nil { + cniMetric.Metric.CustomDimensions[telemetry.StatusStr] = telemetry.FailedStr + } else { + cniMetric.Metric.CustomDimensions[telemetry.StatusStr] = telemetry.SucceededStr + } + + if nwCfg.MultiTenancy { + cniMetric.Metric.CustomDimensions[telemetry.CNIModeStr] = telemetry.MultiTenancyStr + } else { + cniMetric.Metric.CustomDimensions[telemetry.CNIModeStr] = telemetry.SingleTenancyStr + } + + cniMetric.Metric.CustomDimensions[telemetry.CNINetworkModeStr] = nwCfg.Mode + cniMetric.Metric.CustomDimensions[telemetry.OSTypeStr] = runtime.GOOS +} + func (plugin *netPlugin) setCNIReportDetails(nwCfg *cni.NetworkConfig, opType string, msg string) { if nwCfg.MultiTenancy { plugin.report.Context = "AzureCNIMultitenancy" @@ -218,8 +239,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { enableInfraVnet bool enableSnatForDns bool nwDNSInfo network.DNSInfo + cniMetric telemetry.CNIReport ) + startTime := time.Now() + log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v StdinData:%s}.", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) @@ -242,6 +266,15 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { plugin.setCNIReportDetails(nwCfg, CNI_ADD, "") defer func() { + operationTimeMs := time.Since(startTime).Milliseconds() + cniMetric.Metric = aitelemetry.Metric{ + Name: telemetry.CNIAddTimeMetricStr, + Value: float64(operationTimeMs), + CustomDimensions: make(map[string]string), + } + setCustomDimensions(&cniMetric, nwCfg, err) + telemetry.SendCNIMetric(&cniMetric, plugin.tb) + // Add Interfaces to result. if result == nil { result = &cniTypesCurr.Result{} @@ -252,9 +285,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } result.Interfaces = append(result.Interfaces, iface) - addSnatInterface(nwCfg, result) - // Convert result to the requested CNI version. res, vererr := result.GetAsVersion(nwCfg.CNIVersion) if vererr != nil { @@ -650,8 +681,11 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { networkId string nwInfo *network.NetworkInfo epInfo *network.EndpointInfo + cniMetric telemetry.CNIReport ) + startTime := time.Now() + log.Printf("[cni-net] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v, StdinData:%s}.", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) @@ -730,6 +764,15 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { msg := fmt.Sprintf("CNI DEL succeeded : Released ip %+v podname %v namespace %v", nwCfg.Ipam.Address, k8sPodName, k8sNamespace) plugin.setCNIReportDetails(nwCfg, CNI_DEL, msg) + operationTimeMs := time.Since(startTime).Milliseconds() + cniMetric.Metric = aitelemetry.Metric{ + Name: telemetry.CNIDelTimeMetricStr, + Value: float64(operationTimeMs), + CustomDimensions: make(map[string]string), + } + setCustomDimensions(&cniMetric, nwCfg, nil) + telemetry.SendCNIMetric(&cniMetric, plugin.tb) + return nil } @@ -745,8 +788,11 @@ func (plugin *netPlugin) Update(args *cniSkel.CmdArgs) error { cnsClient *cnsclient.CNSClient orchestratorContext []byte targetNetworkConfig *cns.GetNetworkContainerResponse + cniMetric telemetry.CNIReport ) + startTime := time.Now() + log.Printf("[cni-net] Processing UPDATE command with args {Netns:%v Args:%v Path:%v}.", args.Netns, args.Args, args.Path) @@ -761,6 +807,15 @@ func (plugin *netPlugin) Update(args *cniSkel.CmdArgs) error { plugin.setCNIReportDetails(nwCfg, CNI_UPDATE, "") defer func() { + operationTimeMs := time.Since(startTime).Milliseconds() + cniMetric.Metric = aitelemetry.Metric{ + Name: telemetry.CNIUpdateTimeMetricStr, + Value: float64(operationTimeMs), + CustomDimensions: make(map[string]string), + } + setCustomDimensions(&cniMetric, nwCfg, err) + telemetry.SendCNIMetric(&cniMetric, plugin.tb) + if result == nil { result = &cniTypesCurr.Result{} } diff --git a/cni/network/plugin/main.go b/cni/network/plugin/main.go index 53a7d438e4..27134ef49c 100644 --- a/cni/network/plugin/main.go +++ b/cni/network/plugin/main.go @@ -11,6 +11,7 @@ import ( "reflect" "time" + "github.com/Azure/azure-container-networking/aitelemetry" "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cni/network" "github.com/Azure/azure-container-networking/common" @@ -126,6 +127,8 @@ func handleIfCniUpdate(update func(*skel.CmdArgs) error) (bool, error) { // Main is the entry point for CNI network plugin. func main() { + startTime := time.Now() + // Initialize and parse command line arguments. acn.ParseArgs(&args, printVersion) vers := acn.GetArg(acn.OptVersion).(bool) @@ -136,8 +139,9 @@ func main() { } var ( - config common.PluginConfig - err error + config common.PluginConfig + err error + cnimetric telemetry.CNIReport ) log.SetName(name) @@ -169,7 +173,6 @@ func main() { } cniReport.GetReport(pluginName, version, ipamQueryURL) - startTime := time.Now().UnixNano() / int64(time.Millisecond) netPlugin, err := network.NewPlugin(name, &config) if err != nil { @@ -177,8 +180,6 @@ func main() { return } - netPlugin.SetCNIReport(cniReport) - // CNI Acquires lock if err = netPlugin.Plugin.InitializeKeyValueStore(&config); err != nil { log.Errorf("Failed to initialize key-value store of network plugin, err:%v.\n", err) @@ -214,6 +215,8 @@ func main() { tb.ConnectToTelemetryService(telemetryNumRetries, telemetryWaitTimeInMilliseconds) defer tb.Close() + netPlugin.SetCNIReport(cniReport, tb) + t := time.Now() cniReport.Timestamp = t.Format("2006-01-02 15:04:05") @@ -230,9 +233,6 @@ func main() { log.Errorf("Failed to execute network plugin, err:%v.\n", err) } - endTime := time.Now().UnixNano() / int64(time.Millisecond) - reflect.ValueOf(reportManager.Report).Elem().FieldByName("OperationDuration").SetInt(int64(endTime - startTime)) - netPlugin.Stop() // release cni lock @@ -245,8 +245,17 @@ func main() { panic("network plugin execute fatal error") } + executionTimeMs := time.Since(startTime).Milliseconds() + cnimetric.Metric = aitelemetry.Metric{ + Name: telemetry.CNIExecutimeMetricStr, + Value: float64(executionTimeMs), + CustomDimensions: make(map[string]string), + } + telemetry.SendCNIMetric(&cnimetric, tb) + // Report CNI successfully finished execution. reflect.ValueOf(reportManager.Report).Elem().FieldByName("CniSucceeded").SetBool(true) + reflect.ValueOf(reportManager.Report).Elem().FieldByName("OperationDuration").SetInt(executionTimeMs) if err = reportManager.SendReport(tb); err != nil { log.Errorf("SendReport failed due to %v", err) diff --git a/cni/telemetry/service/telemetrymain.go b/cni/telemetry/service/telemetrymain.go index 07cc5e08a3..d032eee73c 100644 --- a/cni/telemetry/service/telemetrymain.go +++ b/cni/telemetry/service/telemetrymain.go @@ -8,6 +8,7 @@ import ( "runtime" "time" + "github.com/Azure/azure-container-networking/aitelemetry" acn "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/telemetry" @@ -15,6 +16,7 @@ import ( const ( reportToHostIntervalInSeconds = 30 + pluginName = "AzureCNI" azureVnetTelemetry = "azure-vnet-telemetry" configExtension = ".config" ) @@ -145,7 +147,18 @@ func main() { config.ReportToHostIntervalInSeconds = reportToHostIntervalInSeconds } + aiConfig := aitelemetry.AIConfig{ + AppName: pluginName, + AppVersion: version, + BatchSize: config.BatchSizeInBytes, + BatchInterval: config.BatchIntervalInSecs, + DisableMetadataRefreshThread: config.DisableMetadataThread, + DebugMode: config.DebugMode, + } + + telemetry.CreateAITelemetryHandle(aiConfig, config.DisableAll, config.DisableTrace, config.DisableMetric) log.Logf("[Telemetry] Report to host for an interval of %d seconds", config.ReportToHostIntervalInSeconds) tb.BufferAndPushData(config.ReportToHostIntervalInSeconds * time.Second) + telemetry.CloseAITelemetryHandle() log.Close() } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 397017a142..05614bdcb1 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -14,6 +14,7 @@ import ( "reflect" "strings" + "github.com/Azure/azure-container-networking/aitelemetry" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/platform" @@ -98,6 +99,7 @@ type CNIReport struct { InterfaceDetails InterfaceInfo BridgeDetails BridgeInfo Metadata common.Metadata `json:"compute"` + Metric aitelemetry.Metric } // Azure CNS Telemetry Report structure. @@ -383,3 +385,21 @@ func (reportMgr *ReportManager) ReportToBytes() ([]byte, error) { report, err = json.Marshal(reportMgr.Report) return report, err } + +func SendCNIMetric(cniReport *CNIReport, tb *TelemetryBuffer) error { + var err error + var report []byte + + if tb != nil && tb.Connected { + reportMgr := &ReportManager{Report: cniReport} + report, err = reportMgr.ReportToBytes() + if err == nil { + // If write fails, try to re-establish connections as server/client + if _, err = tb.Write(report); err != nil { + tb.Cancel() + } + } + } + + return err +} diff --git a/telemetry/telemetrybuffer.go b/telemetry/telemetrybuffer.go index 441f44aec3..db5557df29 100644 --- a/telemetry/telemetrybuffer.go +++ b/telemetry/telemetrybuffer.go @@ -28,6 +28,14 @@ import ( // TelemetryConfig - telemetry config read by telemetry service type TelemetryConfig struct { ReportToHostIntervalInSeconds time.Duration `json:"reportToHostIntervalInSeconds"` + DisableAll bool + DisableTrace bool + DisableMetric bool + DisableMetadataThread bool + DebugMode bool + RefreshTimeout int + BatchIntervalInSecs int + BatchSizeInBytes int } // FdName - file descriptor name @@ -443,7 +451,7 @@ func (buf *Buffer) push(x interface{}) { } cniReport := x.(CNIReport) cniReport.Metadata = metadata - buf.CNIReports = append(buf.CNIReports, cniReport) + SendAITelemetry(cniReport) case NPMReport: if len(buf.NPMReports) >= MaxNumReports { return From bb3c47ecd49fcd0825a89f98d14bd2e77efb780f Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 12 Dec 2019 16:01:55 -0800 Subject: [PATCH 02/17] added new files --- telemetry/aiwrapper.go | 65 ++++++++++++++++++++++++++++++++++++++++++ telemetry/constants.go | 27 ++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 telemetry/aiwrapper.go create mode 100644 telemetry/constants.go diff --git a/telemetry/aiwrapper.go b/telemetry/aiwrapper.go new file mode 100644 index 0000000000..bf6174f48f --- /dev/null +++ b/telemetry/aiwrapper.go @@ -0,0 +1,65 @@ +package telemetry + +import ( + "github.com/Azure/azure-container-networking/aitelemetry" + "github.com/Azure/azure-container-networking/log" +) + +var ( + aiMetadata string + th aitelemetry.TelemetryHandle + gDisableAll bool + gDisableTrace bool + gDisableMetric bool +) + +const ( + waitTimeInSecs = 10 +) + +func CreateAITelemetryHandle(aiConfig aitelemetry.AIConfig, disableAll, disableMetric, disableTrace bool) { + th = aitelemetry.NewAITelemetry(aiMetadata, aiConfig) + gDisableAll = disableAll + gDisableMetric = disableMetric + gDisableTrace = disableTrace +} + +func sendReport(cnireport CNIReport) { + var msg string + if cnireport.ErrorMessage != "" { + msg = cnireport.ErrorMessage + } else { + msg = cnireport.EventMessage + } + + report := aitelemetry.Report{ + Message: "CNI:" + msg, + Context: cnireport.ContainerName, + CustomDimensions: make(map[string]string), + } + + report.CustomDimensions[ContextStr] = cnireport.Context + report.CustomDimensions[SubContextStr] = cnireport.SubContext + report.CustomDimensions[VMUptimeStr] = cnireport.VMUptime + report.CustomDimensions[OperationTypeStr] = cnireport.OperationType + report.CustomDimensions[VersionStr] = cnireport.Version + + th.TrackLog(report) +} + +func SendAITelemetry(cnireport CNIReport) { + if gDisableAll { + log.Printf("Sending Telemetry data is disabled") + return + } + + if cnireport.Metric.Name != "" && !gDisableMetric { + th.TrackMetric(cnireport.Metric) + } else if !gDisableTrace { + sendReport(cnireport) + } +} + +func CloseAITelemetryHandle() { + th.Close(waitTimeInSecs) +} diff --git a/telemetry/constants.go b/telemetry/constants.go new file mode 100644 index 0000000000..0a45edc59a --- /dev/null +++ b/telemetry/constants.go @@ -0,0 +1,27 @@ +package telemetry + +const ( + + // Metric Names + CNIExecutimeMetricStr = "CNIExecutionTimeMs" + CNIAddTimeMetricStr = "CNIAddTimeMs" + CNIDelTimeMetricStr = "CNIDelTimeMs" + CNIUpdateTimeMetricStr = "CNIUpdateTimeMs" + + // Dimension Names + ContextStr = "Context" + SubContextStr = "SubContext" + VMUptimeStr = "VMUptime" + OperationTypeStr = "OperationType" + VersionStr = "Version" + StatusStr = "Status" + CNIModeStr = "CNIMode" + CNINetworkModeStr = "CNINetworkMode" + OSTypeStr = "OSType" + + // Values + SucceededStr = "Succeeded" + FailedStr = "Failed" + SingleTenancyStr = "SingleTenancy" + MultiTenancyStr = "MultiTenancy" +) From 94211ceb4133f3cac2a0d427dac31812a3727d97 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 12 Dec 2019 16:16:28 -0800 Subject: [PATCH 03/17] added other configs in config file --- cni/telemetry/service/telemetrymain.go | 1 + telemetry/azure-vnet-telemetry.config | 10 +++++++++- telemetry/telemetrybuffer.go | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cni/telemetry/service/telemetrymain.go b/cni/telemetry/service/telemetrymain.go index d032eee73c..86fde16b06 100644 --- a/cni/telemetry/service/telemetrymain.go +++ b/cni/telemetry/service/telemetrymain.go @@ -152,6 +152,7 @@ func main() { AppVersion: version, BatchSize: config.BatchSizeInBytes, BatchInterval: config.BatchIntervalInSecs, + RefreshTimeout: config.RefreshTimeoutInSecs, DisableMetadataRefreshThread: config.DisableMetadataThread, DebugMode: config.DebugMode, } diff --git a/telemetry/azure-vnet-telemetry.config b/telemetry/azure-vnet-telemetry.config index dff89c8cf4..5be244fece 100644 --- a/telemetry/azure-vnet-telemetry.config +++ b/telemetry/azure-vnet-telemetry.config @@ -1,3 +1,11 @@ { - "reportToHostIntervalInSeconds": 30 + "reportToHostIntervalInSeconds": 30, + "BatchSizeInBytes":16384, + "BatchIntervalInSecs":15, + "RefreshTimeoutInSecs": 15, + "DisableMetadataThread": false, + "DisableAll": false, + "DisableTrace": false, + "DisableMetric": false, + "DebugMode":true } \ No newline at end of file diff --git a/telemetry/telemetrybuffer.go b/telemetry/telemetrybuffer.go index db5557df29..f31643a30b 100644 --- a/telemetry/telemetrybuffer.go +++ b/telemetry/telemetrybuffer.go @@ -33,7 +33,7 @@ type TelemetryConfig struct { DisableMetric bool DisableMetadataThread bool DebugMode bool - RefreshTimeout int + RefreshTimeoutInSecs int BatchIntervalInSecs int BatchSizeInBytes int } From bae19de3b3d2d637a580dbd2ba4b8a3a2c4d79cb Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 13 Dec 2019 16:27:38 -0800 Subject: [PATCH 04/17] fixed ut --- telemetry/telemetry_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index 4df069ea00..60dc8e3629 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -186,13 +186,6 @@ func TestSendTelemetry(t *testing.T) { } } -func TestReceiveTelemetryData(t *testing.T) { - time.Sleep(300 * time.Millisecond) - if len(tb.buffer.CNIReports) != 1 { - t.Errorf("buffer doesn't contain CNI report") - } -} - func TestCloseTelemetryConnection(t *testing.T) { tb.Cancel() time.Sleep(300 * time.Millisecond) From 91de86e102c4d14e282b3a2c72bd835a8d47ac5c Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 13 Dec 2019 17:01:34 -0800 Subject: [PATCH 05/17] updated disableall similar to cns --- cni/telemetry/service/telemetrymain.go | 1 + telemetry/aiwrapper.go | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cni/telemetry/service/telemetrymain.go b/cni/telemetry/service/telemetrymain.go index 86fde16b06..482893fe21 100644 --- a/cni/telemetry/service/telemetrymain.go +++ b/cni/telemetry/service/telemetrymain.go @@ -161,5 +161,6 @@ func main() { log.Logf("[Telemetry] Report to host for an interval of %d seconds", config.ReportToHostIntervalInSeconds) tb.BufferAndPushData(config.ReportToHostIntervalInSeconds * time.Second) telemetry.CloseAITelemetryHandle() + log.Close() } diff --git a/telemetry/aiwrapper.go b/telemetry/aiwrapper.go index bf6174f48f..9175f73f20 100644 --- a/telemetry/aiwrapper.go +++ b/telemetry/aiwrapper.go @@ -8,7 +8,6 @@ import ( var ( aiMetadata string th aitelemetry.TelemetryHandle - gDisableAll bool gDisableTrace bool gDisableMetric bool ) @@ -18,8 +17,12 @@ const ( ) func CreateAITelemetryHandle(aiConfig aitelemetry.AIConfig, disableAll, disableMetric, disableTrace bool) { + if disableAll { + log.Printf("Telemetry is disabled") + return + } + th = aitelemetry.NewAITelemetry(aiMetadata, aiConfig) - gDisableAll = disableAll gDisableMetric = disableMetric gDisableTrace = disableTrace } @@ -48,8 +51,7 @@ func sendReport(cnireport CNIReport) { } func SendAITelemetry(cnireport CNIReport) { - if gDisableAll { - log.Printf("Sending Telemetry data is disabled") + if th == nil { return } @@ -61,5 +63,7 @@ func SendAITelemetry(cnireport CNIReport) { } func CloseAITelemetryHandle() { - th.Close(waitTimeInSecs) + if th != nil { + th.Close(waitTimeInSecs) + } } From 61e623ffc87e08915cc4a9ce8a32705b89dd1a97 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 16 Dec 2019 11:58:08 -0800 Subject: [PATCH 06/17] added container name to report --- cni/network/network.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cni/network/network.go b/cni/network/network.go index b1b7d15892..0a197eaf99 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -307,6 +307,8 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { return err } + plugin.report.ContainerName = k8sPodName + ":" + k8sNamespace + if nwCfg.MultiTenancy { // Initialize CNSClient cnsclient.InitCnsClient(nwCfg.CNSUrl) From 774ab5cfb4aac6484b469d1655699b7f4a63425e Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 18 Dec 2019 16:42:48 -0800 Subject: [PATCH 07/17] addressed review comments --- cni/telemetry/service/telemetrymain.go | 35 ++++++++++++++++++++------ telemetry/aiwrapper.go | 2 ++ telemetry/azure-vnet-telemetry.config | 7 ++---- telemetry/constants.go | 2 ++ telemetry/telemetry.go | 7 ++++-- 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/cni/telemetry/service/telemetrymain.go b/cni/telemetry/service/telemetrymain.go index 482893fe21..d023a475ab 100644 --- a/cni/telemetry/service/telemetrymain.go +++ b/cni/telemetry/service/telemetrymain.go @@ -15,10 +15,13 @@ import ( ) const ( - reportToHostIntervalInSeconds = 30 - pluginName = "AzureCNI" - azureVnetTelemetry = "azure-vnet-telemetry" - configExtension = ".config" + defaultReportToHostIntervalInSecs = 30 + defaultRefreshTimeoutInSecs = 15 + defaultBatchSizeInBytes = 16384 + defaultBatchIntervalInSecs = 15 + pluginName = "AzureCNI" + azureVnetTelemetry = "azure-vnet-telemetry" + configExtension = ".config" ) var version string @@ -78,6 +81,24 @@ func printVersion() { fmt.Printf("Version %v\n", version) } +func setTelemetryDefaults(config *telemetry.TelemetryConfig) { + if config.ReportToHostIntervalInSeconds == 0 { + config.ReportToHostIntervalInSeconds = defaultReportToHostIntervalInSecs + } + + if config.RefreshTimeoutInSecs == 0 { + config.RefreshTimeoutInSecs = defaultRefreshTimeoutInSecs + } + + if config.BatchIntervalInSecs == 0 { + config.BatchIntervalInSecs = defaultBatchIntervalInSecs + } + + if config.BatchSizeInBytes == 0 { + config.BatchSizeInBytes = defaultBatchSizeInBytes + } +} + func main() { var tb *telemetry.TelemetryBuffer var config telemetry.TelemetryConfig @@ -125,6 +146,8 @@ func main() { log.Logf("read config returned %+v", config) + setTelemetryDefaults(&config) + // Cleaning up orphan socket if present tbtemp := telemetry.NewTelemetryBuffer("") tbtemp.Cleanup(telemetry.FdName) @@ -143,10 +166,6 @@ func main() { time.Sleep(time.Millisecond * 200) } - if config.ReportToHostIntervalInSeconds == 0 { - config.ReportToHostIntervalInSeconds = reportToHostIntervalInSeconds - } - aiConfig := aitelemetry.AIConfig{ AppName: pluginName, AppVersion: version, diff --git a/telemetry/aiwrapper.go b/telemetry/aiwrapper.go index 9175f73f20..042991c5b3 100644 --- a/telemetry/aiwrapper.go +++ b/telemetry/aiwrapper.go @@ -1,3 +1,4 @@ +// Copyright Microsoft. All rights reserved. package telemetry import ( @@ -13,6 +14,7 @@ var ( ) const ( + // Wait time for AI to gracefully close AI telemetry session waitTimeInSecs = 10 ) diff --git a/telemetry/azure-vnet-telemetry.config b/telemetry/azure-vnet-telemetry.config index 5be244fece..daf0c91406 100644 --- a/telemetry/azure-vnet-telemetry.config +++ b/telemetry/azure-vnet-telemetry.config @@ -3,9 +3,6 @@ "BatchSizeInBytes":16384, "BatchIntervalInSecs":15, "RefreshTimeoutInSecs": 15, - "DisableMetadataThread": false, "DisableAll": false, - "DisableTrace": false, - "DisableMetric": false, - "DebugMode":true -} \ No newline at end of file + "DebugMode":false +} diff --git a/telemetry/constants.go b/telemetry/constants.go index 0a45edc59a..9fadc585d6 100644 --- a/telemetry/constants.go +++ b/telemetry/constants.go @@ -1,3 +1,5 @@ +// Copyright Microsoft. All rights reserved. + package telemetry const ( diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 05614bdcb1..b398391a77 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -386,9 +386,12 @@ func (reportMgr *ReportManager) ReportToBytes() ([]byte, error) { return report, err } +// This function for sending CNI metrics to telemetry service func SendCNIMetric(cniReport *CNIReport, tb *TelemetryBuffer) error { - var err error - var report []byte + var ( + err error + report []byte + ) if tb != nil && tb.Connected { reportMgr := &ReportManager{Report: cniReport} From 903c0ec69fcb698f8c309c8859be604605f703ea Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 18 Dec 2019 18:16:39 -0800 Subject: [PATCH 08/17] addressed review comments --- cni/network/network.go | 29 +++++++++++++++++------------ cni/network/plugin/main.go | 11 ++++++----- telemetry/aiwrapper.go | 3 +++ 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 0a197eaf99..7f2616b1da 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -10,7 +10,6 @@ import ( "net" "net/http" "os" - "runtime" "strings" "time" @@ -190,21 +189,27 @@ func (plugin *netPlugin) getPodInfo(args string) (string, string, error) { return k8sPodName, k8sNamespace, nil } -func setCustomDimensions(cniMetric *telemetry.CNIReport, nwCfg *cni.NetworkConfig, err error) { +func SetCustomDimensions(cniMetric *telemetry.CNIReport, nwCfg *cni.NetworkConfig, err error) { + if cniMetric == nil { + log.Errorf("[CNI] Unable to set custom dimension. Report is nil") + return + } + if err != nil { cniMetric.Metric.CustomDimensions[telemetry.StatusStr] = telemetry.FailedStr } else { cniMetric.Metric.CustomDimensions[telemetry.StatusStr] = telemetry.SucceededStr } - if nwCfg.MultiTenancy { - cniMetric.Metric.CustomDimensions[telemetry.CNIModeStr] = telemetry.MultiTenancyStr - } else { - cniMetric.Metric.CustomDimensions[telemetry.CNIModeStr] = telemetry.SingleTenancyStr - } + if nwCfg != nil { + if nwCfg.MultiTenancy { + cniMetric.Metric.CustomDimensions[telemetry.CNIModeStr] = telemetry.MultiTenancyStr + } else { + cniMetric.Metric.CustomDimensions[telemetry.CNIModeStr] = telemetry.SingleTenancyStr + } - cniMetric.Metric.CustomDimensions[telemetry.CNINetworkModeStr] = nwCfg.Mode - cniMetric.Metric.CustomDimensions[telemetry.OSTypeStr] = runtime.GOOS + cniMetric.Metric.CustomDimensions[telemetry.CNINetworkModeStr] = nwCfg.Mode + } } func (plugin *netPlugin) setCNIReportDetails(nwCfg *cni.NetworkConfig, opType string, msg string) { @@ -272,7 +277,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { Value: float64(operationTimeMs), CustomDimensions: make(map[string]string), } - setCustomDimensions(&cniMetric, nwCfg, err) + SetCustomDimensions(&cniMetric, nwCfg, err) telemetry.SendCNIMetric(&cniMetric, plugin.tb) // Add Interfaces to result. @@ -772,7 +777,7 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { Value: float64(operationTimeMs), CustomDimensions: make(map[string]string), } - setCustomDimensions(&cniMetric, nwCfg, nil) + SetCustomDimensions(&cniMetric, nwCfg, nil) telemetry.SendCNIMetric(&cniMetric, plugin.tb) return nil @@ -815,7 +820,7 @@ func (plugin *netPlugin) Update(args *cniSkel.CmdArgs) error { Value: float64(operationTimeMs), CustomDimensions: make(map[string]string), } - setCustomDimensions(&cniMetric, nwCfg, err) + SetCustomDimensions(&cniMetric, nwCfg, err) telemetry.SendCNIMetric(&cniMetric, plugin.tb) if result == nil { diff --git a/cni/network/plugin/main.go b/cni/network/plugin/main.go index 27134ef49c..8feb5adc9f 100644 --- a/cni/network/plugin/main.go +++ b/cni/network/plugin/main.go @@ -240,19 +240,20 @@ func main() { log.Errorf("Failed to uninitialize key-value store of network plugin, err:%v.\n", errUninit) } - if err != nil { - reportPluginError(reportManager, tb, err) - panic("network plugin execute fatal error") - } - executionTimeMs := time.Since(startTime).Milliseconds() cnimetric.Metric = aitelemetry.Metric{ Name: telemetry.CNIExecutimeMetricStr, Value: float64(executionTimeMs), CustomDimensions: make(map[string]string), } + network.SetCustomDimensions(&cnimetric, nil, err) telemetry.SendCNIMetric(&cnimetric, tb) + if err != nil { + reportPluginError(reportManager, tb, err) + panic("network plugin execute fatal error") + } + // Report CNI successfully finished execution. reflect.ValueOf(reportManager.Report).Elem().FieldByName("CniSucceeded").SetBool(true) reflect.ValueOf(reportManager.Report).Elem().FieldByName("OperationDuration").SetInt(executionTimeMs) diff --git a/telemetry/aiwrapper.go b/telemetry/aiwrapper.go index 042991c5b3..ef2a4d1da6 100644 --- a/telemetry/aiwrapper.go +++ b/telemetry/aiwrapper.go @@ -2,6 +2,8 @@ package telemetry import ( + "runtime" + "github.com/Azure/azure-container-networking/aitelemetry" "github.com/Azure/azure-container-networking/log" ) @@ -58,6 +60,7 @@ func SendAITelemetry(cnireport CNIReport) { } if cnireport.Metric.Name != "" && !gDisableMetric { + cnireport.Metric.CustomDimensions[OSTypeStr] = runtime.GOOS th.TrackMetric(cnireport.Metric) } else if !gDisableTrace { sendReport(cnireport) From 225e717e699a67573a277b6231f86cc0c8e1bc74 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 31 Dec 2019 13:02:57 -0800 Subject: [PATCH 09/17] added check for azure environment --- aitelemetry/telemetrywrapper.go | 10 ++++++++ aitelemetry/telemetrywrapper_test.go | 14 ++++++++++++ common/utils.go | 34 ++++++++++++++++++++++++++-- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/aitelemetry/telemetrywrapper.go b/aitelemetry/telemetrywrapper.go index c8bd38b985..502348ec89 100644 --- a/aitelemetry/telemetrywrapper.go +++ b/aitelemetry/telemetrywrapper.go @@ -1,6 +1,7 @@ package aitelemetry import ( + "os" "runtime" "time" @@ -19,6 +20,7 @@ const ( subscriptionIDStr = "SubscriptionID" vmNameStr = "VMName" versionStr = "AppVersion" + azurePublicCloud = "AzurePublicCloud" defaultTimeout = 10 ) @@ -90,6 +92,14 @@ func NewAITelemetry( id string, aiConfig AIConfig, ) TelemetryHandle { + env, _ := os.LookupEnv("AZACN_TESTENV") + if env != "test" { + cloudName, err := common.GetAzureCloud() + if cloudName != azurePublicCloud { + debuglog("[AppInsights] This is not azure public cloud.%s err:%v", cloudName, err) + return nil + } + } telemetryConfig := appinsights.NewTelemetryConfiguration(id) telemetryConfig.MaxBatchSize = aiConfig.BatchSize diff --git a/aitelemetry/telemetrywrapper_test.go b/aitelemetry/telemetrywrapper_test.go index baa16faa31..f64bc40409 100644 --- a/aitelemetry/telemetrywrapper_test.go +++ b/aitelemetry/telemetrywrapper_test.go @@ -7,12 +7,20 @@ import ( "runtime" "testing" + "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/platform" ) var th TelemetryHandle func TestMain(m *testing.M) { + log.SetLogDirectory("/var/log/") + log.SetName("testaitelemetry") + log.SetLevel(log.LevelInfo) + err := log.SetTarget(log.TargetLogfile) + if err == nil { + fmt.Printf("TestST LogDir configuration succeeded\n") + } if runtime.GOOS == "linux" { platform.ExecuteCommand("cp metadata_test.json /tmp/azuremetadata.json") @@ -36,6 +44,8 @@ func TestMain(m *testing.M) { } func TestEmptyAIKey(t *testing.T) { + os.Setenv("AZACN_TESTENV", "test") + aiConfig := AIConfig{ AppName: "testapp", AppVersion: "v1.0.26", @@ -53,6 +63,8 @@ func TestEmptyAIKey(t *testing.T) { } func TestNewAITelemetry(t *testing.T) { + os.Setenv("AZACN_TESTENV", "test") + aiConfig := AIConfig{ AppName: "testapp", AppVersion: "v1.0.26", @@ -95,6 +107,8 @@ func TestClose(t *testing.T) { } func TestClosewithoutSend(t *testing.T) { + os.Setenv("AZACN_TESTENV", "test") + aiConfig := AIConfig{ AppName: "testapp", AppVersion: "v1.0.26", diff --git a/common/utils.go b/common/utils.go index 8b08a37ba3..9f3ed44314 100644 --- a/common/utils.go +++ b/common/utils.go @@ -14,6 +14,7 @@ import ( "net" "net/http" "os" + "strings" "time" "github.com/Azure/azure-container-networking/log" @@ -21,8 +22,9 @@ import ( const ( metadataURL = "http://169.254.169.254/metadata/instance?api-version=2017-08-01&format=json" - httpConnectionTimeout = 10 - headerTimeout = 20 + azCloudUrl = "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2018-10-01&format=text" + httpConnectionTimeout = 7 + headerTimeout = 7 ) // XmlDocument - Azure host agent XML document format. @@ -288,3 +290,31 @@ func SaveHostMetadata(metadata Metadata, fileName string) error { return err } + +func GetAzureCloud() (string, error) { + req, err := http.NewRequest("GET", azCloudUrl, nil) + if err != nil { + return "", err + } + + req.Header.Set("Metadata", "True") + + client := InitHttpClient(httpConnectionTimeout, headerTimeout) + resp, err := client.Do(req) + if err != nil { + return "", err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("Bad http status:%v", resp.Status) + } + + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return strings.TrimSpace(string(bodyBytes)), nil +} From 090760eabe49c8d357151fa83b840a307b0b5d00 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 3 Jan 2020 11:39:49 -0800 Subject: [PATCH 10/17] added log --- aitelemetry/telemetrywrapper.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aitelemetry/telemetrywrapper.go b/aitelemetry/telemetrywrapper.go index 502348ec89..66d3a7c2c0 100644 --- a/aitelemetry/telemetrywrapper.go +++ b/aitelemetry/telemetrywrapper.go @@ -92,6 +92,8 @@ func NewAITelemetry( id string, aiConfig AIConfig, ) TelemetryHandle { + debugMode = aiConfig.DebugMode + env, _ := os.LookupEnv("AZACN_TESTENV") if env != "test" { cloudName, err := common.GetAzureCloud() @@ -99,12 +101,13 @@ func NewAITelemetry( debuglog("[AppInsights] This is not azure public cloud.%s err:%v", cloudName, err) return nil } + + log.Printf("[AppInsights] CloudName: %s\n", cloudName) } telemetryConfig := appinsights.NewTelemetryConfiguration(id) telemetryConfig.MaxBatchSize = aiConfig.BatchSize telemetryConfig.MaxBatchInterval = time.Duration(aiConfig.BatchInterval) * time.Second - debugMode = aiConfig.DebugMode th := &telemetryHandle{ client: appinsights.NewTelemetryClientFromConfig(telemetryConfig), From aa8d7fec5746f54a65500722c8f3c851eafd4a90 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 3 Jan 2020 14:00:39 -0800 Subject: [PATCH 11/17] close log handle in unit test --- aitelemetry/telemetrywrapper_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/aitelemetry/telemetrywrapper_test.go b/aitelemetry/telemetrywrapper_test.go index f64bc40409..58250cebb6 100644 --- a/aitelemetry/telemetrywrapper_test.go +++ b/aitelemetry/telemetrywrapper_test.go @@ -40,6 +40,7 @@ func TestMain(m *testing.M) { platform.ExecuteCommand(cmd) } + log.Close() os.Exit(exitCode) } From d86a65b585c3dbe869952a18ee8b4a8957393228 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 3 Jan 2020 16:37:33 -0800 Subject: [PATCH 12/17] addressed review comments --- aitelemetry/telemetrywrapper.go | 36 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/aitelemetry/telemetrywrapper.go b/aitelemetry/telemetrywrapper.go index 66d3a7c2c0..878fb18490 100644 --- a/aitelemetry/telemetrywrapper.go +++ b/aitelemetry/telemetrywrapper.go @@ -12,16 +12,16 @@ import ( ) const ( - resourceGroupStr = "ResourceGroup" - vmSizeStr = "VMSize" - osVersionStr = "OSVersion" - locationStr = "Region" - appNameStr = "AppName" - subscriptionIDStr = "SubscriptionID" - vmNameStr = "VMName" - versionStr = "AppVersion" - azurePublicCloud = "AzurePublicCloud" - defaultTimeout = 10 + resourceGroupStr = "ResourceGroup" + vmSizeStr = "VMSize" + osVersionStr = "OSVersion" + locationStr = "Region" + appNameStr = "AppName" + subscriptionIDStr = "SubscriptionID" + vmNameStr = "VMName" + versionStr = "AppVersion" + azurePublicCloudStr = "AzurePublicCloud" + defaultTimeout = 10 ) var debugMode bool @@ -29,7 +29,7 @@ var debugMode bool func messageListener() appinsights.DiagnosticsMessageListener { if debugMode { return appinsights.NewDiagnosticsMessageListener(func(msg string) error { - debuglog("[AppInsights] [%s] %s\n", time.Now().Format(time.UnixDate), msg) + debugLog("[AppInsights] [%s] %s\n", time.Now().Format(time.UnixDate), msg) return nil }) } @@ -37,7 +37,7 @@ func messageListener() appinsights.DiagnosticsMessageListener { return nil } -func debuglog(format string, args ...interface{}) { +func debugLog(format string, args ...interface{}) { if debugMode { log.Printf(format, args...) } @@ -58,12 +58,12 @@ func getMetadata(th *telemetryHandle) { break } - debuglog("[AppInsights] Error getting metadata %v. Sleep for %d", err, th.refreshTimeout) + debugLog("[AppInsights] Error getting metadata %v. Sleep for %d", err, th.refreshTimeout) time.Sleep(time.Duration(th.refreshTimeout) * time.Second) } if err != nil { - debuglog("[AppInsights] Error getting metadata %v", err) + debugLog("[AppInsights] Error getting metadata %v", err) return } @@ -75,7 +75,7 @@ func getMetadata(th *telemetryHandle) { // Save metadata retrieved from wireserver to a file kvs, err := store.NewJsonFileStore(metadataFile) if err != nil { - debuglog("[AppInsights] Error initializing kvs store: %v", err) + debugLog("[AppInsights] Error initializing kvs store: %v", err) return } @@ -83,7 +83,7 @@ func getMetadata(th *telemetryHandle) { err = common.SaveHostMetadata(th.metadata, metadataFile) kvs.Unlock(true) if err != nil { - debuglog("[AppInsights] saving host metadata failed with :%v", err) + debugLog("[AppInsights] saving host metadata failed with :%v", err) } } @@ -97,8 +97,8 @@ func NewAITelemetry( env, _ := os.LookupEnv("AZACN_TESTENV") if env != "test" { cloudName, err := common.GetAzureCloud() - if cloudName != azurePublicCloud { - debuglog("[AppInsights] This is not azure public cloud.%s err:%v", cloudName, err) + if cloudName != azurePublicCloudStr { + debugLog("[AppInsights] This is not azure public cloud:%s err:%v", cloudName, err) return nil } From 1b67c8e5017b188b86cadc4cfb6d052aceda366f Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 9 Jan 2020 12:47:47 -0800 Subject: [PATCH 13/17] addressed review comments --- aitelemetry/api.go | 2 ++ aitelemetry/telemetrywrapper.go | 46 ++++++++++++++++++++------ cni/telemetry/service/telemetrymain.go | 15 ++++++++- telemetry/aiwrapper.go | 14 ++++++-- telemetry/telemetrybuffer.go | 3 ++ 5 files changed, 65 insertions(+), 15 deletions(-) diff --git a/aitelemetry/api.go b/aitelemetry/api.go index 3ebe8f6040..d23389edd4 100644 --- a/aitelemetry/api.go +++ b/aitelemetry/api.go @@ -28,6 +28,8 @@ type AIConfig struct { BatchInterval int DisableMetadataRefreshThread bool RefreshTimeout int + GetEnvRetryCount int + GetEnvRetryWaitTimeInSecs int DebugMode bool } diff --git a/aitelemetry/telemetrywrapper.go b/aitelemetry/telemetrywrapper.go index 878fb18490..31bf3f9c7d 100644 --- a/aitelemetry/telemetrywrapper.go +++ b/aitelemetry/telemetrywrapper.go @@ -1,6 +1,7 @@ package aitelemetry import ( + "fmt" "os" "runtime" "time" @@ -87,22 +88,45 @@ func getMetadata(th *telemetryHandle) { } } +func isPublicEnvironment(retryCount, waitTimeInSecs int) (bool, error) { + var ( + cloudName string + err error + ) + + env, _ := os.LookupEnv("AZACN_TESTENV") + if env == "test" { + debugLog("Test environment") + return true, nil + } + + for i := 0; i < retryCount; i++ { + cloudName, err = common.GetAzureCloud() + if cloudName == azurePublicCloudStr { + debugLog("[AppInsights] CloudName: %s\n", cloudName) + return true, nil + } else if err != nil { + debugLog("[AppInsights] This is not azure public cloud:%s", cloudName) + return false, fmt.Errorf("Not an azure public cloud: %s", cloudName) + } + + debugLog("GetAzureCloud returned err :%v", err) + time.Sleep(time.Duration(waitTimeInSecs) * time.Second) + } + + return false, err +} + // NewAITelemetry creates telemetry handle with user specified appinsights id. func NewAITelemetry( id string, aiConfig AIConfig, -) TelemetryHandle { +) (TelemetryHandle, error) { debugMode = aiConfig.DebugMode - env, _ := os.LookupEnv("AZACN_TESTENV") - if env != "test" { - cloudName, err := common.GetAzureCloud() - if cloudName != azurePublicCloudStr { - debugLog("[AppInsights] This is not azure public cloud:%s err:%v", cloudName, err) - return nil - } - - log.Printf("[AppInsights] CloudName: %s\n", cloudName) + isPublic, err := isPublicEnvironment(aiConfig.GetEnvRetryCount, aiConfig.GetEnvRetryWaitTimeInSecs) + if !isPublic { + return nil, err } telemetryConfig := appinsights.NewTelemetryConfiguration(id) @@ -124,7 +148,7 @@ func NewAITelemetry( go getMetadata(th) } - return th + return th, nil } // TrackLog function sends report (trace) to appinsights resource. It overrides few of the existing columns with app information diff --git a/cni/telemetry/service/telemetrymain.go b/cni/telemetry/service/telemetrymain.go index d023a475ab..195dd2da85 100644 --- a/cni/telemetry/service/telemetrymain.go +++ b/cni/telemetry/service/telemetrymain.go @@ -97,6 +97,14 @@ func setTelemetryDefaults(config *telemetry.TelemetryConfig) { if config.BatchSizeInBytes == 0 { config.BatchSizeInBytes = defaultBatchSizeInBytes } + + if config.GetEnvRetryCount == 0 { + config.GetEnvRetryCount = 2 + } + + if config.GetEnvRetryWaitTimeInSecs == 0 { + config.GetEnvRetryWaitTimeInSecs = 3 + } } func main() { @@ -148,6 +156,8 @@ func main() { setTelemetryDefaults(&config) + log.Logf("Config after setting defaults %+v", config) + // Cleaning up orphan socket if present tbtemp := telemetry.NewTelemetryBuffer("") tbtemp.Cleanup(telemetry.FdName) @@ -174,9 +184,12 @@ func main() { RefreshTimeout: config.RefreshTimeoutInSecs, DisableMetadataRefreshThread: config.DisableMetadataThread, DebugMode: config.DebugMode, + GetEnvRetryCount: config.GetEnvRetryCount, + GetEnvRetryWaitTimeInSecs: config.GetEnvRetryWaitTimeInSecs, } - telemetry.CreateAITelemetryHandle(aiConfig, config.DisableAll, config.DisableTrace, config.DisableMetric) + err = telemetry.CreateAITelemetryHandle(aiConfig, config.DisableAll, config.DisableTrace, config.DisableMetric) + log.Printf("[Telemetry] AI Handle creation status:%v", err) log.Logf("[Telemetry] Report to host for an interval of %d seconds", config.ReportToHostIntervalInSeconds) tb.BufferAndPushData(config.ReportToHostIntervalInSeconds * time.Second) telemetry.CloseAITelemetryHandle() diff --git a/telemetry/aiwrapper.go b/telemetry/aiwrapper.go index ef2a4d1da6..a2ba9ed481 100644 --- a/telemetry/aiwrapper.go +++ b/telemetry/aiwrapper.go @@ -2,6 +2,7 @@ package telemetry import ( + "fmt" "runtime" "github.com/Azure/azure-container-networking/aitelemetry" @@ -20,15 +21,22 @@ const ( waitTimeInSecs = 10 ) -func CreateAITelemetryHandle(aiConfig aitelemetry.AIConfig, disableAll, disableMetric, disableTrace bool) { +func CreateAITelemetryHandle(aiConfig aitelemetry.AIConfig, disableAll, disableMetric, disableTrace bool) error { + var err error + if disableAll { log.Printf("Telemetry is disabled") - return + return fmt.Errorf("Telmetry disabled") + } + + th, err = aitelemetry.NewAITelemetry(aiMetadata, aiConfig) + if err != nil { + return err } - th = aitelemetry.NewAITelemetry(aiMetadata, aiConfig) gDisableMetric = disableMetric gDisableTrace = disableTrace + return nil } func sendReport(cnireport CNIReport) { diff --git a/telemetry/telemetrybuffer.go b/telemetry/telemetrybuffer.go index f31643a30b..4fd417fe76 100644 --- a/telemetry/telemetrybuffer.go +++ b/telemetry/telemetrybuffer.go @@ -36,6 +36,8 @@ type TelemetryConfig struct { RefreshTimeoutInSecs int BatchIntervalInSecs int BatchSizeInBytes int + GetEnvRetryCount int + GetEnvRetryWaitTimeInSecs int } // FdName - file descriptor name @@ -452,6 +454,7 @@ func (buf *Buffer) push(x interface{}) { cniReport := x.(CNIReport) cniReport.Metadata = metadata SendAITelemetry(cniReport) + buf.CNIReports = append(buf.CNIReports, cniReport) case NPMReport: if len(buf.NPMReports) >= MaxNumReports { return From 7e1969ab87ca5b6ec366f26350d13bed36c3f0df Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 9 Jan 2020 12:50:41 -0800 Subject: [PATCH 14/17] fixed a condition --- aitelemetry/telemetrywrapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aitelemetry/telemetrywrapper.go b/aitelemetry/telemetrywrapper.go index 31bf3f9c7d..7b343134a7 100644 --- a/aitelemetry/telemetrywrapper.go +++ b/aitelemetry/telemetrywrapper.go @@ -105,7 +105,7 @@ func isPublicEnvironment(retryCount, waitTimeInSecs int) (bool, error) { if cloudName == azurePublicCloudStr { debugLog("[AppInsights] CloudName: %s\n", cloudName) return true, nil - } else if err != nil { + } else if err == nil { debugLog("[AppInsights] This is not azure public cloud:%s", cloudName) return false, fmt.Errorf("Not an azure public cloud: %s", cloudName) } From 97da22f3b5a0f8ea3656f2bc418db27f46ff9f33 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 9 Jan 2020 15:48:21 -0800 Subject: [PATCH 15/17] keep the netagent channel for logs --- cni/network/network.go | 8 ++++---- cni/network/plugin/main.go | 2 +- cni/telemetry/service/telemetrymain.go | 2 +- telemetry/aiwrapper.go | 18 +++++++++--------- telemetry/telemetry.go | 10 +++++++--- telemetry/telemetrybuffer.go | 22 ++++++++++++++++++++-- 6 files changed, 42 insertions(+), 20 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 7f2616b1da..0714df8c7f 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -189,7 +189,7 @@ func (plugin *netPlugin) getPodInfo(args string) (string, string, error) { return k8sPodName, k8sNamespace, nil } -func SetCustomDimensions(cniMetric *telemetry.CNIReport, nwCfg *cni.NetworkConfig, err error) { +func SetCustomDimensions(cniMetric *telemetry.AIMetric, nwCfg *cni.NetworkConfig, err error) { if cniMetric == nil { log.Errorf("[CNI] Unable to set custom dimension. Report is nil") return @@ -244,7 +244,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { enableInfraVnet bool enableSnatForDns bool nwDNSInfo network.DNSInfo - cniMetric telemetry.CNIReport + cniMetric telemetry.AIMetric ) startTime := time.Now() @@ -688,7 +688,7 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { networkId string nwInfo *network.NetworkInfo epInfo *network.EndpointInfo - cniMetric telemetry.CNIReport + cniMetric telemetry.AIMetric ) startTime := time.Now() @@ -795,7 +795,7 @@ func (plugin *netPlugin) Update(args *cniSkel.CmdArgs) error { cnsClient *cnsclient.CNSClient orchestratorContext []byte targetNetworkConfig *cns.GetNetworkContainerResponse - cniMetric telemetry.CNIReport + cniMetric telemetry.AIMetric ) startTime := time.Now() diff --git a/cni/network/plugin/main.go b/cni/network/plugin/main.go index 8feb5adc9f..8b95a41ff5 100644 --- a/cni/network/plugin/main.go +++ b/cni/network/plugin/main.go @@ -141,7 +141,7 @@ func main() { var ( config common.PluginConfig err error - cnimetric telemetry.CNIReport + cnimetric telemetry.AIMetric ) log.SetName(name) diff --git a/cni/telemetry/service/telemetrymain.go b/cni/telemetry/service/telemetrymain.go index 195dd2da85..95154095ad 100644 --- a/cni/telemetry/service/telemetrymain.go +++ b/cni/telemetry/service/telemetrymain.go @@ -166,7 +166,7 @@ func main() { tb = telemetry.NewTelemetryBuffer("") log.Logf("[Telemetry] Starting telemetry server") - err = tb.StartServer() + err = tb.StartServer(config.DisableTelemetryToNetAgent) if err == nil || tb.FdExists { break } diff --git a/telemetry/aiwrapper.go b/telemetry/aiwrapper.go index a2ba9ed481..237e2cfa83 100644 --- a/telemetry/aiwrapper.go +++ b/telemetry/aiwrapper.go @@ -39,7 +39,11 @@ func CreateAITelemetryHandle(aiConfig aitelemetry.AIConfig, disableAll, disableM return nil } -func sendReport(cnireport CNIReport) { +func SendAITelemetry(cnireport CNIReport) { + if th == nil || gDisableTrace { + return + } + var msg string if cnireport.ErrorMessage != "" { msg = cnireport.ErrorMessage @@ -62,17 +66,13 @@ func sendReport(cnireport CNIReport) { th.TrackLog(report) } -func SendAITelemetry(cnireport CNIReport) { - if th == nil { +func SendAIMetric(aiMetric AIMetric) { + if th == nil || gDisableMetric { return } - if cnireport.Metric.Name != "" && !gDisableMetric { - cnireport.Metric.CustomDimensions[OSTypeStr] = runtime.GOOS - th.TrackMetric(cnireport.Metric) - } else if !gDisableTrace { - sendReport(cnireport) - } + aiMetric.Metric.CustomDimensions[OSTypeStr] = runtime.GOOS + th.TrackMetric(aiMetric.Metric) } func CloseAITelemetryHandle() { diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index b398391a77..a774673f5a 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -99,7 +99,10 @@ type CNIReport struct { InterfaceDetails InterfaceInfo BridgeDetails BridgeInfo Metadata common.Metadata `json:"compute"` - Metric aitelemetry.Metric +} + +type AIMetric struct { + Metric aitelemetry.Metric } // Azure CNS Telemetry Report structure. @@ -374,6 +377,7 @@ func (reportMgr *ReportManager) ReportToBytes() ([]byte, error) { case *NPMReport: case *DNCReport: case *CNSReport: + case *AIMetric: default: err = fmt.Errorf("[Telemetry] Invalid report type") } @@ -387,14 +391,14 @@ func (reportMgr *ReportManager) ReportToBytes() ([]byte, error) { } // This function for sending CNI metrics to telemetry service -func SendCNIMetric(cniReport *CNIReport, tb *TelemetryBuffer) error { +func SendCNIMetric(cniMetric *AIMetric, tb *TelemetryBuffer) error { var ( err error report []byte ) if tb != nil && tb.Connected { - reportMgr := &ReportManager{Report: cniReport} + reportMgr := &ReportManager{Report: cniMetric} report, err = reportMgr.ReportToBytes() if err == nil { // If write fails, try to re-establish connections as server/client diff --git a/telemetry/telemetrybuffer.go b/telemetry/telemetrybuffer.go index 4fd417fe76..6be313867a 100644 --- a/telemetry/telemetrybuffer.go +++ b/telemetry/telemetrybuffer.go @@ -33,6 +33,7 @@ type TelemetryConfig struct { DisableMetric bool DisableMetadataThread bool DebugMode bool + DisableTelemetryToNetAgent bool RefreshTimeoutInSecs int BatchIntervalInSecs int BatchSizeInBytes int @@ -60,7 +61,10 @@ const ( cni = "CNI" ) -var payloadSize uint16 = 0 +var ( + payloadSize uint16 = 0 + disableTelemetryToNetAgent bool +) // TelemetryBuffer object type TelemetryBuffer struct { @@ -114,7 +118,8 @@ func remove(s []net.Conn, i int) []net.Conn { } // Starts Telemetry server listening on unix domain socket -func (tb *TelemetryBuffer) StartServer() error { +func (tb *TelemetryBuffer) StartServer(disableNetAgentChannel bool) error { + disableTelemetryToNetAgent = disableNetAgentChannel err := tb.Listen(FdName) if err != nil { tb.FdExists = strings.Contains(err.Error(), "in use") || strings.Contains(err.Error(), "Access is denied") @@ -146,6 +151,10 @@ func (tb *TelemetryBuffer) StartServer() error { var cniReport CNIReport json.Unmarshal([]byte(reportStr), &cniReport) tb.data <- cniReport + } else if _, ok := tmp["Metric"]; ok { + var aiMetric AIMetric + json.Unmarshal([]byte(reportStr), &aiMetric) + tb.data <- aiMetric } else if _, ok := tmp["Allocations"]; ok { var dncReport DNCReport json.Unmarshal([]byte(reportStr), &dncReport) @@ -289,6 +298,10 @@ func (tb *TelemetryBuffer) Close() { // sendToHost - send buffer to host func (tb *TelemetryBuffer) sendToHost() error { + if disableTelemetryToNetAgent { + return nil + } + buf := Buffer{ DNCReports: make([]DNCReport, 0), CNIReports: make([]CNIReport, 0), @@ -455,6 +468,11 @@ func (buf *Buffer) push(x interface{}) { cniReport.Metadata = metadata SendAITelemetry(cniReport) buf.CNIReports = append(buf.CNIReports, cniReport) + + case AIMetric: + aiMetric := x.(AIMetric) + SendAIMetric(aiMetric) + case NPMReport: if len(buf.NPMReports) >= MaxNumReports { return From 2a10e1d74fccaedf2904c68eddf9c5275c3e5baf Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 9 Jan 2020 16:11:24 -0800 Subject: [PATCH 16/17] fixed error --- telemetry/telemetry_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index 60dc8e3629..463ff2301c 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -106,7 +106,7 @@ func TestMain(m *testing.M) { reportManager.Report = &CNIReport{} tb = NewTelemetryBuffer(hostAgentUrl) - err = tb.StartServer() + err = tb.StartServer(false) if err == nil { go tb.BufferAndPushData(0) } @@ -197,7 +197,7 @@ func TestCloseTelemetryConnection(t *testing.T) { func TestServerCloseTelemetryConnection(t *testing.T) { // create server telemetrybuffer and start server tb = NewTelemetryBuffer(hostAgentUrl) - err := tb.StartServer() + err := tb.StartServer(false) if err == nil { go tb.BufferAndPushData(0) } @@ -228,7 +228,7 @@ func TestServerCloseTelemetryConnection(t *testing.T) { func TestClientCloseTelemetryConnection(t *testing.T) { // create server telemetrybuffer and start server tb = NewTelemetryBuffer(hostAgentUrl) - err := tb.StartServer() + err := tb.StartServer(false) if err == nil { go tb.BufferAndPushData(0) } From c3a114251bba863824bbcf7aebd2e70598eb7a32 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 9 Jan 2020 18:10:29 -0800 Subject: [PATCH 17/17] addressed review comments --- aitelemetry/telemetrywrapper.go | 20 +++++----- aitelemetry/telemetrywrapper_test.go | 53 ++++++++++++++++++++------ cni/telemetry/service/telemetrymain.go | 6 ++- common/utils.go | 9 ++++- telemetry/aiwrapper.go | 2 +- 5 files changed, 63 insertions(+), 27 deletions(-) diff --git a/aitelemetry/telemetrywrapper.go b/aitelemetry/telemetrywrapper.go index 7b343134a7..aaad0a3eda 100644 --- a/aitelemetry/telemetrywrapper.go +++ b/aitelemetry/telemetrywrapper.go @@ -2,7 +2,6 @@ package aitelemetry import ( "fmt" - "os" "runtime" "time" @@ -88,20 +87,14 @@ func getMetadata(th *telemetryHandle) { } } -func isPublicEnvironment(retryCount, waitTimeInSecs int) (bool, error) { +func isPublicEnvironment(url string, retryCount, waitTimeInSecs int) (bool, error) { var ( cloudName string err error ) - env, _ := os.LookupEnv("AZACN_TESTENV") - if env == "test" { - debugLog("Test environment") - return true, nil - } - for i := 0; i < retryCount; i++ { - cloudName, err = common.GetAzureCloud() + cloudName, err = common.GetAzureCloud(url) if cloudName == azurePublicCloudStr { debugLog("[AppInsights] CloudName: %s\n", cloudName) return true, nil @@ -119,12 +112,19 @@ func isPublicEnvironment(retryCount, waitTimeInSecs int) (bool, error) { // NewAITelemetry creates telemetry handle with user specified appinsights id. func NewAITelemetry( + azEnvUrl string, id string, aiConfig AIConfig, ) (TelemetryHandle, error) { debugMode = aiConfig.DebugMode - isPublic, err := isPublicEnvironment(aiConfig.GetEnvRetryCount, aiConfig.GetEnvRetryWaitTimeInSecs) + if id == "" { + debugLog("Empty AI key") + return nil, fmt.Errorf("AI key is empty") + } + + // check if azure instance is in public cloud + isPublic, err := isPublicEnvironment(azEnvUrl, aiConfig.GetEnvRetryCount, aiConfig.GetEnvRetryWaitTimeInSecs) if !isPublic { return nil, err } diff --git a/aitelemetry/telemetrywrapper_test.go b/aitelemetry/telemetrywrapper_test.go index 58250cebb6..4edebadae6 100644 --- a/aitelemetry/telemetrywrapper_test.go +++ b/aitelemetry/telemetrywrapper_test.go @@ -2,16 +2,24 @@ package aitelemetry import ( "fmt" + "net/http" + "net/url" "os" "path/filepath" "runtime" "testing" + "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/platform" ) -var th TelemetryHandle +var ( + th TelemetryHandle + hostAgentUrl = "localhost:3501" + getCloudResponse = "AzurePublicCloud" + httpURL = "http://" + hostAgentUrl +) func TestMain(m *testing.M) { log.SetLogDirectory("/var/log/") @@ -30,6 +38,20 @@ func TestMain(m *testing.M) { platform.ExecuteCommand(cmd) } + hostu, _ := url.Parse("tcp://" + hostAgentUrl) + hostAgent, err := common.NewListener(hostu) + if err != nil { + fmt.Printf("Failed to create agent, err:%v.\n", err) + return + } + + hostAgent.AddHandler("/", handleGetCloud) + err = hostAgent.Start(make(chan error, 1)) + if err != nil { + fmt.Printf("Failed to start agent, err:%v.\n", err) + return + } + exitCode := m.Run() if runtime.GOOS == "linux" { @@ -44,8 +66,12 @@ func TestMain(m *testing.M) { os.Exit(exitCode) } +func handleGetCloud(w http.ResponseWriter, req *http.Request) { + w.Write([]byte(getCloudResponse)) +} + func TestEmptyAIKey(t *testing.T) { - os.Setenv("AZACN_TESTENV", "test") + var err error aiConfig := AIConfig{ AppName: "testapp", @@ -56,15 +82,14 @@ func TestEmptyAIKey(t *testing.T) { DebugMode: true, DisableMetadataRefreshThread: true, } - th := NewAITelemetry("", aiConfig) - if th == nil { - t.Errorf("Error intializing AI telemetry") + _, err = NewAITelemetry(httpURL, "", aiConfig) + if err == nil { + t.Errorf("Error intializing AI telemetry:%v", err) } - th.Close(10) } func TestNewAITelemetry(t *testing.T) { - os.Setenv("AZACN_TESTENV", "test") + var err error aiConfig := AIConfig{ AppName: "testapp", @@ -72,12 +97,14 @@ func TestNewAITelemetry(t *testing.T) { BatchSize: 4096, BatchInterval: 2, RefreshTimeout: 10, + GetEnvRetryCount: 1, + GetEnvRetryWaitTimeInSecs: 2, DebugMode: true, DisableMetadataRefreshThread: true, } - th = NewAITelemetry("00ca2a73-c8d6-4929-a0c2-cf84545ec225", aiConfig) + th, err = NewAITelemetry(httpURL, "00ca2a73-c8d6-4929-a0c2-cf84545ec225", aiConfig) if th == nil { - t.Errorf("Error intializing AI telemetry") + t.Errorf("Error intializing AI telemetry: %v", err) } } @@ -108,7 +135,7 @@ func TestClose(t *testing.T) { } func TestClosewithoutSend(t *testing.T) { - os.Setenv("AZACN_TESTENV", "test") + var err error aiConfig := AIConfig{ AppName: "testapp", @@ -117,11 +144,13 @@ func TestClosewithoutSend(t *testing.T) { BatchInterval: 2, DisableMetadataRefreshThread: true, RefreshTimeout: 10, + GetEnvRetryCount: 1, + GetEnvRetryWaitTimeInSecs: 2, } - thtest := NewAITelemetry("00ca2a73-c8d6-4929-a0c2-cf84545ec225", aiConfig) + thtest, err := NewAITelemetry(httpURL, "00ca2a73-c8d6-4929-a0c2-cf84545ec225", aiConfig) if thtest == nil { - t.Errorf("Error intializing AI telemetry") + t.Errorf("Error intializing AI telemetry:%v", err) } thtest.Close(10) diff --git a/cni/telemetry/service/telemetrymain.go b/cni/telemetry/service/telemetrymain.go index 95154095ad..1a8229e3d9 100644 --- a/cni/telemetry/service/telemetrymain.go +++ b/cni/telemetry/service/telemetrymain.go @@ -19,6 +19,8 @@ const ( defaultRefreshTimeoutInSecs = 15 defaultBatchSizeInBytes = 16384 defaultBatchIntervalInSecs = 15 + defaultGetEnvRetryCount = 2 + defaultGetEnvRetryWaitTimeInSecs = 3 pluginName = "AzureCNI" azureVnetTelemetry = "azure-vnet-telemetry" configExtension = ".config" @@ -99,11 +101,11 @@ func setTelemetryDefaults(config *telemetry.TelemetryConfig) { } if config.GetEnvRetryCount == 0 { - config.GetEnvRetryCount = 2 + config.GetEnvRetryCount = defaultGetEnvRetryCount } if config.GetEnvRetryWaitTimeInSecs == 0 { - config.GetEnvRetryWaitTimeInSecs = 3 + config.GetEnvRetryWaitTimeInSecs = defaultGetEnvRetryWaitTimeInSecs } } diff --git a/common/utils.go b/common/utils.go index 9f3ed44314..d392989f3b 100644 --- a/common/utils.go +++ b/common/utils.go @@ -291,8 +291,13 @@ func SaveHostMetadata(metadata Metadata, fileName string) error { return err } -func GetAzureCloud() (string, error) { - req, err := http.NewRequest("GET", azCloudUrl, nil) +func GetAzureCloud(url string) (string, error) { + if url == "" { + url = azCloudUrl + } + + log.Printf("GetAzureCloud querying url: %s", url) + req, err := http.NewRequest("GET", url, nil) if err != nil { return "", err } diff --git a/telemetry/aiwrapper.go b/telemetry/aiwrapper.go index 237e2cfa83..527af04922 100644 --- a/telemetry/aiwrapper.go +++ b/telemetry/aiwrapper.go @@ -29,7 +29,7 @@ func CreateAITelemetryHandle(aiConfig aitelemetry.AIConfig, disableAll, disableM return fmt.Errorf("Telmetry disabled") } - th, err = aitelemetry.NewAITelemetry(aiMetadata, aiConfig) + th, err = aitelemetry.NewAITelemetry("", aiMetadata, aiConfig) if err != nil { return err }