From 509215a48848ed5626f6bdb84e0dc3ee0d824999 Mon Sep 17 00:00:00 2001 From: "Kav. Pather" Date: Mon, 25 Mar 2024 03:36:15 +1100 Subject: [PATCH] feat: add log api output + disable sybase to allow arm 6 to pass (#466) * feat: output to log api * feat: disable sybase --- internal/inputs/database.go | 6 +-- internal/inputs/database_test.go | 2 +- internal/load/load.go | 6 ++- internal/outputs/http.go | 4 +- internal/outputs/logs.go | 71 ++++++++++++++++++++++++++++++++ internal/runtime/flex.go | 6 +++ test/testbed/launch_e2e.sh | 0 7 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 internal/outputs/logs.go mode change 100755 => 100644 test/testbed/launch_e2e.sh diff --git a/internal/inputs/database.go b/internal/inputs/database.go index 03c52880..c2b17a68 100644 --- a/internal/inputs/database.go +++ b/internal/inputs/database.go @@ -25,7 +25,7 @@ import ( //Database Drivers _ "github.com/MonetDB/MonetDB-Go/src" //MonetDB - _ "github.com/SAP/go-ase" //Sybase + // _ "github.com/SAP/go-ase" //Sybase _ "github.com/SAP/go-hdb/driver" //SAP HANA _ "github.com/denisenkom/go-mssqldb" //mssql | sql-server _ "github.com/go-sql-driver/mysql" //mysql @@ -228,8 +228,8 @@ func setDatabaseDriver(database, driver string, yml *load.Config, api load.API) return load.DefaultMySQL case "oracle": return load.DefaultOracle - case "sybase", "ase": - return load.DefaultSybase + // case "sybase", "ase": + // return load.DefaultSybase case "monetdb": return load.DefaultMonetDB case "hana", "go-hdb", "hdb": diff --git a/internal/inputs/database_test.go b/internal/inputs/database_test.go index 3873b7f5..b0071bd0 100644 --- a/internal/inputs/database_test.go +++ b/internal/inputs/database_test.go @@ -26,7 +26,7 @@ func TestDrivers(t *testing.T) { "mariadb": load.DefaultMySQL, "hana": load.DefaultHANA, "oracle": load.DefaultOracle, - "ase": load.DefaultSybase, + // "ase": load.DefaultSybase, "monetdb": load.DefaultMonetDB, "unknown": "", } diff --git a/internal/load/load.go b/internal/load/load.go index d47d39c8..91f8bfb0 100644 --- a/internal/load/load.go +++ b/internal/load/load.go @@ -32,6 +32,10 @@ type ArgumentList struct { DockerAPIVersion string `default:"" help:"Force Docker client API version"` EventLimit int `default:"0" help:"Event limiter - limit events per execution, 0 to disable"` Entity string `default:"" help:"Manually set a remote entity name"` + LogApiURL string `default:"" help:"Set Log API URL"` + LogApiKey string `default:"" help:"Set Log API key"` + LogBatchSize int `default:"5000" help:"Batch Size - number of metrics per post call to log endpoint"` + LogOutput bool `default:"false" help:"Output the events generated to standard out"` InsightsURL string `default:"" help:"Set Insights URL"` InsightsAPIKey string `default:"" help:"Set Insights API key"` InsightsOutput bool `default:"false" help:"Output the events generated to standard out"` @@ -115,7 +119,7 @@ const ( DefaultMSSQLServer = "sqlserver" DefaultMySQL = "mysql" DefaultOracle = "oracle" - DefaultSybase = "ase" + // DefaultSybase = "ase" DefaultMonetDB = "monetdb" DefaultVertica = "vertica" DefaultJmxHost = "127.0.0.1" diff --git a/internal/outputs/http.go b/internal/outputs/http.go index 4af1466a..f21f2f60 100644 --- a/internal/outputs/http.go +++ b/internal/outputs/http.go @@ -9,12 +9,12 @@ import ( "bytes" "compress/zlib" "fmt" - "github.com/pkg/errors" "io/ioutil" "net/http" "time" "github.com/newrelic/nri-flex/internal/load" + "github.com/pkg/errors" ) // postRequest wraps request and attaches needed headers and zlib compression @@ -31,7 +31,7 @@ func postRequest(url string, key string, data []byte) error { } load.Logrus. - Debugf("http: insights - bytes %d events %d", len(zlibCompressedPayload.Bytes()), len(load.Entity.Metrics)) + Debugf("http: bytes %d events %d", len(zlibCompressedPayload.Bytes()), len(load.Entity.Metrics)) tr := &http.Transport{IdleConnTimeout: 15 * time.Second, Proxy: http.ProxyFromEnvironment} client := &http.Client{Transport: tr} diff --git a/internal/outputs/logs.go b/internal/outputs/logs.go new file mode 100644 index 00000000..337b625f --- /dev/null +++ b/internal/outputs/logs.go @@ -0,0 +1,71 @@ +/* +* Copyright 2019 New Relic Corporation. All rights reserved. +* SPDX-License-Identifier: Apache-2.0 + */ + +package outputs + +import ( + "encoding/json" + "fmt" + + "github.com/newrelic/infra-integrations-sdk/data/metric" + "github.com/newrelic/infra-integrations-sdk/integration" + "github.com/newrelic/nri-flex/internal/load" +) + +// GetLogMetricBatches batch metrics by entity with a maximum batch size defined by 'LogBatchSize' config. +func GetLogMetricBatches() [][]*metric.Set { + var result [][]*metric.Set + for _, entity := range load.Integration.Entities { + // split the payload into smaller batches so that the payload size does not exceed the Log endpoint size limit + batchSize := load.Args.InsightBatchSize + sizeMetrics := len(entity.Metrics) + batches := sizeMetrics/batchSize + 1 + + modifyLogEventType(entity) + + for i := 0; i < batches; i++ { + start := i * batchSize + end := start + batchSize + if end > sizeMetrics { + end = sizeMetrics + } + result = append(result, (entity.Metrics)[start:end]) + } + // empty the infrastructure entity metrics by default + entity.Metrics = []*metric.Set{} + } + return result +} + +// SendBatchToLogApi - Send processed events to log api. +func SendBatchToLogApi(metrics []*metric.Set) error { + jsonData, err := json.Marshal(metrics) + if err != nil { + return fmt.Errorf("log api: failed to marshal json, %v", err) + } + + load.Logrus.Debugf("posting %d events to log api", len(metrics)) + + if load.Args.LogOutput { + fmt.Println(string(jsonData)) + } + + err = postRequest(load.Args.LogApiURL, load.Args.LogApiKey, jsonData) + if err != nil { + return fmt.Errorf("log api: sending events failed, %v", err) + } + + return nil +} + +// modifyEventType insights uses eventType key in camel case whereas infrastructure uses event_type +func modifyLogEventType(entity *integration.Entity) { + for _, event := range entity.Metrics { + if event.Metrics["event_type"] != nil { + event.Metrics["flexEventType"] = event.Metrics["event_type"].(string) + } + delete(event.Metrics, "event_type") + } +} diff --git a/internal/runtime/flex.go b/internal/runtime/flex.go index 3d4c3ef0..b99a852f 100644 --- a/internal/runtime/flex.go +++ b/internal/runtime/flex.go @@ -103,6 +103,12 @@ func RunFlex(instance Instance) error { log.WithError(err).Error("runtime.RunFlex: failed to send batch to insights") } } + } else if load.Args.LogApiURL != "" && load.Args.LogApiKey != "" { + for _, batch := range outputs.GetLogMetricBatches() { + if err := outputs.SendBatchToLogApi(batch); err != nil { + log.WithError(err).Error("runtime.RunFlex: failed to send batch to log api") + } + } } else if load.Args.MetricAPIUrl != "" && (load.Args.InsightsAPIKey != "" || load.Args.MetricAPIKey != "") && len(load.MetricsStore.Data) > 0 { if err := outputs.SendToMetricAPI(); err != nil { log.WithError(err).Error("runtime.RunFlex: failed to send metrics") diff --git a/test/testbed/launch_e2e.sh b/test/testbed/launch_e2e.sh old mode 100755 new mode 100644