feat: add log api output + disable sybase to allow arm 6 to pass (#466)

* feat: output to log api

* feat: disable sybase

internal/inputs/database.go

@@ -25,7 +25,7 @@ import (
 
 	//Database Drivers
 	_ "github.com/MonetDB/MonetDB-Go/src"       //MonetDB
-	_ "github.com/SAP/go-ase"                   //Sybase
+	// _ "github.com/SAP/go-ase"                   //Sybase
 	_ "github.com/SAP/go-hdb/driver"            //SAP HANA
 	_ "github.com/denisenkom/go-mssqldb"        //mssql | sql-server
 	_ "github.com/go-sql-driver/mysql"          //mysql
@@ -228,8 +228,8 @@ func setDatabaseDriver(database, driver string, yml *load.Config, api load.API)
 		return load.DefaultMySQL
 	case "oracle":
 		return load.DefaultOracle
-	case "sybase", "ase":
-		return load.DefaultSybase
+	// case "sybase", "ase":
+	// 	return load.DefaultSybase
 	case "monetdb":
 		return load.DefaultMonetDB
 	case "hana", "go-hdb", "hdb":

internal/inputs/database_test.go

@@ -26,7 +26,7 @@ func TestDrivers(t *testing.T) {
 		"mariadb":   load.DefaultMySQL,
 		"hana":      load.DefaultHANA,
 		"oracle":    load.DefaultOracle,
-		"ase":       load.DefaultSybase,
+		// "ase":       load.DefaultSybase,
 		"monetdb":   load.DefaultMonetDB,
 		"unknown":   "",
 	}

internal/load/load.go

@@ -32,6 +32,10 @@ type ArgumentList struct {
 	DockerAPIVersion        string `default:"" help:"Force Docker client API version"`
 	EventLimit              int    `default:"0" help:"Event limiter - limit events per execution, 0 to disable"`
 	Entity                  string `default:"" help:"Manually set a remote entity name"`
+	LogApiURL               string `default:"" help:"Set Log API URL"`
+	LogApiKey               string `default:"" help:"Set Log API key"`
+	LogBatchSize            int    `default:"5000" help:"Batch Size - number of metrics per post call to log endpoint"`
+	LogOutput               bool   `default:"false" help:"Output the events generated to standard out"`
 	InsightsURL             string `default:"" help:"Set Insights URL"`
 	InsightsAPIKey          string `default:"" help:"Set Insights API key"`
 	InsightsOutput          bool   `default:"false" help:"Output the events generated to standard out"`
@@ -115,7 +119,7 @@ const (
 	DefaultMSSQLServer = "sqlserver"
 	DefaultMySQL       = "mysql"
 	DefaultOracle      = "oracle"
-	DefaultSybase      = "ase"
+	// DefaultSybase      = "ase"
 	DefaultMonetDB     = "monetdb"
 	DefaultVertica     = "vertica"
 	DefaultJmxHost     = "127.0.0.1"

internal/outputs/http.go

@@ -9,12 +9,12 @@ import (
 	"bytes"
 	"compress/zlib"
 	"fmt"
-	"github.com/pkg/errors"
 	"io/ioutil"
 	"net/http"
 	"time"
 
 	"github.com/newrelic/nri-flex/internal/load"
+	"github.com/pkg/errors"
 )
 
 // postRequest wraps request and attaches needed headers and zlib compression
@@ -31,7 +31,7 @@ func postRequest(url string, key string, data []byte) error {
 	}
 
 	load.Logrus.
-		Debugf("http: insights - bytes %d events %d", len(zlibCompressedPayload.Bytes()), len(load.Entity.Metrics))
+		Debugf("http: bytes %d events %d", len(zlibCompressedPayload.Bytes()), len(load.Entity.Metrics))
 
 	tr := &http.Transport{IdleConnTimeout: 15 * time.Second, Proxy: http.ProxyFromEnvironment}
 	client := &http.Client{Transport: tr}

internal/outputs/logs.go

@@ -0,0 +1,71 @@
+/*
+* Copyright 2019 New Relic Corporation. All rights reserved.
+* SPDX-License-Identifier: Apache-2.0
+ */
+
+package outputs
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/newrelic/infra-integrations-sdk/data/metric"
+	"github.com/newrelic/infra-integrations-sdk/integration"
+	"github.com/newrelic/nri-flex/internal/load"
+)
+
+// GetLogMetricBatches batch metrics by entity with a maximum batch size defined by 'LogBatchSize' config.
+func GetLogMetricBatches() [][]*metric.Set {
+	var result [][]*metric.Set
+	for _, entity := range load.Integration.Entities {
+		// split the payload into smaller batches so that the payload size does not exceed the Log endpoint size limit
+		batchSize := load.Args.InsightBatchSize
+		sizeMetrics := len(entity.Metrics)
+		batches := sizeMetrics/batchSize + 1
+
+		modifyLogEventType(entity)
+
+		for i := 0; i < batches; i++ {
+			start := i * batchSize
+			end := start + batchSize
+			if end > sizeMetrics {
+				end = sizeMetrics
+			}
+			result = append(result, (entity.Metrics)[start:end])
+		}
+		// empty the infrastructure entity metrics by default
+		entity.Metrics = []*metric.Set{}
+	}
+	return result
+}
+
+// SendBatchToLogApi - Send processed events to log api.
+func SendBatchToLogApi(metrics []*metric.Set) error {
+	jsonData, err := json.Marshal(metrics)
+	if err != nil {
+		return fmt.Errorf("log api: failed to marshal json, %v", err)
+	}
+
+	load.Logrus.Debugf("posting %d events to log api", len(metrics))
+
+	if load.Args.LogOutput {
+		fmt.Println(string(jsonData))
+	}
+
+	err = postRequest(load.Args.LogApiURL, load.Args.LogApiKey, jsonData)
+	if err != nil {
+		return fmt.Errorf("log api: sending events failed, %v", err)
+	}
+
+	return nil
+}
+
+// modifyEventType insights uses eventType key in camel case whereas infrastructure uses event_type
+func modifyLogEventType(entity *integration.Entity) {
+	for _, event := range entity.Metrics {
+		if event.Metrics["event_type"] != nil {
+			event.Metrics["flexEventType"] = event.Metrics["event_type"].(string)
+		}
+		delete(event.Metrics, "event_type")
+	}
+}

internal/runtime/flex.go

@@ -103,6 +103,12 @@ func RunFlex(instance Instance) error {
 				log.WithError(err).Error("runtime.RunFlex: failed to send batch to insights")
 			}
 		}
+	} else if load.Args.LogApiURL != "" && load.Args.LogApiKey != "" {
+		for _, batch := range outputs.GetLogMetricBatches() {
+			if err := outputs.SendBatchToLogApi(batch); err != nil {
+				log.WithError(err).Error("runtime.RunFlex: failed to send batch to log api")
+			}
+		}
 	} else if load.Args.MetricAPIUrl != "" && (load.Args.InsightsAPIKey != "" || load.Args.MetricAPIKey != "") && len(load.MetricsStore.Data) > 0 {
 		if err := outputs.SendToMetricAPI(); err != nil {
 			log.WithError(err).Error("runtime.RunFlex: failed to send metrics")