diff --git a/go-telemetry-api-extension/go.mod b/go-telemetry-api-extension/go.mod index 1b8d1ed3..aec2d487 100644 --- a/go-telemetry-api-extension/go.mod +++ b/go-telemetry-api-extension/go.mod @@ -3,8 +3,9 @@ module newrelic-lambda-extension/go-telemetry-api-extension go 1.18 require ( - github.com/golang-collections/go-datastructures v0.0.0-20150211160725-59788d5eb259 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect - golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect + github.com/golang-collections/go-datastructures v0.0.0-20150211160725-59788d5eb259 + github.com/pkg/errors v0.9.1 + github.com/sirupsen/logrus v1.9.0 ) + +require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect diff --git a/go-telemetry-api-extension/go.sum b/go-telemetry-api-extension/go.sum index 47e4990e..d65581a4 100644 --- a/go-telemetry-api-extension/go.sum +++ b/go-telemetry-api-extension/go.sum @@ -1,15 +1,19 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang-collections/go-datastructures v0.0.0-20150211160725-59788d5eb259 h1:ZHJ7+IGpuOXtVf6Zk/a3WuHQgkC+vXwaqfUBDFwahtI= github.com/golang-collections/go-datastructures v0.0.0-20150211160725-59788d5eb259/go.mod h1:9Qcha0gTWLw//0VNka1Cbnjvg3pNKGFdAm7E9sBabxE= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go-telemetry-api-extension/telemetryApi/dispatcher.go b/go-telemetry-api-extension/telemetryApi/dispatcher.go index 99867f2d..1496cbf1 100644 --- a/go-telemetry-api-extension/telemetryApi/dispatcher.go +++ b/go-telemetry-api-extension/telemetryApi/dispatcher.go @@ -9,15 +9,13 @@ import ( "os" "strconv" - "errors" - "github.com/golang-collections/go-datastructures/queue" ) type Dispatcher struct { httpClient *http.Client postUri string - licenseKey string + licenseKey string minBatchSize int64 } @@ -40,7 +38,7 @@ func NewDispatcher() *Dispatcher { return &Dispatcher{ httpClient: &http.Client{}, postUri: dispatchPostUri, - licenseKey: licenseKey, + licenseKey: licenseKey, minBatchSize: dispatchMinBatchSize, } @@ -51,7 +49,7 @@ func (d *Dispatcher) Dispatch(ctx context.Context, logEventsQueue *queue.Queue, l.Info("[dispatcher:Dispatch] Dispatching", logEventsQueue.Len(), "log events") logEntries, _ := logEventsQueue.Get(logEventsQueue.Len()) - err = sendDataToNR(ctx, logEntries, d) + err := sendDataToNR(ctx, logEntries, d) if err != nil { l.Error("[dispatcher:Dispatch] Failed to dispatch, returning to queue:", err) for logEntry := range logEntries { diff --git a/go-telemetry-api-extension/telemetryApi/jsonEncode.go b/go-telemetry-api-extension/telemetryApi/jsonEncode.go new file mode 100644 index 00000000..cff22f8a --- /dev/null +++ b/go-telemetry-api-extension/telemetryApi/jsonEncode.go @@ -0,0 +1,190 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package jsonx extends the encoding/json package to encode JSON +// incrementally and without requiring reflection. +package telemetryApi + +import ( + "bytes" + "encoding/json" + "math" + "reflect" + "strconv" + "unicode/utf8" +) + +var hex = "0123456789abcdef" + +// AppendString escapes s appends it to buf. +func AppendString(buf *bytes.Buffer, s string) { + buf.WriteByte('"') + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { + i++ + continue + } + if start < i { + buf.WriteString(s[start:i]) + } + switch b { + case '\\', '"': + buf.WriteByte('\\') + buf.WriteByte(b) + case '\n': + buf.WriteByte('\\') + buf.WriteByte('n') + case '\r': + buf.WriteByte('\\') + buf.WriteByte('r') + case '\t': + buf.WriteByte('\\') + buf.WriteByte('t') + default: + // This encodes bytes < 0x20 except for \n and \r, + // as well as <, > and &. The latter are escaped because they + // can lead to security holes when user-controlled strings + // are rendered into JSON and served to some browsers. + buf.WriteString(`\u00`) + buf.WriteByte(hex[b>>4]) + buf.WriteByte(hex[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRuneInString(s[i:]) + if c == utf8.RuneError && size == 1 { + if start < i { + buf.WriteString(s[start:i]) + } + buf.WriteString(`\ufffd`) + i += size + start = i + continue + } + // U+2028 is LINE SEPARATOR. + // U+2029 is PARAGRAPH SEPARATOR. + // They are both technically valid characters in JSON strings, + // but don't work in JSONP, which has to be evaluated as JavaScript, + // and can lead to security holes there. It is valid JSON to + // escape them, so we do so unconditionally. + // See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion. + if c == '\u2028' || c == '\u2029' { + if start < i { + buf.WriteString(s[start:i]) + } + buf.WriteString(`\u202`) + buf.WriteByte(hex[c&0xF]) + i += size + start = i + continue + } + i += size + } + if start < len(s) { + buf.WriteString(s[start:]) + } + buf.WriteByte('"') +} + +// AppendStringArray appends an array of string literals to buf. +func AppendStringArray(buf *bytes.Buffer, a ...string) { + buf.WriteByte('[') + for i, s := range a { + if i > 0 { + buf.WriteByte(',') + } + AppendString(buf, s) + } + buf.WriteByte(']') +} + +// AppendFloat appends a numeric literal representing the value to buf. +func AppendFloat(buf *bytes.Buffer, x float64) error { + var scratch [64]byte + + if math.IsInf(x, 0) || math.IsNaN(x) { + return &json.UnsupportedValueError{ + Value: reflect.ValueOf(x), + Str: strconv.FormatFloat(x, 'g', -1, 64), + } + } + + buf.Write(strconv.AppendFloat(scratch[:0], x, 'g', -1, 64)) + return nil +} + +// AppendFloat32 appends a numeric literal representingthe value to buf. +func AppendFloat32(buf *bytes.Buffer, x float32) error { + var scratch [64]byte + x64 := float64(x) + + if math.IsInf(x64, 0) || math.IsNaN(x64) { + return &json.UnsupportedValueError{ + Value: reflect.ValueOf(x64), + Str: strconv.FormatFloat(x64, 'g', -1, 32), + } + } + + buf.Write(strconv.AppendFloat(scratch[:0], x64, 'g', -1, 32)) + return nil +} + +// AppendFloatArray appends an array of numeric literals to buf. +func AppendFloatArray(buf *bytes.Buffer, a ...float64) error { + buf.WriteByte('[') + for i, x := range a { + if i > 0 { + buf.WriteByte(',') + } + if err := AppendFloat(buf, x); err != nil { + return err + } + } + buf.WriteByte(']') + return nil +} + +// AppendInt appends a numeric literal representing the value to buf. +func AppendInt(buf *bytes.Buffer, x int64) { + var scratch [64]byte + buf.Write(strconv.AppendInt(scratch[:0], x, 10)) +} + +// AppendIntArray appends an array of numeric literals to buf. +func AppendIntArray(buf *bytes.Buffer, a ...int64) { + var scratch [64]byte + + buf.WriteByte('[') + for i, x := range a { + if i > 0 { + buf.WriteByte(',') + } + buf.Write(strconv.AppendInt(scratch[:0], x, 10)) + } + buf.WriteByte(']') +} + +// AppendUint appends a numeric literal representing the value to buf. +func AppendUint(buf *bytes.Buffer, x uint64) { + var scratch [64]byte + buf.Write(strconv.AppendUint(scratch[:0], x, 10)) +} + +// AppendUintArray appends an array of numeric literals to buf. +func AppendUintArray(buf *bytes.Buffer, a ...uint64) { + var scratch [64]byte + + buf.WriteByte('[') + for i, x := range a { + if i > 0 { + buf.WriteByte(',') + } + buf.Write(strconv.AppendUint(scratch[:0], x, 10)) + } + buf.WriteByte(']') +} diff --git a/go-telemetry-api-extension/telemetryApi/jsonUtil.go b/go-telemetry-api-extension/telemetryApi/jsonUtil.go new file mode 100644 index 00000000..6e555776 --- /dev/null +++ b/go-telemetry-api-extension/telemetryApi/jsonUtil.go @@ -0,0 +1,104 @@ +package telemetryApi + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + "time" +) + +// jsonString assists in logging JSON: Based on the formatter used to log +// Context contents, the contents could be marshalled as JSON or just printed +// directly. +type jsonString string + +// MarshalJSON returns the jsonString unmodified without any escaping. +func (js jsonString) MarshalJSON() ([]byte, error) { + if "" == js { + return []byte("null"), nil + } + return []byte(js), nil +} + +func removeFirstSegment(name string) string { + idx := strings.Index(name, "/") + if -1 == idx { + return name + } + return name[idx+1:] +} + +func timeToIntMillis(t time.Time) int64 { + return t.UnixNano() / (1000 * 1000) +} + +func timeToFloatMilliseconds(t time.Time) float64 { + return float64(t.UnixNano()) / float64(1000*1000) +} + +// compactJSONString removes the whitespace from a JSON string. This function +// will panic if the string provided is not valid JSON. Thus is must only be +// used in testing code! +func compactJSONString(js string) string { + buf := new(bytes.Buffer) + if err := json.Compact(buf, []byte(js)); err != nil { + panic(fmt.Errorf("unable to compact JSON: %v", err)) + } + return buf.String() +} + +// getContentLengthFromHeader gets the content length from a HTTP header, or -1 +// if no content length is available. +func getContentLengthFromHeader(h http.Header) int64 { + if cl := h.Get("Content-Length"); cl != "" { + if contentLength, err := strconv.ParseInt(cl, 10, 64); err == nil { + return contentLength + } + } + + return -1 +} + +// stringLengthByteLimit truncates strings using a byte-limit boundary and +// avoids terminating in the middle of a multibyte character. +func stringLengthByteLimit(str string, byteLimit int) string { + if len(str) <= byteLimit { + return str + } + + limitIndex := 0 + for pos := range str { + if pos > byteLimit { + break + } + limitIndex = pos + } + return str[0:limitIndex] +} + +func timeFromUnixMilliseconds(millis uint64) time.Time { + secs := int64(millis) / 1000 + msecsRemaining := int64(millis) % 1000 + nsecsRemaining := msecsRemaining * (1000 * 1000) + return time.Unix(secs, nsecsRemaining) +} + +// timeToUnixMilliseconds converts a time into a Unix timestamp in millisecond +// units. +func timeToUnixMilliseconds(tm time.Time) uint64 { + return uint64(tm.UnixNano()) / uint64(1000*1000) +} + +// minorVersion takes a given version string and returns only the major and +// minor portions of it. If the input is malformed, it returns the input +// untouched. +func minorVersion(v string) string { + split := strings.SplitN(v, ".", 3) + if len(split) < 2 { + return v + } + return split[0] + "." + split[1] +} diff --git a/go-telemetry-api-extension/telemetryApi/jsonWriter.go b/go-telemetry-api-extension/telemetryApi/jsonWriter.go new file mode 100644 index 00000000..4381d16a --- /dev/null +++ b/go-telemetry-api-extension/telemetryApi/jsonWriter.go @@ -0,0 +1,64 @@ +package telemetryApi + +import ( + "bytes" +) + +type jsonWriter interface { + WriteJSON(buf *bytes.Buffer) +} + +type jsonFieldsWriter struct { + buf *bytes.Buffer + needsComma bool +} + +func (w *jsonFieldsWriter) addKey(key string) { + if w.needsComma { + w.buf.WriteByte(',') + } else { + w.needsComma = true + } + // defensively assume that the key needs escaping: + AppendString(w.buf, key) + w.buf.WriteByte(':') +} + +func (w *jsonFieldsWriter) stringField(key string, val string) { + w.addKey(key) + AppendString(w.buf, val) +} + +func (w *jsonFieldsWriter) intField(key string, val int64) { + w.addKey(key) + AppendInt(w.buf, val) +} + +func (w *jsonFieldsWriter) floatField(key string, val float64) { + w.addKey(key) + AppendFloat(w.buf, val) +} + +func (w *jsonFieldsWriter) float32Field(key string, val float32) { + w.addKey(key) + AppendFloat32(w.buf, val) +} + +func (w *jsonFieldsWriter) boolField(key string, val bool) { + w.addKey(key) + if val { + w.buf.WriteString("true") + } else { + w.buf.WriteString("false") + } +} + +func (w *jsonFieldsWriter) rawField(key string, val jsonString) { + w.addKey(key) + w.buf.WriteString(string(val)) +} + +func (w *jsonFieldsWriter) writerField(key string, val jsonWriter) { + w.addKey(key) + val.WriteJSON(w.buf) +} diff --git a/go-telemetry-api-extension/telemetryApi/new-relic-logs.go b/go-telemetry-api-extension/telemetryApi/new-relic-logs.go new file mode 100644 index 00000000..d2a8e5f8 --- /dev/null +++ b/go-telemetry-api-extension/telemetryApi/new-relic-logs.go @@ -0,0 +1,102 @@ +package telemetryApi + +import ( + "bytes" +) + +const ( + // LogLevelFieldName is the name of the log level field in New Relic logging JSON + LogLevelFieldName = "level" + + // LogMessageFieldName is the name of the log message field in New Relic logging JSON + LogMessageFieldName = "message" + + // LogTimestampFieldName is the name of the timestamp field in New Relic logging JSON + LogTimestampFieldName = "timestamp" + + // LogSpanIDFieldName is the name of the span ID field in the New Relic logging JSON + LogSpanIDFieldName = "span.id" + + // LogTraceIDFieldName is the name of the trace ID field in the New Relic logging JSON + LogTraceIDFieldName = "trace.id" + + // LogSeverityUnknown is the value the log severity should be set to if no log severity is known + LogSeverityUnknown = "UNKNOWN" + + // JSON Attribute Constants + HostnameAttributeKey = "hostname" + EntityNameAttributeKey = "entity.name" + entityGUIDAttributeKey = "entity.guid" + + MaxLogLength = 32768 +) + +type LogPayload struct { + *bytes.Buffer + done bool +} + +// NewLogLine creates an object for processing a single log line and sending it to New Relic +func NewLogPayload(commonAttributes map[string]string) *LogPayload { + buf := bytes.NewBuffer([]byte{}) + buf.WriteByte('[') + buf.WriteByte('{') + buf.WriteString(`"common":`) + buf.WriteByte('{') + buf.WriteString(`"attributes":`) + buf.WriteByte('{') + + for name, value := range commonAttributes { + name = "\"" + name + "\":" + buf.WriteString(name) + AppendString(buf, value) + buf.WriteByte(',') + } + buf.WriteByte('}') + buf.WriteByte('}') + buf.WriteByte(',') + buf.WriteString(`"logs":`) + buf.WriteByte('[') + + return &LogPayload{Buffer: buf} +} + +// AddLogLine prepares a Log Event JSON object in the format expected by the collector. +// Timestamp must be unix millisecond time +func (buf *LogPayload) AddLogLine(Timestamp int64, Level, Message string) { + if buf.done { + return + } + + if Level == "" { + Level = LogSeverityUnknown + } + + if len(Message) > MaxLogLength { + Message = Message[:MaxLogLength] + } + + w := jsonFieldsWriter{buf: buf.Buffer} + buf.WriteByte('{') + w.stringField(LogLevelFieldName, Level) + w.stringField(LogMessageFieldName, Message) + + w.needsComma = false + buf.WriteByte(',') + w.intField(LogTimestampFieldName, Timestamp) + buf.WriteByte('}') +} + +func (buf *LogPayload) Marshal() []byte { + if buf.done { + return buf.Bytes() + } + + // prevent Duplication of JSON closure + buf.done = true + + buf.WriteByte(']') + buf.WriteByte('}') + buf.WriteByte(']') + return buf.Bytes() +} diff --git a/go-telemetry-api-extension/telemetryApi/send_to_new_relic.go b/go-telemetry-api-extension/telemetryApi/send_to_new_relic.go index 426adb63..e08a76af 100644 --- a/go-telemetry-api-extension/telemetryApi/send_to_new_relic.go +++ b/go-telemetry-api-extension/telemetryApi/send_to_new_relic.go @@ -5,13 +5,26 @@ import ( "context" "encoding/json" "fmt" - "errors" - "net/http" -// "github.com/pkg/errors" + "net/http" ) -func sendDataToNR(ctx context.Context, logEntries []interface{}, d *Dispatcher) (error) { +func sendDataToNR(ctx context.Context, logEntries []interface{}, d *Dispatcher) error { + + /* + logAttributes := map[string]string{} + logAttributes[HostnameAttributeKey] = "a host name" + logAttributes["mykey"] = "my value" + + payload := NewLogPayload(logAttributes) + + for _, logLine := range logEntries{ + // do some processing and add line to payload + payload.AddLogLine(time.Now().UnixMilli(), "debug", "message") + } + + bodyBytes := payload.Marshal() + */ bodyBytes, _ := json.Marshal(map[string]string{"message": fmt.Sprintf("%v", logEntries)}) req, err := http.NewRequestWithContext(ctx, "POST", d.postUri, bytes.NewBuffer(bodyBytes))