Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
lib/protoparser: adds opentelemetry parser
app/{vmagent,vminsert}: adds opentelemetry ingestion path
Adds ability to ingest data with opentelemetry protocol
protobuf and json encoding is supported
data converted into prometheus protobuf timeseries
each data type has own converter and it may produce multiple timeseries
from single datapoint (for summary and histogram).
only cumulative aggregationFamily is supported for sum(prometheus
counter) and histogram.
Apply suggestions from code review
Co-authored-by: Roman Khavronenko <roman@victoriametrics.com>- Loading branch information
Showing
162 changed files
with
72,985 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| { | ||
| "name": "ingest all types", | ||
| "query": ["/api/v1/export?match={__name__=~'(my-gauge|my-histogram|my-sum|my-summary).*'}"], | ||
| "result_metrics": [ | ||
| {"metric":{"__name__":"my-gauge","job":"vm","label1":"value1"},"values":[15], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-sum","job":"vm","label5":"value5"},"values":[15.5], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-summary","job":"vm","label6":"value6","quantile": "0.1"},"values":[7.5], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-summary","job":"vm","label6":"value6","quantile": "1"},"values":[15], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-summary","job":"vm","label6":"value6","quantile": "0.5"},"values":[10], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-summary_count","job":"vm","label6":"value6"},"values":[5], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-summary_sum","job":"vm","label6":"value6"},"values":[32.5], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_sum","job":"vm","label2":"value2"},"values":[30], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_count","job":"vm","label2":"value2"},"values":[15], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_bucket","job":"vm","label2":"value2","le": "0.1"},"values":[0], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_bucket","job":"vm","label2":"value2","le": "0.5"},"values":[5], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_bucket","job":"vm","label2":"value2","le": "1"},"values":[15], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_bucket","job":"vm","label2":"value2","le": "5"},"values":[15], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_bucket","job":"vm","label2":"value2","le": "+Inf"},"values":[15], "timestamps": ["{TIME_MS}"]} | ||
| ], | ||
| "data": ["{\"resourceMetrics\":[{\"resource\":{\"attributes\":[{\"key\":\"job\",\"value\":{\"stringValue\":\"vm\"}}]},\"scopeMetrics\":[{\"metrics\":[{\"name\":\"my-gauge\",\"gauge\":{\"dataPoints\":[{\"attributes\":[{\"key\":\"label1\",\"value\":{\"stringValue\":\"value1\"}}],\"timeUnixNano\":\"{TIME_NS}\",\"asInt\":\"15\"}]}},{\"name\":\"my-histogram\",\"histogram\":{\"dataPoints\":[{\"attributes\":[{\"key\":\"label2\",\"value\":{\"stringValue\":\"value2\"}}],\"timeUnixNano\":\"{TIME_NS}\",\"count\":\"15\",\"sum\":30,\"bucketCounts\":[\"0\",\"5\",\"10\",\"0\",\"0\"],\"explicitBounds\":[0.1,0.5,1,5]}],\"aggregationTemporality\":\"AGGREGATION_TEMPORALITY_CUMULATIVE\"}},{\"name\":\"my-sum\",\"sum\":{\"dataPoints\":[{\"attributes\":[{\"key\":\"label5\",\"value\":{\"stringValue\":\"value5\"}}],\"timeUnixNano\":\"{TIME_NS}\",\"asDouble\":15.5}],\"aggregationTemporality\":\"AGGREGATION_TEMPORALITY_CUMULATIVE\"}},{\"name\":\"my-summary\",\"summary\":{\"dataPoints\":[{\"attributes\":[{\"key\":\"label6\",\"value\":{\"stringValue\":\"value6\"}}],\"timeUnixNano\":\"{TIME_NS}\",\"count\":\"5\",\"sum\":32.5,\"quantileValues\":[{\"quantile\":0.1,\"value\":7.5},{\"quantile\":0.5,\"value\":10},{\"quantile\":1,\"value\":15}]}]}}]}]}]}"] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| package opentelemetryhttp | ||
|
|
||
| import ( | ||
| "net/http" | ||
|
|
||
| "github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" | ||
| parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" | ||
| parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter" | ||
| "github.com/VictoriaMetrics/metrics" | ||
| ) | ||
|
|
||
| var ( | ||
| rowsInserted = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`) | ||
| rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`) | ||
| rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`) | ||
| ) | ||
|
|
||
| // InsertHandler processes opentelemetry metrics. | ||
| func InsertHandler(at *auth.Token, req *http.Request) error { | ||
| extraLabels, err := parserCommon.GetExtraLabels(req) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| isJSON := req.Header.Get("Content-Type") == "application/json" | ||
| isGzipped := req.Header.Get("Content-Encoding") == "gzip" | ||
| return writeconcurrencylimiter.Do(func() error { | ||
| return parser.ParseStream(req.Body, isJSON, isGzipped, func(tss []prompb.TimeSeries) error { | ||
| return insertRows(at, tss, extraLabels) | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error { | ||
| ctx := common.GetPushCtx() | ||
| defer common.PutPushCtx(ctx) | ||
|
|
||
| rowsTotal := 0 | ||
| tssDst := ctx.WriteRequest.Timeseries[:0] | ||
| labels := ctx.Labels[:0] | ||
| samples := ctx.Samples[:0] | ||
| for i := range timeseries { | ||
| ts := ×eries[i] | ||
| rowsTotal += len(ts.Samples) | ||
| labelsLen := len(labels) | ||
| for i := range ts.Labels { | ||
| label := &ts.Labels[i] | ||
| labels = append(labels, prompbmarshal.Label{ | ||
| Name: bytesutil.ToUnsafeString(label.Name), | ||
| Value: bytesutil.ToUnsafeString(label.Value), | ||
| }) | ||
| } | ||
| labels = append(labels, extraLabels...) | ||
| samplesLen := len(samples) | ||
| for i := range ts.Samples { | ||
| sample := &ts.Samples[i] | ||
| samples = append(samples, prompbmarshal.Sample{ | ||
| Value: sample.Value, | ||
| Timestamp: sample.Timestamp, | ||
| }) | ||
| } | ||
| tssDst = append(tssDst, prompbmarshal.TimeSeries{ | ||
| Labels: labels[labelsLen:], | ||
| Samples: samples[samplesLen:], | ||
| }) | ||
| } | ||
| ctx.WriteRequest.Timeseries = tssDst | ||
| ctx.Labels = labels | ||
| ctx.Samples = samples | ||
| remotewrite.PushWithAuthToken(at, &ctx.WriteRequest) | ||
| rowsInserted.Add(rowsTotal) | ||
| if at != nil { | ||
| rowsTenantInserted.Get(at).Add(rowsTotal) | ||
| } | ||
| rowsPerInsert.Update(float64(rowsTotal)) | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| package opentelemetryhttp | ||
|
|
||
| import ( | ||
| "net/http" | ||
|
|
||
| "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" | ||
| parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" | ||
| parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter" | ||
| "github.com/VictoriaMetrics/metrics" | ||
| ) | ||
|
|
||
| var ( | ||
| rowsInserted = metrics.NewCounter(`vm_rows_inserted_total{type="opentelemetry"}`) | ||
| rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="opentelemetry"}`) | ||
| ) | ||
|
|
||
| // InsertHandler processes opentelemetry metrics. | ||
| func InsertHandler(req *http.Request) error { | ||
| extraLabels, err := parserCommon.GetExtraLabels(req) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| isJSON := req.Header.Get("Content-Type") == "application/json" | ||
| isGzipped := req.Header.Get("Content-Encoding") == "gzip" | ||
| return writeconcurrencylimiter.Do(func() error { | ||
| return parser.ParseStream(req.Body, isJSON, isGzipped, func(tss []prompb.TimeSeries) error { | ||
| return insertRows(tss, extraLabels) | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| func insertRows(timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error { | ||
| ctx := common.GetInsertCtx() | ||
| defer common.PutInsertCtx(ctx) | ||
|
|
||
| rowsLen := 0 | ||
| for i := range timeseries { | ||
| rowsLen += len(timeseries[i].Samples) | ||
| } | ||
| ctx.Reset(rowsLen) | ||
| rowsTotal := 0 | ||
| hasRelabeling := relabel.HasRelabeling() | ||
| for i := range timeseries { | ||
| ts := ×eries[i] | ||
| rowsTotal += len(ts.Samples) | ||
| ctx.Labels = ctx.Labels[:0] | ||
| srcLabels := ts.Labels | ||
| for _, srcLabel := range srcLabels { | ||
| ctx.AddLabelBytes(srcLabel.Name, srcLabel.Value) | ||
| } | ||
| for j := range extraLabels { | ||
| label := &extraLabels[j] | ||
| ctx.AddLabel(label.Name, label.Value) | ||
| } | ||
| if hasRelabeling { | ||
| ctx.ApplyRelabeling() | ||
| } | ||
| if len(ctx.Labels) == 0 { | ||
| // Skip metric without labels. | ||
| continue | ||
| } | ||
| ctx.SortLabelsIfNeeded() | ||
| var metricNameRaw []byte | ||
| var err error | ||
| samples := ts.Samples | ||
| for i := range samples { | ||
| r := &samples[i] | ||
| metricNameRaw, err = ctx.WriteDataPointExt(metricNameRaw, ctx.Labels, r.Timestamp, r.Value) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| } | ||
| rowsInserted.Add(rowsTotal) | ||
| rowsPerInsert.Update(float64(rowsTotal)) | ||
| return ctx.FlushBufs() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.