Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
lib/protoparser: adds opentelemetry parser
app/{vmagent,vminsert}: adds opentelemetry ingestion path
Adds ability to ingest data with opentelemetry protocol
protobuf and json encoding is supported
data converted into prometheus protobuf timeseries
each data type has own converter and it may produce multiple timeseries
from single datapoint (for summary and histogram).
only cumulative aggregationFamily is supported for sum(prometheus
counter) and histogram.
Apply suggestions from code review
Co-authored-by: Roman Khavronenko <roman@victoriametrics.com>
updates deps
fixes tests
wip
wip
wip
wip
lib/protoparser/opentelemetry: moves to vtprotobuf generator
go mod vendor
lib/protoparse/opentelemetry: reduce memory allocations- Loading branch information
Showing
31 changed files
with
10,261 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| { | ||
| "name": "ingest all types", | ||
| "query": ["/api/v1/export?match={__name__=~'(my-gauge|my-histogram|my-sum|my-summary).*'}"], | ||
| "result_metrics": [ | ||
| {"metric":{"__name__":"my-gauge","job":"vm","label1":"value1"},"values":[15], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-sum","job":"vm","label5":"value5"},"values":[15.5], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-summary","job":"vm","label6":"value6","quantile": "0.1"},"values":[7.5], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-summary","job":"vm","label6":"value6","quantile": "1"},"values":[15], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-summary","job":"vm","label6":"value6","quantile": "0.5"},"values":[10], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-summary_count","job":"vm","label6":"value6"},"values":[5], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-summary_sum","job":"vm","label6":"value6"},"values":[32.5], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_sum","job":"vm","label2":"value2"},"values":[30], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_count","job":"vm","label2":"value2"},"values":[15], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_bucket","job":"vm","label2":"value2","le": "0.1"},"values":[0], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_bucket","job":"vm","label2":"value2","le": "0.5"},"values":[5], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_bucket","job":"vm","label2":"value2","le": "1"},"values":[15], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_bucket","job":"vm","label2":"value2","le": "5"},"values":[15], "timestamps": ["{TIME_MS}"]}, | ||
| {"metric":{"__name__":"my-histogram_bucket","job":"vm","label2":"value2","le": "+Inf"},"values":[15], "timestamps": ["{TIME_MS}"]} | ||
| ], | ||
| "data": ["{\"resourceMetrics\":[{\"resource\":{\"attributes\":[{\"key\":\"job\",\"value\":{\"stringValue\":\"vm\"}}]},\"scopeMetrics\":[{\"metrics\":[{\"name\":\"my-gauge\",\"gauge\":{\"dataPoints\":[{\"attributes\":[{\"key\":\"label1\",\"value\":{\"stringValue\":\"value1\"}}],\"timeUnixNano\":\"{TIME_NS}\",\"asInt\":\"15\"}]}},{\"name\":\"my-histogram\",\"histogram\":{\"dataPoints\":[{\"attributes\":[{\"key\":\"label2\",\"value\":{\"stringValue\":\"value2\"}}],\"timeUnixNano\":\"{TIME_NS}\",\"count\":\"15\",\"sum\":30,\"bucketCounts\":[\"0\",\"5\",\"10\",\"0\",\"0\"],\"explicitBounds\":[0.1,0.5,1,5]}],\"aggregationTemporality\":\"AGGREGATION_TEMPORALITY_CUMULATIVE\"}},{\"name\":\"my-sum\",\"sum\":{\"dataPoints\":[{\"attributes\":[{\"key\":\"label5\",\"value\":{\"stringValue\":\"value5\"}}],\"timeUnixNano\":\"{TIME_NS}\",\"asDouble\":15.5}],\"aggregationTemporality\":\"AGGREGATION_TEMPORALITY_CUMULATIVE\"}},{\"name\":\"my-summary\",\"summary\":{\"dataPoints\":[{\"attributes\":[{\"key\":\"label6\",\"value\":{\"stringValue\":\"value6\"}}],\"timeUnixNano\":\"{TIME_NS}\",\"count\":\"5\",\"sum\":32.5,\"quantileValues\":[{\"quantile\":0.1,\"value\":7.5},{\"quantile\":0.5,\"value\":10},{\"quantile\":1,\"value\":15}]}]}}]}]}]}"] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| package opentelemetryhttp | ||
|
|
||
| import ( | ||
| "net/http" | ||
|
|
||
| "github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" | ||
| parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" | ||
| parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry" | ||
| "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" | ||
| "github.com/VictoriaMetrics/metrics" | ||
| ) | ||
|
|
||
| var ( | ||
| rowsInserted = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`) | ||
| rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`) | ||
| rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`) | ||
| ) | ||
|
|
||
| // InsertHandler processes opentelemetry metrics. | ||
| func InsertHandler(at *auth.Token, req *http.Request) error { | ||
| extraLabels, err := parserCommon.GetExtraLabels(req) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| isJSON := req.Header.Get("Content-Type") == "application/json" | ||
| isGzipped := req.Header.Get("Content-Encoding") == "gzip" | ||
| return parser.ParseStream(req.Body, isJSON, isGzipped, func(tss []prompb.TimeSeries) error { | ||
| return insertRows(at, tss, extraLabels) | ||
| }) | ||
| } | ||
|
|
||
| func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error { | ||
| ctx := common.GetPushCtx() | ||
| defer common.PutPushCtx(ctx) | ||
|
|
||
| rowsTotal := 0 | ||
| tssDst := ctx.WriteRequest.Timeseries[:0] | ||
| labels := ctx.Labels[:0] | ||
| samples := ctx.Samples[:0] | ||
| for i := range timeseries { | ||
| ts := ×eries[i] | ||
| rowsTotal += len(ts.Samples) | ||
| labelsLen := len(labels) | ||
| for i := range ts.Labels { | ||
| label := &ts.Labels[i] | ||
| labels = append(labels, prompbmarshal.Label{ | ||
| Name: bytesutil.ToUnsafeString(label.Name), | ||
| Value: bytesutil.ToUnsafeString(label.Value), | ||
| }) | ||
| } | ||
| labels = append(labels, extraLabels...) | ||
| samplesLen := len(samples) | ||
| for i := range ts.Samples { | ||
| sample := &ts.Samples[i] | ||
| samples = append(samples, prompbmarshal.Sample{ | ||
| Value: sample.Value, | ||
| Timestamp: sample.Timestamp, | ||
| }) | ||
| } | ||
| tssDst = append(tssDst, prompbmarshal.TimeSeries{ | ||
| Labels: labels[labelsLen:], | ||
| Samples: samples[samplesLen:], | ||
| }) | ||
| } | ||
| ctx.WriteRequest.Timeseries = tssDst | ||
| ctx.Labels = labels | ||
| ctx.Samples = samples | ||
| remotewrite.Push(at, &ctx.WriteRequest) | ||
| rowsInserted.Add(rowsTotal) | ||
| if at != nil { | ||
| rowsTenantInserted.Get(at).Add(rowsTotal) | ||
| } | ||
| rowsPerInsert.Update(float64(rowsTotal)) | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.