diff --git a/config/config.go b/config/config.go index 6cd0accb..6a69ed58 100644 --- a/config/config.go +++ b/config/config.go @@ -23,6 +23,7 @@ import ( // Metric contains values that define a metric type Metric struct { Name string + Engine EngineType Path string Labels map[string]string Type ScrapeType @@ -44,7 +45,14 @@ type ValueType string const ( ValueTypeGauge ValueType = "gauge" ValueTypeCounter ValueType = "counter" - ValueTypeUntyped ValueType = "untyped" + ValueTypeUntyped ValueType = "untyped" // default +) + +type EngineType string + +const ( + EngineTypeJSONPath EngineType = "jsonpath" // default + EngineTypeCEL EngineType = "cel" ) // Config contains multiple modules. @@ -89,6 +97,9 @@ func LoadConfig(configPath string) (Config, error) { if module.Metrics[i].ValueType == "" { module.Metrics[i].ValueType = ValueTypeUntyped } + if module.Metrics[i].Engine == "" { + module.Metrics[i].Engine = EngineTypeJSONPath + } } } diff --git a/examples/config.yml b/examples/config.yml index 9d0745c0..735acb95 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -9,7 +9,17 @@ modules: help: Example of a top-level global value scrape in the json labels: environment: beta # static label - location: 'planet-{.location}' # dynamic label + location: 'planet-{ .location }' # dynamic label + + - name: example_cel_global_value + engine: cel + path: '.counter' + help: Example of a top-level global value scrape in the json using cel + valuetype: 'gauge' + labels: + environment: "\"beta\"" # static label. Quotes need to be escaped for CEL + location: "\"planet-\"+.location" # dynamic label. Quotes need to be escaped for CEL + - name: example_timestamped_value type: object path: '{ .values[?(@.state == "INACTIVE")] }' @@ -18,7 +28,19 @@ modules: labels: environment: beta # static label values: - count: '{.count}' # dynamic value + count: '{ .count }' # dynamic value + + - name: example_cel_timestamped_value + type: object + engine: cel + path: ".values.filter(i, i.state == \"INACTIVE\")" + epochTimestamp: '.timestamp' + help: Example of a timestamped value scrape in the json + labels: + environment: "\"beta\"" # static label + values: + count: '.count' # dynamic value + - name: example_value type: object help: Example of sub-level value scrapes from a json @@ -31,6 +53,20 @@ modules: count: '{.count}' # dynamic value boolean: '{.some_boolean}' + - name: example_cel_value + type: object + engine: cel + help: Example of sub-level value scrapes from a json + path: ".values.filter(i, i.state == \"ACTIVE\")" + labels: + environment: "\"beta\"" # static label + id: '.id' # dynamic label + values: + active: 1 # static value + count: '.count' # dynamic value + boolean: '.some_boolean' + + animals: metrics: - name: animal diff --git a/exporter/collector.go b/exporter/collector.go index 4effc10f..b1a02eda 100644 --- a/exporter/collector.go +++ b/exporter/collector.go @@ -16,12 +16,19 @@ package exporter import ( "bytes" "encoding/json" + "fmt" + "reflect" "time" "github.com/go-kit/log" "github.com/go-kit/log/level" + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/types/ref" "github.com/prometheus-community/json_exporter/config" "github.com/prometheus/client_golang/prometheus" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + structpb "google.golang.org/protobuf/types/known/structpb" "k8s.io/client-go/util/jsonpath" ) @@ -34,6 +41,7 @@ type JSONMetricCollector struct { type JSONMetric struct { Desc *prometheus.Desc Type config.ScrapeType + EngineType config.EngineType KeyJSONPath string ValueJSONPath string LabelsJSONPaths []string @@ -51,7 +59,8 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { for _, m := range mc.JSONMetrics { switch m.Type { case config.ValueScrape: - value, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, false) + level.Debug(mc.Logger).Log("msg", "Extracting value for metric", "path", m.KeyJSONPath, "metric", m.Desc) + value, err := extractValue(mc.Logger, m.EngineType, mc.Data, m.KeyJSONPath, false) if err != nil { level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc) continue @@ -62,7 +71,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { m.Desc, m.ValueType, floatValue, - extractLabels(mc.Logger, mc.Data, m.LabelsJSONPaths)..., + extractLabels(mc.Logger, m.EngineType, mc.Data, m.LabelsJSONPaths)..., ) ch <- timestampMetric(mc.Logger, m, mc.Data, metric) } else { @@ -71,7 +80,8 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { } case config.ObjectScrape: - values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true) + level.Debug(mc.Logger).Log("msg", "Extracting object for metric", "path", m.KeyJSONPath, "metric", m.Desc) + values, err := extractValue(mc.Logger, m.EngineType, mc.Data, m.KeyJSONPath, true) if err != nil { level.Error(mc.Logger).Log("msg", "Failed to extract json objects for metric", "err", err, "metric", m.Desc) continue @@ -85,7 +95,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { level.Error(mc.Logger).Log("msg", "Failed to marshal data to json", "path", m.ValueJSONPath, "err", err, "metric", m.Desc, "data", data) continue } - value, err := extractValue(mc.Logger, jdata, m.ValueJSONPath, false) + value, err := extractValue(mc.Logger, m.EngineType, jdata, m.ValueJSONPath, false) if err != nil { level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.ValueJSONPath, "err", err, "metric", m.Desc) continue @@ -96,7 +106,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { m.Desc, m.ValueType, floatValue, - extractLabels(mc.Logger, jdata, m.LabelsJSONPaths)..., + extractLabels(mc.Logger, m.EngineType, jdata, m.LabelsJSONPaths)..., ) ch <- timestampMetric(mc.Logger, m, jdata, metric) } else { @@ -105,7 +115,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { } } } else { - level.Error(mc.Logger).Log("msg", "Failed to convert extracted objects to json", "err", err, "metric", m.Desc) + level.Error(mc.Logger).Log("msg", "Failed to convert extracted objects to json", "value", values, "err", err, "metric", m.Desc) continue } default: @@ -115,8 +125,19 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { } } +func extractValue(logger log.Logger, engine config.EngineType, data []byte, path string, enableJSONOutput bool) (string, error) { + switch engine { + case config.EngineTypeJSONPath: + return extractValueJSONPath(logger, data, path, enableJSONOutput) + case config.EngineTypeCEL: + return extractValueCEL(logger, data, path, enableJSONOutput) + default: + return "", fmt.Errorf("Unknown engine type: %s", engine) + } +} + // Returns the last matching value at the given json path -func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput bool) (string, error) { +func extractValueJSONPath(logger log.Logger, data []byte, path string, enableJSONOutput bool) (string, error) { var jsonData interface{} buf := new(bytes.Buffer) @@ -148,11 +169,71 @@ func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput return buf.String(), nil } +// Returns the last matching value at the given json path +func extractValueCEL(logger log.Logger, data []byte, expression string, enableJSONOutput bool) (string, error) { + + var jsonData map[string]any + + err := json.Unmarshal(data, &jsonData) + if err != nil { + level.Error(logger).Log("msg", "Failed to unmarshal data to json", "err", err, "data", data) + return "", err + } + + inputVars := make([]cel.EnvOption, 0, len(jsonData)) + for k := range jsonData { + inputVars = append(inputVars, cel.Variable(k, cel.DynType)) + } + + env, err := cel.NewEnv(inputVars...) + + if err != nil { + level.Error(logger).Log("msg", "Failed to set up CEL environment", "err", err, "data", data) + return "", err + } + + ast, issues := env.Compile(expression) + if issues != nil && issues.Err() != nil { + level.Error(logger).Log("CEL type-check error", issues.Err(), "expression", expression) + return "", err + } + prg, err := env.Program(ast) + if err != nil { + level.Error(logger).Log("CEL program construction error", err) + return "", err + } + + out, _, err := prg.Eval(jsonData) + if err != nil { + level.Error(logger).Log("msg", "Failed to evaluate cel query", "err", err, "expression", expression, "data", jsonData) + return "", err + } + + // Since we are finally going to extract only float64, unquote if necessary + + //res, err := jsonpath.UnquoteExtend(fmt.Sprintf("%g", out)) + //if err == nil { + // level.Error(logger).Log("msg","Triggered") + // return res, nil + //} + level.Error(logger).Log("msg", "Triggered later", "val", out) + if enableJSONOutput { + res, err := valueToJSON(out) + if err != nil { + return "", err + } else { + return res, nil + } + } + + return fmt.Sprintf("%v", out), nil +} + // Returns the list of labels created from the list of provided json paths -func extractLabels(logger log.Logger, data []byte, paths []string) []string { +func extractLabels(logger log.Logger, engine config.EngineType, data []byte, paths []string) []string { labels := make([]string, len(paths)) for i, path := range paths { - if result, err := extractValue(logger, data, path, false); err == nil { + if result, err := extractValue(logger, engine, data, path, false); err == nil { labels[i] = result } else { level.Error(logger).Log("msg", "Failed to extract label value", "err", err, "path", path, "data", data) @@ -165,7 +246,7 @@ func timestampMetric(logger log.Logger, m JSONMetric, data []byte, pm prometheus if m.EpochTimestampJSONPath == "" { return pm } - ts, err := extractValue(logger, data, m.EpochTimestampJSONPath, false) + ts, err := extractValue(logger, m.EngineType, data, m.EpochTimestampJSONPath, false) if err != nil { level.Error(logger).Log("msg", "Failed to extract timestamp for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc) return pm @@ -178,3 +259,18 @@ func timestampMetric(logger log.Logger, m JSONMetric, data []byte, pm prometheus timestamp := time.UnixMilli(epochTime) return prometheus.NewMetricWithTimestamp(timestamp, pm) } + +// valueToJSON converts the CEL type to a protobuf JSON representation and +// marshals the result to a string. +func valueToJSON(val ref.Val) (string, error) { + v, err := val.ConvertToNative(reflect.TypeOf(&structpb.Value{})) + if err != nil { + return "", err + } + marshaller := protojson.MarshalOptions{Indent: " "} + bytes, err := marshaller.Marshal(v.(proto.Message)) + if err != nil { + return "", err + } + return string(bytes), err +} diff --git a/exporter/util.go b/exporter/util.go index 8374ddce..42e1077d 100644 --- a/exporter/util.go +++ b/exporter/util.go @@ -103,6 +103,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) { variableLabels, nil, ), + EngineType: metric.Engine, KeyJSONPath: metric.Path, LabelsJSONPaths: variableLabelsValues, ValueType: valueType, @@ -125,6 +126,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) { variableLabels, nil, ), + EngineType: metric.Engine, KeyJSONPath: metric.Path, ValueJSONPath: valuePath, LabelsJSONPaths: variableLabelsValues, @@ -134,7 +136,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) { metrics = append(metrics, jsonMetric) } default: - return nil, fmt.Errorf("Unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name) + return nil, fmt.Errorf("unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name) } } return metrics, nil diff --git a/go.mod b/go.mod index b70ce1db..3dfb863c 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,11 @@ require ( github.com/Masterminds/sprig/v3 v3.2.3 github.com/alecthomas/kingpin/v2 v2.3.2 github.com/go-kit/log v0.2.1 + github.com/google/cel-go v0.18.2 github.com/prometheus/client_golang v1.17.0 github.com/prometheus/common v0.45.0 github.com/prometheus/exporter-toolkit v0.10.0 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/client-go v0.28.3 ) @@ -17,6 +19,7 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect @@ -35,13 +38,16 @@ require ( github.com/prometheus/procfs v0.11.1 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect golang.org/x/crypto v0.14.0 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect ) diff --git a/go.sum b/go.sum index 9f931237..9b5d2ac7 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWr github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -27,6 +29,8 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/cel-go v0.18.2 h1:L0B6sNBSVmt0OyECi8v6VOS74KOc9W/tLiWKfZABvf4= +github.com/google/cel-go v0.18.2/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -66,6 +70,8 @@ github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXY github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -79,6 +85,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -118,6 +126,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=