Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

WIP: Initial support for CEL #267

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

WIP: Initial support for CEL #267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
20 changes: 19 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ json_exporter
========================
[![CircleCI](https://circleci.com/gh/prometheus-community/json_exporter.svg?style=svg)](https://circleci.com/gh/prometheus-community/json_exporter)

A [prometheus](https://prometheus.io/) exporter which scrapes remote JSON by JSONPath.
A [prometheus](https://prometheus.io/) exporter which scrapes remote JSON by JSONPath or [CEL (Common Expression Language)](https://github.com/google/cel-spec).

- [Supported JSONPath Syntax](https://kubernetes.io/docs/reference/kubectl/jsonpath/)
- [Examples configurations](/examples)
Expand All @@ -21,6 +21,24 @@ Serving HTTP on :: port 8000 (http://[::]:8000/) ...
## TEST with 'default' module

$ curl "http://localhost:7979/probe?module=default&target=http://localhost:8000/examples/data.json"
# HELP example_cel_global_value Example of a top-level global value scrape in the json using cel
# TYPE example_cel_global_value gauge
example_cel_global_value{environment="beta",location="planet-mars"} 1234
# HELP example_cel_timestamped_value_count Example of a timestamped value scrape in the json
# TYPE example_cel_timestamped_value_count untyped
example_cel_timestamped_value_count{environment="beta"} 2
# HELP example_cel_value_active Example of sub-level value scrapes from a json
# TYPE example_cel_value_active untyped
example_cel_value_active{environment="beta",id="id-A"} 1
example_cel_value_active{environment="beta",id="id-C"} 1
# HELP example_cel_value_boolean Example of sub-level value scrapes from a json
# TYPE example_cel_value_boolean untyped
example_cel_value_boolean{environment="beta",id="id-A"} 1
example_cel_value_boolean{environment="beta",id="id-C"} 0
# HELP example_cel_value_count Example of sub-level value scrapes from a json
# TYPE example_cel_value_count untyped
example_cel_value_count{environment="beta",id="id-A"} 1
example_cel_value_count{environment="beta",id="id-C"} 3
# HELP example_global_value Example of a top-level global value scrape in the json
# TYPE example_global_value untyped
example_global_value{environment="beta",location="planet-mars"} 1234
Expand Down
13 changes: 12 additions & 1 deletion config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
// Metric contains values that define a metric
type Metric struct {
Name string
Engine EngineType
Path string
Labels map[string]string
Type ScrapeType
Expand All @@ -44,7 +45,14 @@ type ValueType string
const (
ValueTypeGauge ValueType = "gauge"
ValueTypeCounter ValueType = "counter"
ValueTypeUntyped ValueType = "untyped"
ValueTypeUntyped ValueType = "untyped" // default
)

type EngineType string

const (
EngineTypeJSONPath EngineType = "jsonpath" // default
EngineTypeCEL EngineType = "cel"
)

// Config contains multiple modules.
Expand Down Expand Up @@ -89,6 +97,9 @@ func LoadConfig(configPath string) (Config, error) {
if module.Metrics[i].ValueType == "" {
module.Metrics[i].ValueType = ValueTypeUntyped
}
if module.Metrics[i].Engine == "" {
module.Metrics[i].Engine = EngineTypeJSONPath
}
}
}

Expand Down
63 changes: 55 additions & 8 deletions examples/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,17 @@ modules:
help: Example of a top-level global value scrape in the json
labels:
environment: beta # static label
location: 'planet-{.location}' # dynamic label
location: 'planet-{ .location }' # dynamic label

- name: example_cel_global_value
engine: cel
path: '.counter'
help: Example of a top-level global value scrape in the json using cel
valuetype: 'gauge'
labels:
environment: "\"beta\"" # static label. Quotes need to be escaped for CEL
location: "\"planet-\"+.location" # dynamic label. Quotes need to be escaped for CEL

- name: example_timestamped_value
type: object
path: '{ .values[?(@.state == "INACTIVE")] }'
Expand All @@ -18,18 +28,44 @@ modules:
labels:
environment: beta # static label
values:
count: '{.count}' # dynamic value
count: '{ .count }' # dynamic value

- name: example_cel_timestamped_value
type: object
engine: cel
path: ".values.filter(i, i.state == \"INACTIVE\")"
epochTimestamp: '.timestamp'
help: Example of a timestamped value scrape in the json
labels:
environment: "\"beta\"" # static label
values:
count: '.count' # dynamic value

- name: example_value
type: object
help: Example of sub-level value scrapes from a json
path: '{.values[?(@.state == "ACTIVE")]}'
path: '{ .values[?(@.state == "ACTIVE")] }'
labels:
environment: beta # static label
id: '{.id}' # dynamic label
id: '{ .id }' # dynamic label
values:
active: 1 # static value
count: '{.count}' # dynamic value
boolean: '{.some_boolean}'
count: '{ .count }' # dynamic value
boolean: '{ .some_boolean }'

- name: example_cel_value
type: object
engine: cel
help: Example of sub-level value scrapes from a json
path: ".values.filter(i, i.state == \"ACTIVE\")"
labels:
environment: "\"beta\"" # static label
id: '.id' # dynamic label
values:
active: 1 # static value
count: '.count' # dynamic value
boolean: '.some_boolean'


animals:
metrics:
Expand All @@ -43,6 +79,17 @@ modules:
values:
population: '{ .population }'

- name: animal_cel
type: object
engine: cel
help: Example of top-level lists in a separate module
path: '[*]'
labels:
name: '.noun'
predator: '.predator'
values:
population: '.population'

## HTTP connection configurations can be set in 'modules.<module_name>.http_client_config' field. For full http client config parameters, ref: https://pkg.go.dev/github.com/prometheus/common/config?tab=doc#HTTPClientConfig
#
# http_client_config:
Expand All @@ -59,11 +106,11 @@ modules:
## If 'modueles.<module_name>.body' field is set, it will be sent by the exporter as the body content in the scrape request. The HTTP method will also be set as 'POST' in this case.
# body:
# content: |
# {"time_diff": "1m25s", "anotherVar": "some value"}
# { "time_diff": "1m25s", "anotherVar": "some value" }

## The body content can also be a Go Template (https://golang.org/pkg/text/template), with all the functions from the Sprig library (https://masterminds.github.io/sprig/) available. All the query parameters sent by prometheus in the scrape query to the exporter, are available in the template.
# body:
# content: |
# {"time_diff": "{{ duration `95` }}","anotherVar": "{{ .myVal | first }}"}
# { "time_diff": "{{ duration `95` }}","anotherVar": "{{ .myVal | first }}" }
# templatize: true

115 changes: 105 additions & 10 deletions exporter/collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,19 @@ package exporter
import (
"bytes"
"encoding/json"
"fmt"
"reflect"
"time"

"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/common/types/ref"
"github.com/prometheus-community/json_exporter/config"
"github.com/prometheus/client_golang/prometheus"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
structpb "google.golang.org/protobuf/types/known/structpb"
"k8s.io/client-go/util/jsonpath"
)

Expand All @@ -34,6 +41,7 @@ type JSONMetricCollector struct {
type JSONMetric struct {
Desc *prometheus.Desc
Type config.ScrapeType
EngineType config.EngineType
KeyJSONPath string
ValueJSONPath string
LabelsJSONPaths []string
Expand All @@ -51,7 +59,8 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
for _, m := range mc.JSONMetrics {
switch m.Type {
case config.ValueScrape:
value, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, false)
level.Debug(mc.Logger).Log("msg", "Extracting value for metric", "path", m.KeyJSONPath, "metric", m.Desc)
value, err := extractValue(mc.Logger, m.EngineType, mc.Data, m.KeyJSONPath, false)
if err != nil {
level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
continue
Expand All @@ -62,7 +71,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
m.Desc,
m.ValueType,
floatValue,
extractLabels(mc.Logger, mc.Data, m.LabelsJSONPaths)...,
extractLabels(mc.Logger, m.EngineType, mc.Data, m.LabelsJSONPaths)...,
)
ch <- timestampMetric(mc.Logger, m, mc.Data, metric)
} else {
Expand All @@ -71,7 +80,8 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
}

case config.ObjectScrape:
values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true)
level.Debug(mc.Logger).Log("msg", "Extracting object for metric", "path", m.KeyJSONPath, "metric", m.Desc)
values, err := extractValue(mc.Logger, m.EngineType, mc.Data, m.KeyJSONPath, true)
if err != nil {
level.Error(mc.Logger).Log("msg", "Failed to extract json objects for metric", "err", err, "metric", m.Desc)
continue
Expand All @@ -85,7 +95,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
level.Error(mc.Logger).Log("msg", "Failed to marshal data to json", "path", m.ValueJSONPath, "err", err, "metric", m.Desc, "data", data)
continue
}
value, err := extractValue(mc.Logger, jdata, m.ValueJSONPath, false)
value, err := extractValue(mc.Logger, m.EngineType, jdata, m.ValueJSONPath, false)
if err != nil {
level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.ValueJSONPath, "err", err, "metric", m.Desc)
continue
Expand All @@ -96,7 +106,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
m.Desc,
m.ValueType,
floatValue,
extractLabels(mc.Logger, jdata, m.LabelsJSONPaths)...,
extractLabels(mc.Logger, m.EngineType, jdata, m.LabelsJSONPaths)...,
)
ch <- timestampMetric(mc.Logger, m, jdata, metric)
} else {
Expand All @@ -105,7 +115,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
}
}
} else {
level.Error(mc.Logger).Log("msg", "Failed to convert extracted objects to json", "err", err, "metric", m.Desc)
level.Error(mc.Logger).Log("msg", "Failed to convert extracted objects to json", "value", values, "err", err, "metric", m.Desc)
continue
}
default:
Expand All @@ -115,8 +125,19 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
}
}

func extractValue(logger log.Logger, engine config.EngineType, data []byte, path string, enableJSONOutput bool) (string, error) {
switch engine {
case config.EngineTypeJSONPath:
return extractValueJSONPath(logger, data, path, enableJSONOutput)
case config.EngineTypeCEL:
return extractValueCEL(logger, data, path, enableJSONOutput)
default:
return "", fmt.Errorf("Unknown engine type: %s", engine)
}
}

// Returns the last matching value at the given json path
func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput bool) (string, error) {
func extractValueJSONPath(logger log.Logger, data []byte, path string, enableJSONOutput bool) (string, error) {
var jsonData interface{}
buf := new(bytes.Buffer)

Expand Down Expand Up @@ -148,11 +169,70 @@ func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput
return buf.String(), nil
}

// Returns the last matching value at the given json path
func extractValueCEL(logger log.Logger, data []byte, expression string, enableJSONOutput bool) (string, error) {

var jsonData map[string]any

err := json.Unmarshal(data, &jsonData)
if err != nil {
level.Error(logger).Log("msg", "Failed to unmarshal data to json", "err", err, "data", data)
return "", err
}

inputVars := make([]cel.EnvOption, 0, len(jsonData))
for k := range jsonData {
inputVars = append(inputVars, cel.Variable(k, cel.DynType))
}

env, err := cel.NewEnv(inputVars...)

if err != nil {
level.Error(logger).Log("msg", "Failed to set up CEL environment", "err", err, "data", data)
return "", err
}

ast, issues := env.Compile(expression)
if issues != nil && issues.Err() != nil {
level.Error(logger).Log("CEL type-check error", issues.Err(), "expression", expression)
return "", err
}
prg, err := env.Program(ast)
if err != nil {
level.Error(logger).Log("CEL program construction error", err)
return "", err
}

out, _, err := prg.Eval(jsonData)
if err != nil {
level.Error(logger).Log("msg", "Failed to evaluate cel query", "err", err, "expression", expression, "data", jsonData)
return "", err
}

// Since we are finally going to extract only float64, unquote if necessary

//res, err := jsonpath.UnquoteExtend(fmt.Sprintf("%g", out))
//if err == nil {
// level.Error(logger).Log("msg","Triggered")
// return res, nil
//}
level.Error(logger).Log("msg", "Triggered later", "val", out)
if enableJSONOutput {
res, err := valueToJSON(out)
if err != nil {
return "", err
}
return res, nil
}

return fmt.Sprintf("%v", out), nil
}

// Returns the list of labels created from the list of provided json paths
func extractLabels(logger log.Logger, data []byte, paths []string) []string {
func extractLabels(logger log.Logger, engine config.EngineType, data []byte, paths []string) []string {
labels := make([]string, len(paths))
for i, path := range paths {
if result, err := extractValue(logger, data, path, false); err == nil {
if result, err := extractValue(logger, engine, data, path, false); err == nil {
labels[i] = result
} else {
level.Error(logger).Log("msg", "Failed to extract label value", "err", err, "path", path, "data", data)
Expand All @@ -165,7 +245,7 @@ func timestampMetric(logger log.Logger, m JSONMetric, data []byte, pm prometheus
if m.EpochTimestampJSONPath == "" {
return pm
}
ts, err := extractValue(logger, data, m.EpochTimestampJSONPath, false)
ts, err := extractValue(logger, m.EngineType, data, m.EpochTimestampJSONPath, false)
if err != nil {
level.Error(logger).Log("msg", "Failed to extract timestamp for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
return pm
Expand All @@ -178,3 +258,18 @@ func timestampMetric(logger log.Logger, m JSONMetric, data []byte, pm prometheus
timestamp := time.UnixMilli(epochTime)
return prometheus.NewMetricWithTimestamp(timestamp, pm)
}

// valueToJSON converts the CEL type to a protobuf JSON representation and
// marshals the result to a string.
func valueToJSON(val ref.Val) (string, error) {
v, err := val.ConvertToNative(reflect.TypeOf(&structpb.Value{}))
if err != nil {
return "", err
}
marshaller := protojson.MarshalOptions{Indent: " "}
bytes, err := marshaller.Marshal(v.(proto.Message))
if err != nil {
return "", err
}
return string(bytes), err
}
4 changes: 3 additions & 1 deletion exporter/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
variableLabels,
nil,
),
EngineType: metric.Engine,
KeyJSONPath: metric.Path,
LabelsJSONPaths: variableLabelsValues,
ValueType: valueType,
Expand All @@ -125,6 +126,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
variableLabels,
nil,
),
EngineType: metric.Engine,
KeyJSONPath: metric.Path,
ValueJSONPath: valuePath,
LabelsJSONPaths: variableLabelsValues,
Expand All @@ -134,7 +136,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
metrics = append(metrics, jsonMetric)
}
default:
return nil, fmt.Errorf("Unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name)
return nil, fmt.Errorf("unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name)
}
}
return metrics, nil
Expand Down