Initial import

.gitignore

@@ -22,3 +22,6 @@ _testmain.go
 *.exe
 *.test
 *.prof
+
+/build
+/json_exporter

README.md

@@ -1,2 +1,33 @@
-# prometheus-json-exporter
-A prometheus exporter which scrapes remote JSON
+prometheus-json-exporter
+========================
+
+A [prometheus](https://prometheus.io/) exporter which scrapes remote JSON by JSONPath.
+
+Build
+=====
+```sh
+./gow get .
+./gow build -o json_exporter .
+```
+
+Example Usage
+=============
+```sh
+$ python -m SimpleHTTPServer 8000 &
+Serving HTTP on 0.0.0.0 port 8000 ...
+$ ./json_exporter http://localhost:8000/example/data.json example/config.yml &
+INFO[2016-02-08T22:44:38+09:00] metric registered;name:<example_global_value>
+INFO[2016-02-08T22:44:38+09:00] metric registered;name:<example_value_active>
+INFO[2016-02-08T22:44:38+09:00] metric registered;name:<example_value_count>
+127.0.0.1 - - [08/Feb/2016 22:44:38] "GET /example/data.json HTTP/1.1" 200 -
+$ curl http://localhost:7979/metrics | grep ^example
+example_global_value{environment="beta"} 1234
+example_value_active{environment="beta",id="id-A"} 1
+example_value_active{environment="beta",id="id-C"} 1
+example_value_count{environment="beta",id="id-A"} 1
+example_value_count{environment="beta",id="id-C"} 3
+```
+
+See Also
+========
+- [nicksardo/jsonpath](https://github.com/nicksardo/jsonpath) : For syntax reference of JSONPath

example/config.yml

@@ -0,0 +1,14 @@
+- name: example_global_value
+  path: $.counter
+  labels:
+    environment: beta # static label
+
+- name: example_value
+  type: object
+  path: $.values[*]?(@.state == "ACTIVE")
+  labels:
+    environment: beta # static label
+    id: $.id          # dynamic label
+  values:
+    active: 1      # static value
+    count: $.count # dynamic value

example/data.json

@@ -0,0 +1,20 @@
+{
+    "counter": 1234,
+    "values": [
+        {
+            "id": "id-A",
+            "count": 1,
+            "state": "ACTIVE"
+        },
+        {
+            "id": "id-B",
+            "count": 2,
+            "state": "INACTIVE"
+        },
+        {
+            "id": "id-C",
+            "count": 3,
+            "state": "ACTIVE"
+        },
+    ]
+}

gow

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+GO=go
+PACKAGE_LOCATION_GIT_REMOTE=origin
+PACKAGE_LOCATION=$(git config --get "remote.$PACKAGE_LOCATION_GIT_REMOTE.url" | sed 's/^[^@]*@//' | sed 's/^.*:\/\///' | sed 's/\.git$//')
+
+PROJECT_ROOT_DIR=$(cd $(dirname $0); pwd)
+GOPATH=$PROJECT_ROOT_DIR/build
+
+LOCAL_PACKAGE_PATH=$GOPATH/src/$(echo "$PACKAGE_LOCATION" | tr ':' '/')
+LOCAL_PACKAGE_DIR=$(dirname $LOCAL_PACKAGE_PATH)
+
+mkdir -p $LOCAL_PACKAGE_DIR
+ln -sfh $PROJECT_ROOT_DIR $LOCAL_PACKAGE_PATH
+
+export GOBIN="$GOPATH/bin"
+export GOPATH
+
+exec $GO "$@"

json_exporter.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"github.com/kawamuray/prometheus-exporter-harness/harness"
+	"github.com/kawamuray/prometheus-json-exporter/jsonexporter"
+)
+
+func main() {
+	opts := harness.NewExporterOpts("json_exporter", jsonexporter.Version)
+	opts.Usage = "[OPTIONS] HTTP_ENDPOINT CONFIG_PATH"
+	opts.Init = jsonexporter.Init
+	harness.Main(opts)
+}

jsonexporter/collector.go

@@ -0,0 +1,82 @@
+package jsonexporter
+
+import (
+	"fmt"
+	"github.com/NickSardo/jsonpath"
+	log "github.com/Sirupsen/logrus"
+	"github.com/kawamuray/prometheus-exporter-harness/harness"
+	"io/ioutil"
+	"net/http"
+)
+
+type Collector struct {
+	Endpoint string
+	scrapers []JsonScraper
+}
+
+func compilePath(path string) (*jsonpath.Path, error) {
+	// All paths in this package is for extracting a value.
+	// Complete trailing '+' sign if necessary.
+	if path[len(path)-1] != '+' {
+		path += "+"
+	}
+
+	paths, err := jsonpath.ParsePaths(path)
+	if err != nil {
+		return nil, err
+	}
+	return paths[0], nil
+}
+
+func compilePaths(paths map[string]string) (map[string]*jsonpath.Path, error) {
+	compiledPaths := make(map[string]*jsonpath.Path)
+	for name, value := range paths {
+		if len(value) < 1 || value[0] != '$' {
+			// Static value
+			continue
+		}
+		compiledPath, err := compilePath(value)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse path;path:<%s>,err:<%s>", value, err)
+		}
+		compiledPaths[name] = compiledPath
+	}
+	return compiledPaths, nil
+}
+
+func NewCollector(endpoint string, scrapers []JsonScraper) *Collector {
+	return &Collector{
+		Endpoint: endpoint,
+		scrapers: scrapers,
+	}
+}
+
+func (col *Collector) fetchJson() ([]byte, error) {
+	resp, err := http.Get(col.Endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch json from endpoint;endpoint:<%s>,err:<%s>", col.Endpoint, err)
+	}
+	defer resp.Body.Close()
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body;err:<%s>", err)
+	}
+
+	return data, nil
+}
+
+func (col *Collector) Collect(reg *harness.MetricRegistry) {
+	json, err := col.fetchJson()
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	for _, scraper := range col.scrapers {
+		if err := scraper.Scrape(json, reg); err != nil {
+			log.Errorf("error while scraping json;err:<%s>", err)
+			continue
+		}
+	}
+}

jsonexporter/config.go

@@ -0,0 +1,47 @@
+package jsonexporter
+
+import (
+	"fmt"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+)
+
+type Config struct {
+	Name   string            `yaml:name`
+	Path   string            `yaml:path`
+	Labels map[string]string `yaml:labels`
+	Type   string            `yaml:type`
+	Help   string            `yaml:help`
+	Values map[string]string `yaml:values`
+}
+
+func (config *Config) labelNames() []string {
+	labelNames := make([]string, 0, len(config.Labels))
+	for name := range config.Labels {
+		labelNames = append(labelNames, name)
+	}
+	return labelNames
+}
+
+func loadConfig(configPath string) ([]*Config, error) {
+	data, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config;path:<%s>,err:<%s>", configPath, err)
+	}
+
+	var configs []*Config
+	if err := yaml.Unmarshal(data, &configs); err != nil {
+		return nil, fmt.Errorf("failed to parse yaml;err:<%s>", err)
+	}
+	// Complete defaults
+	for _, config := range configs {
+		if config.Type == "" {
+			config.Type = DefaultScrapeType
+		}
+		if config.Help == "" {
+			config.Help = config.Name
+		}
+	}
+
+	return configs, nil
+}

jsonexporter/init.go

@@ -0,0 +1,80 @@
+package jsonexporter
+
+import (
+	"fmt"
+	"github.com/codegangsta/cli"
+	"github.com/kawamuray/prometheus-exporter-harness/harness"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type ScrapeType struct {
+	Configure  func(*Config, *harness.MetricRegistry)
+	NewScraper func(*Config) (JsonScraper, error)
+}
+
+var ScrapeTypes = map[string]*ScrapeType{
+	"object": {
+		Configure: func(config *Config, reg *harness.MetricRegistry) {
+			for subName := range config.Values {
+				name := harness.MakeMetricName(config.Name, subName)
+				reg.Register(
+					name,
+					prometheus.NewGaugeVec(prometheus.GaugeOpts{
+						Name: name,
+						Help: config.Help + " - " + subName,
+					}, config.labelNames()),
+				)
+			}
+		},
+		NewScraper: NewObjectScraper,
+	},
+	"value": {
+		Configure: func(config *Config, reg *harness.MetricRegistry) {
+			reg.Register(
+				config.Name,
+				prometheus.NewGaugeVec(prometheus.GaugeOpts{
+					Name: config.Name,
+					Help: config.Help,
+				}, config.labelNames()),
+			)
+		},
+		NewScraper: NewValueScraper,
+	},
+}
+
+var DefaultScrapeType = "value"
+
+func Init(c *cli.Context, reg *harness.MetricRegistry) (harness.Collector, error) {
+	args := c.Args()
+
+	if len(args) < 2 {
+		cli.ShowAppHelp(c)
+		return nil, fmt.Errorf("not enough arguments")
+	}
+
+	var (
+		endpoint   = args[0]
+		configPath = args[1]
+	)
+
+	configs, err := loadConfig(configPath)
+	if err != nil {
+		return nil, err
+	}
+
+	scrapers := make([]JsonScraper, len(configs))
+	for i, config := range configs {
+		tpe := ScrapeTypes[config.Type]
+		if tpe == nil {
+			return nil, fmt.Errorf("unknown scrape type;type:<%s>", config.Type)
+		}
+		tpe.Configure(config, reg)
+		scraper, err := tpe.NewScraper(config)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create scraper;name:<%s>,err:<%s>", config.Name, err)
+		}
+		scrapers[i] = scraper
+	}
+
+	return NewCollector(endpoint, scrapers), nil
+}