Merge pull request #9 from stephen-soltesz/pr-multi-values

Support exporting multiple values per query

Dockerfile.testing

@@ -0,0 +1,7 @@
+FROM google/cloud-sdk
+RUN curl -O  https://storage.googleapis.com/golang/go1.8.5.linux-amd64.tar.gz
+RUN tar -C /usr/local -xf /go1.8.5.linux-amd64.tar.gz
+RUN rm -f /go1.8.5.linux-amd64.tar.gz
+ENV GOPATH /go
+ENV PATH $PATH:/usr/local/go/bin
+ENTRYPOINT ["/bin/bash"]

README.md

@@ -77,3 +77,43 @@ Visit http://localhost:9393/metrics and you will find metrics like:
     ndt_test_count{machine="mlab2.foo01.measurement-lab.org"} 200
     ...
 ```
+
+
+# Testing
+
+To run the bigquery exporter locally (e.g. with a new query) you can build a
+test environment based on the google/cloud-sdk with a golang tools installed.
+
+Use the following steps:
+
+1. Build the testing docker image.
+
+```
+$ docker build -t bqe.testing -f Dockerfile.testing .
+```
+
+2. Run the testing image, with fowarded ports and shared volume. The
+   `--volumes-from` option is created automatically by the cloud-sdk base image.
+   This volume preserves credentials across runs of the docker image.
+
+```
+$ docker run -p 9050:9050 --rm -ti -v $PWD:/go/src/github.com/m-lab/prometheus-bigquery-exporter --volumes-from gcloud-config bqe.testing
+```
+
+3. Authenticate using your account. Both steps are necessary, the first to run
+   gcloud commands (which uses user credentials), the second to run the bigquery
+   exporter (which uses application default credentials).
+
+```
+# gcloud auth login
+# gcloud auth application-default login
+```
+
+4. Start the bigquery exporter.
+
+```
+go get -v github.com/m-lab/prometheus-bigquery-exporter/cmd/bigquery_exporter
+./go/bin/bigquery_exporter \
+    --project mlab-sandbox \
+    --type gauge --query <path-to-some-query-file>/bq_ndt_metrics.sql
+```

bq/bq.go

@@ -8,15 +8,23 @@ import (
 )
 
 type Collector struct {
-	runner     QueryRunner
+	// runner must be a QueryRunner instance for collecting metrics.
+	runner QueryRunner
+	// metricName is the base name for prometheus metrics created for this query.
 	metricName string
-	query      string
+	// query contains the standardSQL query.
+	query string
 
+	// valType defines whether the metric is a Gauge or Counter type.
 	valType prometheus.ValueType
-	desc    *prometheus.Desc
+	// descs maps metric suffixes to the prometheus description. These descriptions
+	// are generated once and must be stable over time.
+	descs map[string]*prometheus.Desc
 
+	// metrics caches the last set of collected results from a query.
 	metrics []Metric
-	mux     sync.Mutex
+	// mux locks access to types above.
+	mux sync.Mutex
 }
 
 // NewCollector creates a new BigQuery Collector instance.
@@ -26,7 +34,7 @@ func NewCollector(runner QueryRunner, valType prometheus.ValueType, metricName,
 		metricName: metricName,
 		query:      query,
 		valType:    valType,
-		desc:       nil,
+		descs:      nil,
 		metrics:    nil,
 		mux:        sync.Mutex{},
 	}
@@ -35,13 +43,16 @@ func NewCollector(runner QueryRunner, valType prometheus.ValueType, metricName,
 // Describe satisfies the prometheus.Collector interface. Describe is called
 // immediately after registering the collector.
 func (col *Collector) Describe(ch chan<- *prometheus.Desc) {
-	if col.desc == nil {
+	if col.descs == nil {
 		// TODO: collect metrics for query exec time.
+		col.descs = make(map[string]*prometheus.Desc, 1)
 		col.Update()
 		col.setDesc()
 	}
 	// NOTE: if Update returns no metrics, this will fail.
-	ch <- col.desc
+	for _, desc := range col.descs {
+		ch <- desc
+	}
 }
 
 // Collect satisfies the prometheus.Collector interface. Collect reports values
@@ -53,8 +64,10 @@ func (col *Collector) Collect(ch chan<- prometheus.Metric) {
 	col.mux.Unlock()
 
 	for i := range col.metrics {
-		ch <- prometheus.MustNewConstMetric(
-			col.desc, col.valType, metrics[i].value, metrics[i].values...)
+		for k, desc := range col.descs {
+			ch <- prometheus.MustNewConstMetric(
+				desc, col.valType, metrics[i].values[k], metrics[i].labelValues...)
+		}
 	}
 }
 
@@ -82,8 +95,10 @@ func (col *Collector) Update() error {
 func (col *Collector) setDesc() {
 	// The query may return no results.
 	if len(col.metrics) > 0 {
-		// TODO: allow passing meaningful help text.
-		col.desc = prometheus.NewDesc(col.metricName, "help text", col.metrics[0].labels, nil)
+		for k, _ := range col.metrics[0].values {
+			// TODO: allow passing meaningful help text.
+			col.descs[k] = prometheus.NewDesc(col.metricName+k, "help text", col.metrics[0].labelKeys, nil)
+		}
 	} else {
 		// TODO: this is a problem.
 		return

bq/bq_test.go

@@ -23,14 +23,14 @@ func (qr *fakeQueryRunner) Query(query string) ([]Metric, error) {
 func TestCollector(t *testing.T) {
 	metrics := []Metric{
 		Metric{
-			labels: []string{"key"},
-			values: []string{"thing"},
-			value:  1.1,
+			labelKeys:   []string{"key"},
+			labelValues: []string{"thing"},
+			values:      map[string]float64{"": 1.1},
 		},
 		Metric{
-			labels: []string{"key"},
-			values: []string{"thing2"},
-			value:  2.1,
+			labelKeys:   []string{"key"},
+			labelValues: []string{"thing2"},
+			values:      map[string]float64{"": 2.1},
 		},
 	}
 	expectedMetrics := []string{

bq/live_test.go

@@ -28,28 +28,39 @@ func TestLiveQuery(t *testing.T) {
 		metrics []bq.Metric
 	}{
 		{
-			name:  "Single value",
+			name:  "Single row, single value",
 			query: "SELECT 1 as value",
 			metrics: []bq.Metric{
-				bq.NewMetric(nil, nil, 1.0),
+				bq.NewMetric(nil, nil, map[string]float64{"": 1.0}),
 			},
 		},
 		{
-			name:  "Single value with label",
+			name:  "Single row, single value with label",
 			query: "SELECT 'foo' as key, 2 as value",
 			metrics: []bq.Metric{
-				bq.NewMetric([]string{"key"}, []string{"foo"}, 2.0),
+				bq.NewMetric([]string{"key"}, []string{"foo"}, map[string]float64{"": 2.0}),
 			},
 		},
 		{
-			name: "Multiple values with labels",
+			name: "Multiple rows, single value with labels",
 			query: `#standardSQL
 			        SELECT key, value
 					FROM (SELECT "foo" AS key, 1 AS value UNION ALL
 					      SELECT "bar" AS key, 2 AS value);`,
 			metrics: []bq.Metric{
-				bq.NewMetric([]string{"key"}, []string{"foo"}, 1.0),
-				bq.NewMetric([]string{"key"}, []string{"bar"}, 2.0),
+				bq.NewMetric([]string{"key"}, []string{"foo"}, map[string]float64{"": 1.0}),
+				bq.NewMetric([]string{"key"}, []string{"bar"}, map[string]float64{"": 2.0}),
+			},
+		},
+		{
+			name: "Multiple rows, multiple values with labels",
+			query: `#standardSQL
+			        SELECT key, value_foo, value_bar
+					FROM (SELECT "foo" AS key, 1 AS value_foo, 3 as value_bar UNION ALL
+					      SELECT "bar" AS key, 2 AS value_foo, 4 as value_bar);`,
+			metrics: []bq.Metric{
+				bq.NewMetric([]string{"key"}, []string{"foo"}, map[string]float64{"_foo": 1.0, "_bar": 3.0}),
+				bq.NewMetric([]string{"key"}, []string{"bar"}, map[string]float64{"_foo": 2.0, "_bar": 4.0}),
 			},
 		},
 	}

bq/query_runner.go

@@ -8,6 +8,7 @@ import (
 	"log"
 	"math"
 	"sort"
+	"strings"
 
 	"cloud.google.com/go/bigquery"
 	"google.golang.org/api/iterator"
@@ -20,14 +21,14 @@ type QueryRunner interface {
 
 // Metric holds raw data from query results needed to create a prometheus.Metric.
 type Metric struct {
-	labels []string
-	values []string
-	value  float64
+	labelKeys   []string
+	labelValues []string
+	values      map[string]float64
 }
 
 // NewMetric creates a Metric with given values.
-func NewMetric(labels []string, values []string, value float64) Metric {
-	return Metric{labels, values, value}
+func NewMetric(labelKeys []string, labelValues []string, values map[string]float64) Metric {
+	return Metric{labelKeys, labelValues, values}
 }
 
 // queryRunnerImpl is a concerete implementation of QueryRunner for BigQuery.
@@ -100,20 +101,23 @@ func valToString(v bigquery.Value) string {
 // rowToMetric converts a bigquery result row to a bq.Metric
 func rowToMetric(row map[string]bigquery.Value) Metric {
 	m := Metric{}
+	m.values = make(map[string]float64, 1)
 
 	// Note that `range` does not guarantee map key order. So, we extract label
 	// names, sort them, and then extract values.
 	for k, v := range row {
-		if k == "value" {
-			m.value = valToFloat(v)
+		if strings.HasPrefix(k, "value") {
+			// Get the value suffix used to augment the metric name. If k is
+			// "value", then the default name will just be the empty string.
+			m.values[k[5:]] = valToFloat(v)
 		} else {
-			m.labels = append(m.labels, k)
+			m.labelKeys = append(m.labelKeys, k)
 		}
 	}
-	sort.Strings(m.labels)
+	sort.Strings(m.labelKeys)
 
-	for i := range m.labels {
-		m.values = append(m.values, valToString(row[m.labels[i]]))
+	for i := range m.labelKeys {
+		m.labelValues = append(m.labelValues, valToString(row[m.labelKeys[i]]))
 	}
 	return m
 }

bq/query_runner_test.go

@@ -20,9 +20,9 @@ func TestRowToMetric(t *testing.T) {
 				"value":   1.0,
 			},
 			metric: Metric{
-				labels: []string{"machine"},
-				values: []string{"mlab1.foo01.measurement-lab.org"},
-				value:  1.0,
+				labelKeys:   []string{"machine"},
+				labelValues: []string{"mlab1.foo01.measurement-lab.org"},
+				values:      map[string]float64{"": 1.0},
 			},
 		},
 		{
@@ -31,9 +31,9 @@ func TestRowToMetric(t *testing.T) {
 				"value": 1.1,
 			},
 			metric: Metric{
-				labels: nil,
-				values: nil,
-				value:  1.1,
+				labelKeys:   nil,
+				labelValues: nil,
+				values:      map[string]float64{"": 1.1},
 			},
 		},
 		{
@@ -42,9 +42,21 @@ func TestRowToMetric(t *testing.T) {
 				"value": int64(10),
 			},
 			metric: Metric{
-				labels: nil,
-				values: nil,
-				value:  10,
+				labelKeys:   nil,
+				labelValues: nil,
+				values:      map[string]float64{"": 10},
+			},
+		},
+		{
+			name: "Multiple values",
+			row: map[string]bigquery.Value{
+				"value_foo": int64(10),
+				"value_bar": int64(20),
+			},
+			metric: Metric{
+				labelKeys:   nil,
+				labelValues: nil,
+				values:      map[string]float64{"_foo": 10, "_bar": 20},
 			},
 		},
 		{
@@ -54,9 +66,9 @@ func TestRowToMetric(t *testing.T) {
 				"value": 2.1,
 			},
 			metric: Metric{
-				labels: []string{"name"},
-				values: []string{"invalid string"}, // converted to a string.
-				value:  2.1,
+				labelKeys:   []string{"name"},
+				labelValues: []string{"invalid string"}, // converted to a string.
+				values:      map[string]float64{"": 2.1},
 			},
 		},
 	}