Basic MyRocks support (PMM-1290)

.travis.yml

@@ -11,6 +11,7 @@ env:
 - MYSQL_IMAGE=mysql/mysql-server:5.6
 - MYSQL_IMAGE=mysql/mysql-server:5.7
 - MYSQL_IMAGE=mysql/mysql-server:8.0
+- MYSQL_IMAGE=perconalab/percona-server:5.7.18_rocksdb
 
 services:
 - docker

collector/engine_rocksdb.go

@@ -0,0 +1,80 @@
+// Scrape `SHOW ENGINE ROCKSDB STATUS`.
+
+package collector
+
+import (
+	"bufio"
+	"database/sql"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
+)
+
+const (
+	// Subsystem.
+	rocksDB = "engine_rocksdb"
+	// Query.
+	engineRocksDBStatusQuery = `SHOW ENGINE ROCKSDB STATUS`
+)
+
+var (
+	rocksDBCounter = regexp.MustCompile(`^([a-z\.]+) COUNT : (\d+)$`)
+)
+
+func parseRocksDBStatistics(data string) ([]prometheus.Metric, error) {
+	var metrics []prometheus.Metric
+	scanner := bufio.NewScanner(strings.NewReader(data))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if len(line) == 0 {
+			continue
+		}
+		m := rocksDBCounter.FindStringSubmatch(line)
+		if len(m) == 0 {
+			continue
+		}
+
+		name := strings.Replace(m[1], ".", "_", -1)
+		value, err := strconv.Atoi(m[2])
+		if err != nil {
+			log.Warnf("failed to parse: %s", scanner.Text())
+			continue
+		}
+		metrics = append(metrics, prometheus.MustNewConstMetric(
+			newDesc(rocksDB, name, m[1]),
+			prometheus.CounterValue,
+			float64(value),
+		))
+	}
+	return metrics, scanner.Err()
+}
+
+// ScrapeEngineRocksDBStatus scrapes from `SHOW ENGINE ROCKSDB STATUS`.
+func ScrapeEngineRocksDBStatus(db *sql.DB, ch chan<- prometheus.Metric) error {
+	rows, err := db.Query(engineRocksDBStatusQuery)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	var typeCol, nameCol, statusCol string
+	for rows.Next() {
+		if err := rows.Scan(&typeCol, &nameCol, &statusCol); err != nil {
+			return err
+		}
+
+		if typeCol == "STATISTICS" && nameCol == "rocksdb" {
+			metrics, err := parseRocksDBStatistics(statusCol)
+			for _, m := range metrics {
+				ch <- m
+			}
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return rows.Err()
+}

collector/engine_rocksdb_test.go

@@ -0,0 +1,91 @@
+package collector
+
+import (
+	"database/sql"
+	"testing"
+	"time"
+
+	_ "github.com/go-sql-driver/mysql" // register driver
+	"github.com/percona/exporter_shared/helpers"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/smartystreets/goconvey/convey"
+)
+
+// getDB waits until MySQL is up and returns opened connection.
+func getDB(t testing.TB) *sql.DB {
+	var db *sql.DB
+	var err error
+	for i := 0; i < 20; i++ {
+		db, err = sql.Open("mysql", "root@/mysql")
+		if err == nil {
+			err = db.Ping()
+		}
+		if err == nil {
+			return db
+		}
+		t.Log(err)
+		time.Sleep(time.Second)
+	}
+	t.Fatalf("Failed to get database connection: %s", err)
+	panic("not reached")
+}
+
+// rocksDBEnabled returns true if RocksDB is enabled, false otherwise.
+func rocksDBEnabled(db *sql.DB, t testing.TB) bool {
+	rows, err := db.Query("SHOW ENGINES")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rows.Close()
+
+	var engine, support string
+	var dummy interface{}
+	for rows.Next() {
+		if err = rows.Scan(&engine, &support, &dummy, &dummy, &dummy, &dummy); err != nil {
+			t.Fatal(err)
+		}
+
+		if engine != "ROCKSDB" {
+			continue
+		}
+		return support == "YES"
+	}
+	if err = rows.Err(); err != nil {
+		t.Fatal(err)
+	}
+	return false
+}
+
+func TestScrapeEngineRocksDBStatus(t *testing.T) {
+	if testing.Short() {
+		t.Skip("-short is passed, skipping test")
+	}
+
+	db := getDB(t)
+	if !rocksDBEnabled(db, t) {
+		t.Skip("RocksDB is not enabled, skipping test")
+	}
+
+	convey.Convey("Metrics collection", t, func() {
+		ch := make(chan prometheus.Metric)
+		go func() {
+			err := ScrapeEngineRocksDBStatus(db, ch)
+			if err != nil {
+				t.Error(err)
+			}
+			close(ch)
+		}()
+
+		var found int
+		for m := range ch {
+			got := helpers.ReadMetric(m)
+			if got.Name == "mysql_engine_rocksdb_rocksdb_bytes_read" {
+				convey.So(got.Type, convey.ShouldEqual, dto.MetricType_COUNTER)
+				convey.So(got.Value, convey.ShouldBeGreaterThan, 0)
+				found++
+			}
+		}
+		convey.So(found, convey.ShouldEqual, 1)
+	})
+}

collector/info_schema_rocksdb_cfstats.go

@@ -0,0 +1,40 @@
+// Scrape `information_schema.rocksdb_cfstats`.
+
+package collector
+
+import (
+	"database/sql"
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const infoSchemaRocksDBCFStatsQuery = `SELECT cf_name, stat_type, value FROM information_schema.rocksdb_cfstats`
+
+// ScrapeRocksDBCFStats collects from `information_schema.rocksdb_cfstats`.
+func ScrapeRocksDBCFStats(db *sql.DB, ch chan<- prometheus.Metric) error {
+	rows, err := db.Query(infoSchemaRocksDBCFStatsQuery)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	var nameCol, typeCol string
+	var valueCol int64
+	for rows.Next() {
+		if err = rows.Scan(&nameCol, &typeCol, &valueCol); err != nil {
+			return err
+		}
+
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName(namespace, "rocksdb_cfstats", strings.ToLower(typeCol)),
+				typeCol, []string{"name"}, nil,
+			),
+			prometheus.UntypedValue,
+			float64(valueCol),
+			nameCol,
+		)
+	}
+	return rows.Err()
+}

collector/info_schema_rocksdb_cfstats_test.go

@@ -0,0 +1,44 @@
+package collector
+
+import (
+	"testing"
+
+	"github.com/percona/exporter_shared/helpers"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestScrapeRocksDBCFStats(t *testing.T) {
+	if testing.Short() {
+		t.Skip("-short is passed, skipping test")
+	}
+
+	db := getDB(t)
+	if !rocksDBEnabled(db, t) {
+		t.Skip("RocksDB is not enabled, skipping test")
+	}
+
+	convey.Convey("Metrics collection", t, func() {
+		ch := make(chan prometheus.Metric)
+		go func() {
+			err := ScrapeRocksDBCFStats(db, ch)
+			if err != nil {
+				t.Error(err)
+			}
+			close(ch)
+		}()
+
+		var found int
+		for m := range ch {
+			got := helpers.ReadMetric(m)
+			if got.Name == "mysql_rocksdb_cfstats_cur_size_all_mem_tables" {
+				convey.So(got.Type, convey.ShouldEqual, dto.MetricType_UNTYPED)
+				convey.So(got.Value, convey.ShouldBeGreaterThan, 0)
+				convey.So(got.Labels, convey.ShouldContainKey, "name")
+				found += 1
+			}
+		}
+		convey.So(found, convey.ShouldEqual, 2)
+	})
+}

collector/info_schema_rocksdb_dbstats.go

@@ -0,0 +1,36 @@
+// Scrape `information_schema.rocksdb_dbstats`.
+
+package collector
+
+import (
+	"database/sql"
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const infoSchemaRocksDBDBStatsQuery = `SELECT stat_type, value FROM information_schema.rocksdb_dbstats`
+
+// ScrapeRocksDBDBStats collects from `information_schema.rocksdb_dbstats`.
+func ScrapeRocksDBDBStats(db *sql.DB, ch chan<- prometheus.Metric) error {
+	rows, err := db.Query(infoSchemaRocksDBDBStatsQuery)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	var typeCol string
+	var valueCol int64
+	for rows.Next() {
+		if err = rows.Scan(&typeCol, &valueCol); err != nil {
+			return err
+		}
+
+		ch <- prometheus.MustNewConstMetric(
+			newDesc("rocksdb_dbstats", strings.ToLower(typeCol), typeCol),
+			prometheus.UntypedValue,
+			float64(valueCol),
+		)
+	}
+	return rows.Err()
+}

collector/info_schema_rocksdb_dbstats_test.go

@@ -0,0 +1,43 @@
+package collector
+
+import (
+	"testing"
+
+	"github.com/percona/exporter_shared/helpers"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestScrapeRocksDBDBStats(t *testing.T) {
+	if testing.Short() {
+		t.Skip("-short is passed, skipping test")
+	}
+
+	db := getDB(t)
+	if !rocksDBEnabled(db, t) {
+		t.Skip("RocksDB is not enabled, skipping test")
+	}
+
+	convey.Convey("Metrics collection", t, func() {
+		ch := make(chan prometheus.Metric)
+		go func() {
+			err := ScrapeRocksDBDBStats(db, ch)
+			if err != nil {
+				t.Error(err)
+			}
+			close(ch)
+		}()
+
+		var found int
+		for m := range ch {
+			got := helpers.ReadMetric(m)
+			if got.Name == "mysql_rocksdb_dbstats_db_block_cache_usage" {
+				convey.So(got.Type, convey.ShouldEqual, dto.MetricType_UNTYPED)
+				convey.So(got.Value, convey.ShouldBeGreaterThan, 0)
+				found++
+			}
+		}
+		convey.So(found, convey.ShouldEqual, 1)
+	})
+}

docker-compose.yml

@@ -7,5 +7,6 @@ services:
     environment:
       - MYSQL_ALLOW_EMPTY_PASSWORD=yes
       - MYSQL_ROOT_HOST=%
+      - INIT_ROCKSDB=1
     ports:
       - 127.0.0.1:3306:3306