From 403e93afa7e407b60d4ccec8a20d3e1a330abefb Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Mon, 28 Aug 2017 18:01:57 +0300 Subject: [PATCH 1/8] Add basic RocksDB metrics (PMM-1290). --- .travis.yml | 1 + collector/engine_rocksdb.go | 74 +++++++++++++++++++++++++++++++++++++ docker-compose.yml | 1 + mysqld_exporter.go | 11 ++++++ 4 files changed, 87 insertions(+) create mode 100644 collector/engine_rocksdb.go diff --git a/.travis.yml b/.travis.yml index 0a7cd188a..b8cff16ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ env: - MYSQL_IMAGE=mysql/mysql-server:5.6 - MYSQL_IMAGE=mysql/mysql-server:5.7 - MYSQL_IMAGE=mysql/mysql-server:8.0 +- MYSQL_IMAGE=perconalab/percona-server:5.7.18_rocksdb services: - docker diff --git a/collector/engine_rocksdb.go b/collector/engine_rocksdb.go new file mode 100644 index 000000000..ae734073f --- /dev/null +++ b/collector/engine_rocksdb.go @@ -0,0 +1,74 @@ +// Scrape `SHOW ENGINE ROCKSDB STATUS`. + +package collector + +import ( + "bufio" + "database/sql" + "regexp" + "strconv" + "strings" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/log" +) + +const ( + // Subsystem. + rocksDB = "engine_rocksdb" + // Query. + engineRocksDBStatusQuery = `SHOW ENGINE ROCKSDB STATUS` +) + +var ( + rocksDBCounter = regexp.MustCompile(`^([a-z\.]+) COUNT : (\d+)$`) +) + +func parseRocksDBStatistics(data string) ([]prometheus.Metric, error) { + var metrics []prometheus.Metric + scanner := bufio.NewScanner(strings.NewReader(data)) + for scanner.Scan() { + m := rocksDBCounter.FindStringSubmatch(scanner.Text()) + if len(m) > 0 { + name := strings.Replace(m[1], ".", "_", -1) + value, err := strconv.Atoi(m[2]) + if err != nil { + log.Warnf("failed to parse: %s", scanner.Text()) + continue + } + metrics = append(metrics, prometheus.MustNewConstMetric( + newDesc(rocksDB, name, m[1]), + prometheus.CounterValue, + float64(value), + )) + } + } + return metrics, scanner.Err() +} + +// ScrapeEngineRocksDBStatus scrapes from `SHOW ENGINE ROCKSDB STATUS`. +func ScrapeEngineRocksDBStatus(db *sql.DB, ch chan<- prometheus.Metric) error { + rows, err := db.Query(engineRocksDBStatusQuery) + if err != nil { + return err + } + defer rows.Close() + + var typeCol, nameCol, statusCol string + for rows.Next() { + if err := rows.Scan(&typeCol, &nameCol, &statusCol); err != nil { + return err + } + + if typeCol == "STATISTICS" && nameCol == "rocksdb" { + metrics, err := parseRocksDBStatistics(statusCol) + for _, m := range metrics { + ch <- m + } + if err != nil { + return err + } + } + } + return rows.Err() +} diff --git a/docker-compose.yml b/docker-compose.yml index aa10716fc..e476d7821 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,5 +7,6 @@ services: environment: - MYSQL_ALLOW_EMPTY_PASSWORD=yes - MYSQL_ROOT_HOST=% + - INIT_ROCKSDB=1 ports: - 127.0.0.1:3306:3306 diff --git a/mysqld_exporter.go b/mysqld_exporter.go index 99c3882bb..795387145 100644 --- a/mysqld_exporter.go +++ b/mysqld_exporter.go @@ -141,6 +141,9 @@ var ( collectEngineInnodbStatus = flag.Bool("collect.engine_innodb_status", false, "Collect from SHOW ENGINE INNODB STATUS", ) + collectEngineRocksDBStatus = flag.Bool("collect.engine_rocksdb_status", false, + "Collect from SHOW ENGINE ROCKSDB STATUS", + ) collectHeartbeat = flag.Bool( "collect.heartbeat", false, "Collect from heartbeat", @@ -543,6 +546,14 @@ func (e *ExporterMr) scrape(ch chan<- prometheus.Metric) { } ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds(), "collect.engine_innodb_status") } + if *collectEngineRocksDBStatus { + scrapeTime = time.Now() + if err = collector.ScrapeEngineRocksDBStatus(db, ch); err != nil { + log.Errorln("Error scraping for collect.engine_rocksdb_status:", err) + e.scrapeErrors.WithLabelValues("collect.engine_rocksdb_status").Inc() + } + ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds(), "collect.engine_innodb_status") + } } func (e *ExporterLr) scrape(ch chan<- prometheus.Metric) { From 493380d0db45564b6f95cc03991141cab24744a8 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 29 Aug 2017 15:17:26 +0300 Subject: [PATCH 2/8] Add our shared helpers. --- .../percona/exporter_shared/LICENSE | 202 ++++++++++++++++++ .../github.com/percona/exporter_shared/NOTICE | 2 + .../exporter_shared/helpers/helpers.go | 69 ++++++ vendor/vendor.json | 6 + 4 files changed, 279 insertions(+) create mode 100644 vendor/github.com/percona/exporter_shared/LICENSE create mode 100644 vendor/github.com/percona/exporter_shared/NOTICE create mode 100644 vendor/github.com/percona/exporter_shared/helpers/helpers.go diff --git a/vendor/github.com/percona/exporter_shared/LICENSE b/vendor/github.com/percona/exporter_shared/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/github.com/percona/exporter_shared/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/percona/exporter_shared/NOTICE b/vendor/github.com/percona/exporter_shared/NOTICE new file mode 100644 index 000000000..d3b84271b --- /dev/null +++ b/vendor/github.com/percona/exporter_shared/NOTICE @@ -0,0 +1,2 @@ +Shared code for Percona Prometheus exporters. +Copyright 2017 Percona LLC diff --git a/vendor/github.com/percona/exporter_shared/helpers/helpers.go b/vendor/github.com/percona/exporter_shared/helpers/helpers.go new file mode 100644 index 000000000..6bdc1214e --- /dev/null +++ b/vendor/github.com/percona/exporter_shared/helpers/helpers.go @@ -0,0 +1,69 @@ +// Copyright 2017 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package helpers provides test helpers for Prometheus exporters. +// +// It contains workarounds for the following issues: +// * https://github.com/prometheus/client_golang/issues/322 +// * https://github.com/prometheus/client_golang/issues/323 +package helpers + +import ( + "regexp" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +var nameRE = regexp.MustCompile(`fqName: "(\w+)"`) + +func getName(d *prometheus.Desc) string { + m := nameRE.FindStringSubmatch(d.String()) + if len(m) != 2 { + panic("failed to get metric name from " + d.String()) + } + return m[1] +} + +// Metric contains Prometheus metric details. +type Metric struct { + Name string + Labels prometheus.Labels + Type dto.MetricType + Value float64 +} + +// ReadMetric extracts details from Prometheus metric. +func ReadMetric(m prometheus.Metric) *Metric { + pb := &dto.Metric{} + if err := m.Write(pb); err != nil { + panic(err) + } + + name := getName(m.Desc()) + labels := make(prometheus.Labels, len(pb.Label)) + for _, v := range pb.Label { + labels[v.GetName()] = v.GetValue() + } + if pb.Gauge != nil { + return &Metric{name, labels, dto.MetricType_GAUGE, pb.GetGauge().GetValue()} + } + if pb.Counter != nil { + return &Metric{name, labels, dto.MetricType_COUNTER, pb.GetCounter().GetValue()} + } + if pb.Untyped != nil { + return &Metric{name, labels, dto.MetricType_UNTYPED, pb.GetUntyped().GetValue()} + } + panic("Unsupported metric type") +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 5fdfd6128..c6b02e28a 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -48,6 +48,12 @@ "revision": "c12348ce28de40eed0136aa2b644d0ee0650e56c", "revisionTime": "2016-04-24T11:30:07Z" }, + { + "checksumSHA1": "hlMPi8/SmdReaBRR0dqEWsYVMvg=", + "path": "github.com/percona/exporter_shared/helpers", + "revision": "b61ecbac3b73768e613063577adfcabe7628cb9d", + "revisionTime": "2017-08-29T10:33:34Z" + }, { "checksumSHA1": "KkB+77Ziom7N6RzSbyUwYGrmDeU=", "path": "github.com/prometheus/client_golang/prometheus", From 13fb8562019dd46ad02114bd4cd75139acf7ac16 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 29 Aug 2017 15:18:03 +0300 Subject: [PATCH 3/8] Add basic tests for RocksDB. --- collector/engine_rocksdb.go | 32 +++++++++++------- collector/engine_rocksdb_test.go | 58 ++++++++++++++++++++++++++++++++ mysqld_exporter.go | 2 +- 3 files changed, 78 insertions(+), 14 deletions(-) create mode 100644 collector/engine_rocksdb_test.go diff --git a/collector/engine_rocksdb.go b/collector/engine_rocksdb.go index ae734073f..29c4adaf9 100644 --- a/collector/engine_rocksdb.go +++ b/collector/engine_rocksdb.go @@ -28,20 +28,26 @@ func parseRocksDBStatistics(data string) ([]prometheus.Metric, error) { var metrics []prometheus.Metric scanner := bufio.NewScanner(strings.NewReader(data)) for scanner.Scan() { - m := rocksDBCounter.FindStringSubmatch(scanner.Text()) - if len(m) > 0 { - name := strings.Replace(m[1], ".", "_", -1) - value, err := strconv.Atoi(m[2]) - if err != nil { - log.Warnf("failed to parse: %s", scanner.Text()) - continue - } - metrics = append(metrics, prometheus.MustNewConstMetric( - newDesc(rocksDB, name, m[1]), - prometheus.CounterValue, - float64(value), - )) + line := strings.TrimSpace(scanner.Text()) + if len(line) == 0 { + continue + } + m := rocksDBCounter.FindStringSubmatch(line) + if len(m) == 0 { + continue + } + + name := strings.Replace(m[1], ".", "_", -1) + value, err := strconv.Atoi(m[2]) + if err != nil { + log.Warnf("failed to parse: %s", scanner.Text()) + continue } + metrics = append(metrics, prometheus.MustNewConstMetric( + newDesc(rocksDB, name, m[1]), + prometheus.CounterValue, + float64(value), + )) } return metrics, scanner.Err() } diff --git a/collector/engine_rocksdb_test.go b/collector/engine_rocksdb_test.go new file mode 100644 index 000000000..ce0c1567d --- /dev/null +++ b/collector/engine_rocksdb_test.go @@ -0,0 +1,58 @@ +package collector + +import ( + "database/sql" + "testing" + "time" + + _ "github.com/go-sql-driver/mysql" // register driver + "github.com/percona/exporter_shared/helpers" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +// getDB waits until MySQL is up and returns opened connection. +func getDB(t testing.TB) *sql.DB { + var db *sql.DB + var err error + for i := 0; i < 20; i++ { + db, err = sql.Open("mysql", "root@/mysql") + if err == nil { + err = db.Ping() + } + if err == nil { + return db + } + t.Log(err) + time.Sleep(time.Second) + } + t.Fatalf("Failed to get database connection: %s", err) + panic("not reached") +} + +func TestScrapeEngineRocksDBStatus(t *testing.T) { + if testing.Short() { + t.Skip("-short is passed, skipping test") + } + + db := getDB(t) + + convey.Convey("Metrics collection", t, func() { + ch := make(chan prometheus.Metric) + go func() { + err := ScrapeEngineRocksDBStatus(db, ch) + if err != nil { + t.Error(err) + } + close(ch) + }() + for m := range ch { + got := helpers.ReadMetric(m) + if got.Name == "mysql_engine_rocksdb_rocksdb_bytes_read" { + convey.So(got.Type, convey.ShouldEqual, dto.MetricType_COUNTER) + convey.So(got.Value, convey.ShouldBeGreaterThan, 0) + } + } + }) +} diff --git a/mysqld_exporter.go b/mysqld_exporter.go index 795387145..63ec2d806 100644 --- a/mysqld_exporter.go +++ b/mysqld_exporter.go @@ -552,7 +552,7 @@ func (e *ExporterMr) scrape(ch chan<- prometheus.Metric) { log.Errorln("Error scraping for collect.engine_rocksdb_status:", err) e.scrapeErrors.WithLabelValues("collect.engine_rocksdb_status").Inc() } - ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds(), "collect.engine_innodb_status") + ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds(), "collect.engine_rocksdb_status") } } From a2b739fb3473dd8e01ccd60043bf5fc3f0be84a1 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 29 Aug 2017 15:36:52 +0300 Subject: [PATCH 4/8] Add RocksDB CFStats. --- collector/info_schema_rocksdb_cfstats.go | 40 +++++++++++++++++++ collector/info_schema_rocksdb_cfstats_test.go | 37 +++++++++++++++++ mysqld_exporter.go | 11 +++++ 3 files changed, 88 insertions(+) create mode 100644 collector/info_schema_rocksdb_cfstats.go create mode 100644 collector/info_schema_rocksdb_cfstats_test.go diff --git a/collector/info_schema_rocksdb_cfstats.go b/collector/info_schema_rocksdb_cfstats.go new file mode 100644 index 000000000..c8c9cf4c1 --- /dev/null +++ b/collector/info_schema_rocksdb_cfstats.go @@ -0,0 +1,40 @@ +// Scrape `information_schema.rocksdb_cfstats`. + +package collector + +import ( + "database/sql" + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +const infoSchemaRocksDBCFStatsQuery = `SELECT cf_name, stat_type, value FROM information_schema.rocksdb_cfstats` + +// ScrapeRocksDBCFStats collects from `information_schema.rocksdb_cfstats`. +func ScrapeRocksDBCFStats(db *sql.DB, ch chan<- prometheus.Metric) error { + rows, err := db.Query(infoSchemaRocksDBCFStatsQuery) + if err != nil { + return err + } + defer rows.Close() + + var nameCol, typeCol string + var valueCol int64 + for rows.Next() { + if err = rows.Scan(&nameCol, &typeCol, &valueCol); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(namespace, "rocksdb_cfstats", strings.ToLower(typeCol)), + typeCol, []string{"name"}, nil, + ), + prometheus.UntypedValue, + float64(valueCol), + nameCol, + ) + } + return rows.Err() +} diff --git a/collector/info_schema_rocksdb_cfstats_test.go b/collector/info_schema_rocksdb_cfstats_test.go new file mode 100644 index 000000000..b8e177750 --- /dev/null +++ b/collector/info_schema_rocksdb_cfstats_test.go @@ -0,0 +1,37 @@ +package collector + +import ( + "testing" + + "github.com/percona/exporter_shared/helpers" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestScrapeRocksDBCFStats(t *testing.T) { + if testing.Short() { + t.Skip("-short is passed, skipping test") + } + + db := getDB(t) + + convey.Convey("Metrics collection", t, func() { + ch := make(chan prometheus.Metric) + go func() { + err := ScrapeRocksDBCFStats(db, ch) + if err != nil { + t.Error(err) + } + close(ch) + }() + for m := range ch { + got := helpers.ReadMetric(m) + if got.Name == "mysql_rocksdb_cfstats_cur_size_all_mem_tables" { + convey.So(got.Type, convey.ShouldEqual, dto.MetricType_UNTYPED) + convey.So(got.Value, convey.ShouldBeGreaterThan, 0) + convey.So(got.Labels, convey.ShouldContainKey, "name") + } + } + }) +} diff --git a/mysqld_exporter.go b/mysqld_exporter.go index 63ec2d806..0544ddb88 100644 --- a/mysqld_exporter.go +++ b/mysqld_exporter.go @@ -135,6 +135,9 @@ var ( collectQueryResponseTime = flag.Bool("collect.info_schema.query_response_time", false, "Collect query response time distribution if query_response_time_stats is ON.", ) + collectRocksDBCFStats = flag.Bool("collect.info_schema.rocksdb_cfstats", false, + "Collect RocksDB column family statistics", + ) collectEngineTokudbStatus = flag.Bool("collect.engine_tokudb_status", false, "Collect from SHOW ENGINE TOKUDB STATUS", ) @@ -538,6 +541,14 @@ func (e *ExporterMr) scrape(ch chan<- prometheus.Metric) { } ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds(), "collect.info_schema.query_response_time") } + if *collectRocksDBCFStats { + scrapeTime = time.Now() + if err = collector.ScrapeRocksDBCFStats(db, ch); err != nil { + log.Errorln("Error scraping for collect.info_schema.rocksdb_cfstats:", err) + e.scrapeErrors.WithLabelValues("collect.info_schema.rocksdb_cfstats").Inc() + } + ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds(), "collect.info_schema.rocksdb_cfstats") + } if *collectEngineInnodbStatus { scrapeTime = time.Now() if err = collector.ScrapeEngineInnodbStatus(db, ch); err != nil { From 568091540531f47221782550b6dd195f6d15e676 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 29 Aug 2017 15:44:53 +0300 Subject: [PATCH 5/8] Add RocksDB DBStats. --- collector/info_schema_rocksdb_dbstats.go | 36 +++++++++++++++++++ collector/info_schema_rocksdb_dbstats_test.go | 36 +++++++++++++++++++ mysqld_exporter.go | 11 ++++++ 3 files changed, 83 insertions(+) create mode 100644 collector/info_schema_rocksdb_dbstats.go create mode 100644 collector/info_schema_rocksdb_dbstats_test.go diff --git a/collector/info_schema_rocksdb_dbstats.go b/collector/info_schema_rocksdb_dbstats.go new file mode 100644 index 000000000..36f69d6ca --- /dev/null +++ b/collector/info_schema_rocksdb_dbstats.go @@ -0,0 +1,36 @@ +// Scrape `information_schema.rocksdb_dbstats`. + +package collector + +import ( + "database/sql" + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +const infoSchemaRocksDBDBStatsQuery = `SELECT stat_type, value FROM information_schema.rocksdb_dbstats` + +// ScrapeRocksDBDBStats collects from `information_schema.rocksdb_dbstats`. +func ScrapeRocksDBDBStats(db *sql.DB, ch chan<- prometheus.Metric) error { + rows, err := db.Query(infoSchemaRocksDBDBStatsQuery) + if err != nil { + return err + } + defer rows.Close() + + var typeCol string + var valueCol int64 + for rows.Next() { + if err = rows.Scan(&typeCol, &valueCol); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + newDesc("rocksdb_dbstats", strings.ToLower(typeCol), typeCol), + prometheus.UntypedValue, + float64(valueCol), + ) + } + return rows.Err() +} diff --git a/collector/info_schema_rocksdb_dbstats_test.go b/collector/info_schema_rocksdb_dbstats_test.go new file mode 100644 index 000000000..75acca449 --- /dev/null +++ b/collector/info_schema_rocksdb_dbstats_test.go @@ -0,0 +1,36 @@ +package collector + +import ( + "testing" + + "github.com/percona/exporter_shared/helpers" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestScrapeRocksDBDBStats(t *testing.T) { + if testing.Short() { + t.Skip("-short is passed, skipping test") + } + + db := getDB(t) + + convey.Convey("Metrics collection", t, func() { + ch := make(chan prometheus.Metric) + go func() { + err := ScrapeRocksDBDBStats(db, ch) + if err != nil { + t.Error(err) + } + close(ch) + }() + for m := range ch { + got := helpers.ReadMetric(m) + if got.Name == "mysql_rocksdb_dbstats_db_block_cache_usage" { + convey.So(got.Type, convey.ShouldEqual, dto.MetricType_UNTYPED) + convey.So(got.Value, convey.ShouldBeGreaterThan, 0) + } + } + }) +} diff --git a/mysqld_exporter.go b/mysqld_exporter.go index 0544ddb88..a24fe03d1 100644 --- a/mysqld_exporter.go +++ b/mysqld_exporter.go @@ -138,6 +138,9 @@ var ( collectRocksDBCFStats = flag.Bool("collect.info_schema.rocksdb_cfstats", false, "Collect RocksDB column family statistics", ) + collectRocksDBDBStats = flag.Bool("collect.info_schema.rocksdb_dbstats", false, + "Collect RocksDB database statistics", + ) collectEngineTokudbStatus = flag.Bool("collect.engine_tokudb_status", false, "Collect from SHOW ENGINE TOKUDB STATUS", ) @@ -549,6 +552,14 @@ func (e *ExporterMr) scrape(ch chan<- prometheus.Metric) { } ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds(), "collect.info_schema.rocksdb_cfstats") } + if *collectRocksDBDBStats { + scrapeTime = time.Now() + if err = collector.ScrapeRocksDBDBStats(db, ch); err != nil { + log.Errorln("Error scraping for collect.info_schema.rocksdb_dbstats:", err) + e.scrapeErrors.WithLabelValues("collect.info_schema.rocksdb_dbstats").Inc() + } + ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds(), "collect.info_schema.rocksdb_dbstats") + } if *collectEngineInnodbStatus { scrapeTime = time.Now() if err = collector.ScrapeEngineInnodbStatus(db, ch); err != nil { From ce7acdae2d78ea952a8d0276ceb369aa81e5b7ce Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 29 Aug 2017 16:00:23 +0300 Subject: [PATCH 6/8] Skip RocksDB tests if it is disabled. --- collector/engine_rocksdb_test.go | 28 +++++++++++++++++++ collector/info_schema_rocksdb_cfstats_test.go | 3 ++ collector/info_schema_rocksdb_dbstats_test.go | 3 ++ 3 files changed, 34 insertions(+) diff --git a/collector/engine_rocksdb_test.go b/collector/engine_rocksdb_test.go index ce0c1567d..48a01c1f0 100644 --- a/collector/engine_rocksdb_test.go +++ b/collector/engine_rocksdb_test.go @@ -31,12 +31,40 @@ func getDB(t testing.TB) *sql.DB { panic("not reached") } +// rocksDBEnabled returns true if RocksDB is enabled, false otherwise. +func rocksDBEnabled(db *sql.DB, t testing.TB) bool { + rows, err := db.Query("SHOW ENGINES") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + + var engine, support, dummy string + for rows.Next() { + if err = rows.Scan(&engine, &support, &dummy, &dummy, &dummy, &dummy); err != nil { + t.Fatal(err) + } + + if engine != "ROCKSDB" { + continue + } + return support == "YES" + } + if err = rows.Err(); err != nil { + t.Fatal(err) + } + return false +} + func TestScrapeEngineRocksDBStatus(t *testing.T) { if testing.Short() { t.Skip("-short is passed, skipping test") } db := getDB(t) + if !rocksDBEnabled(db, t) { + t.Skip("RocksDB is not enabled, skipping test") + } convey.Convey("Metrics collection", t, func() { ch := make(chan prometheus.Metric) diff --git a/collector/info_schema_rocksdb_cfstats_test.go b/collector/info_schema_rocksdb_cfstats_test.go index b8e177750..7cc2699b7 100644 --- a/collector/info_schema_rocksdb_cfstats_test.go +++ b/collector/info_schema_rocksdb_cfstats_test.go @@ -15,6 +15,9 @@ func TestScrapeRocksDBCFStats(t *testing.T) { } db := getDB(t) + if !rocksDBEnabled(db, t) { + t.Skip("RocksDB is not enabled, skipping test") + } convey.Convey("Metrics collection", t, func() { ch := make(chan prometheus.Metric) diff --git a/collector/info_schema_rocksdb_dbstats_test.go b/collector/info_schema_rocksdb_dbstats_test.go index 75acca449..c85c4e07c 100644 --- a/collector/info_schema_rocksdb_dbstats_test.go +++ b/collector/info_schema_rocksdb_dbstats_test.go @@ -15,6 +15,9 @@ func TestScrapeRocksDBDBStats(t *testing.T) { } db := getDB(t) + if !rocksDBEnabled(db, t) { + t.Skip("RocksDB is not enabled, skipping test") + } convey.Convey("Metrics collection", t, func() { ch := make(chan prometheus.Metric) From caa3ee90b966999190e4bc1003f80ffb022c1b4a Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 29 Aug 2017 16:04:40 +0300 Subject: [PATCH 7/8] Check that we actually found metrics. --- collector/engine_rocksdb_test.go | 4 ++++ collector/info_schema_rocksdb_cfstats_test.go | 4 ++++ collector/info_schema_rocksdb_dbstats_test.go | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/collector/engine_rocksdb_test.go b/collector/engine_rocksdb_test.go index 48a01c1f0..fff8f2211 100644 --- a/collector/engine_rocksdb_test.go +++ b/collector/engine_rocksdb_test.go @@ -75,12 +75,16 @@ func TestScrapeEngineRocksDBStatus(t *testing.T) { } close(ch) }() + + var found int for m := range ch { got := helpers.ReadMetric(m) if got.Name == "mysql_engine_rocksdb_rocksdb_bytes_read" { convey.So(got.Type, convey.ShouldEqual, dto.MetricType_COUNTER) convey.So(got.Value, convey.ShouldBeGreaterThan, 0) + found++ } } + convey.So(found, convey.ShouldEqual, 1) }) } diff --git a/collector/info_schema_rocksdb_cfstats_test.go b/collector/info_schema_rocksdb_cfstats_test.go index 7cc2699b7..6871d3a86 100644 --- a/collector/info_schema_rocksdb_cfstats_test.go +++ b/collector/info_schema_rocksdb_cfstats_test.go @@ -28,13 +28,17 @@ func TestScrapeRocksDBCFStats(t *testing.T) { } close(ch) }() + + var found int for m := range ch { got := helpers.ReadMetric(m) if got.Name == "mysql_rocksdb_cfstats_cur_size_all_mem_tables" { convey.So(got.Type, convey.ShouldEqual, dto.MetricType_UNTYPED) convey.So(got.Value, convey.ShouldBeGreaterThan, 0) convey.So(got.Labels, convey.ShouldContainKey, "name") + found += 1 } } + convey.So(found, convey.ShouldEqual, 2) }) } diff --git a/collector/info_schema_rocksdb_dbstats_test.go b/collector/info_schema_rocksdb_dbstats_test.go index c85c4e07c..f0a5547c2 100644 --- a/collector/info_schema_rocksdb_dbstats_test.go +++ b/collector/info_schema_rocksdb_dbstats_test.go @@ -28,12 +28,16 @@ func TestScrapeRocksDBDBStats(t *testing.T) { } close(ch) }() + + var found int for m := range ch { got := helpers.ReadMetric(m) if got.Name == "mysql_rocksdb_dbstats_db_block_cache_usage" { convey.So(got.Type, convey.ShouldEqual, dto.MetricType_UNTYPED) convey.So(got.Value, convey.ShouldBeGreaterThan, 0) + found++ } } + convey.So(found, convey.ShouldEqual, 1) }) } From 931f2e40caae1c44716eb5a7aa34901f876b2dd3 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 29 Aug 2017 17:31:51 +0300 Subject: [PATCH 8/8] Fix tests for upstream MySQL. --- collector/engine_rocksdb_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/collector/engine_rocksdb_test.go b/collector/engine_rocksdb_test.go index fff8f2211..ee4147934 100644 --- a/collector/engine_rocksdb_test.go +++ b/collector/engine_rocksdb_test.go @@ -39,7 +39,8 @@ func rocksDBEnabled(db *sql.DB, t testing.TB) bool { } defer rows.Close() - var engine, support, dummy string + var engine, support string + var dummy interface{} for rows.Next() { if err = rows.Scan(&engine, &support, &dummy, &dummy, &dummy, &dummy); err != nil { t.Fatal(err)