From 32a2a3eadacfef667c91c0cb84ec6c69a707d139 Mon Sep 17 00:00:00 2001 From: Kanak Agrawal Date: Fri, 6 Aug 2021 08:28:04 +0000 Subject: [PATCH 1/4] https://jira.percona.com/browse/PMM-7424 Add dbStatsCollector --- README.md | 1 + exporter/dbstats_collector.go | 76 ++++++++++++++++++++ exporter/dbstats_collector_test.go | 112 +++++++++++++++++++++++++++++ exporter/exporter.go | 12 ++++ main.go | 2 + 5 files changed, 203 insertions(+) create mode 100644 exporter/dbstats_collector.go create mode 100644 exporter/dbstats_collector_test.go diff --git a/README.md b/README.md index fcc79fdaf..6ec2f98a5 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ Currently, these metric sources are implemented: |\-\-log.level|Only log messages with the given severity or above. Valid levels: [debug, info, warn, error]|\-\-log.level="error"| |\-\-disable.diagnosticdata|Disable collecting metrics from getDiagnosticData|| |\-\-disable.replicasetstatus|Disable collecting metrics from replSetGetStatus|| +|\-\-disable.dbstats|Disable collecting metrics from dbStats|| |--version|Show version and exit| ### Build the exporter diff --git a/exporter/dbstats_collector.go b/exporter/dbstats_collector.go new file mode 100644 index 000000000..03b846706 --- /dev/null +++ b/exporter/dbstats_collector.go @@ -0,0 +1,76 @@ +// mongodb_exporter +// Copyright (C) 2017 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package exporter + +import ( + "context" + + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +type dbstatsCollector struct { + ctx context.Context + client *mongo.Client + compatibleMode bool + logger *logrus.Logger + topologyInfo labelsGetter +} + +func (d *dbstatsCollector) Describe(ch chan<- *prometheus.Desc) { + prometheus.DescribeByCollect(d, ch) +} + +func (d *dbstatsCollector) Collect(ch chan<- prometheus.Metric) { + + // List all databases names + dbNames, err := d.client.ListDatabaseNames(d.ctx, bson.M{}) + if err != nil { + d.logger.Errorf("Failed to get database names: %s", err) + return + } + d.logger.Debugf("getting stats for databases: %v", dbNames) + for _, db := range dbNames { + var dbStats bson.M + cmd := bson.D{{Key: "dbStats", Value: 1}, {Key: "scale", Value: 1}} + r := d.client.Database(db).RunCommand(d.ctx, cmd) + err := r.Decode(&dbStats) + if err != nil { + d.logger.Errorf("Failed to get $dbstats for database %s: %s", db, err) + continue + } + + d.logger.Debugf("$dbStats metrics for %s", db) + debugResult(d.logger, dbStats) + + // Since all dbstats will have the same fields, we need to use a metric prefix (db) + // to differentiate metrics between different databases. Labels are being set only to make it easier + // to filter + prefix := db + + labels := d.topologyInfo.baseLabels() + labels["database"] = db + + for _, metric := range makeMetrics(prefix, dbStats, d.topologyInfo.baseLabels(), d.compatibleMode) { + ch <- metric + } + } +} + +var _ prometheus.Collector = (*dbstatsCollector)(nil) diff --git a/exporter/dbstats_collector_test.go b/exporter/dbstats_collector_test.go new file mode 100644 index 000000000..9db83ae5e --- /dev/null +++ b/exporter/dbstats_collector_test.go @@ -0,0 +1,112 @@ +// mongodb_exporter +// Copyright (C) 2017 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package exporter + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/bson" + + "github.com/percona/mongodb_exporter/internal/tu" +) + +func TestDBStatsCollector(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + client := tu.DefaultTestClient(ctx, t) + + database := client.Database("testdb") + database.Drop(ctx) //nolint + + defer func() { + err := database.Drop(ctx) + assert.NoError(t, err) + }() + + for i := 0; i < 3; i++ { + coll := fmt.Sprintf("testcol_%02d", i) + for j := 0; j < 10; j++ { + _, err := database.Collection(coll).InsertOne(ctx, bson.M{"f1": j, "f2": "2"}) + assert.NoError(t, err) + } + } + + ti := labelsGetterMock{} + + c := &dbstatsCollector{ + client: client, + logger: logrus.New(), + topologyInfo: ti, + } + + // The last \n at the end of this string is important + expected := strings.NewReader(` +# HELP mongodb_testdb_avgObjSize testdb. +# TYPE mongodb_testdb_avgObjSize untyped +mongodb_testdb_avgObjSize 40 +# HELP mongodb_testdb_collections testdb. +# TYPE mongodb_testdb_collections untyped +mongodb_testdb_collections 3 +# HELP mongodb_testdb_dataSize testdb. +# TYPE mongodb_testdb_dataSize untyped +mongodb_testdb_dataSize 1200 +# HELP mongodb_testdb_indexSize testdb. +# TYPE mongodb_testdb_indexSize untyped +mongodb_testdb_indexSize 12288 +# HELP mongodb_testdb_indexes testdb. +# TYPE mongodb_testdb_indexes untyped +mongodb_testdb_indexes 3 +# HELP mongodb_testdb_objects testdb. +# TYPE mongodb_testdb_objects untyped +mongodb_testdb_objects 30 +# HELP mongodb_testdb_ok testdb. +# TYPE mongodb_testdb_ok untyped +mongodb_testdb_ok 1 +# HELP mongodb_testdb_storageSize testdb. +# TYPE mongodb_testdb_storageSize untyped +mongodb_testdb_storageSize 12288 +# HELP mongodb_testdb_views testdb. +# TYPE mongodb_testdb_views untyped +mongodb_testdb_views 0` + + "\n") + + // Filter metrics for 2 reasons: + // 1. The result is huge + // 2. We need to check against know values. Don't use metrics that return counters like uptime + // or counters like the number of transactions because they won't return a known value to compare + filter := []string{ + "mongodb_testdb_avgObjSize", + "mongodb_testdb_collections", + "mongodb_testdb_dataSize", + "mongodb_testdb_indexSize", + "mongodb_testdb_indexes", + "mongodb_testdb_objects", + "mongodb_testdb_views", + "mongodb_testdb_storageSize", + "mongodb_testdb_ok", + } + err := testutil.CollectAndCompare(c, expected, filter...) + assert.NoError(t, err) +} diff --git a/exporter/exporter.go b/exporter/exporter.go index f176c5f62..447696e02 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -54,6 +54,7 @@ type Opts struct { Logger *logrus.Logger DisableDiagnosticData bool DisableReplicasetStatus bool + DisableDBStats bool } var ( @@ -147,6 +148,17 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol registry.MustRegister(&ddc) } + if !e.opts.DisableDBStats { + cc := dbstatsCollector{ + ctx: ctx, + client: client, + compatibleMode: e.opts.CompatibleMode, + logger: e.opts.Logger, + topologyInfo: topologyInfo, + } + registry.MustRegister(&cc) + } + // replSetGetStatus is not supported through mongos if !e.opts.DisableReplicasetStatus && nodeType != typeMongos { rsgsc := replSetGetStatusCollector{ diff --git a/main.go b/main.go index 5ebaaf289..6a9488175 100644 --- a/main.go +++ b/main.go @@ -47,6 +47,7 @@ type GlobalFlags struct { DisableDiagnosticData bool `name:"disable.diagnosticdata" help:"Disable collecting metrics from getDiagnosticData"` DisableReplicasetStatus bool `name:"disable.replicasetstatus" help:"Disable collecting metrics from replSetGetStatus"` + DisableDBStats bool `name:"disable.dbstats" help:"Disable collecting metrics from dbStats"` DiscoveringMode bool `name:"discovering-mode" help:"Enable autodiscover collections"` CompatibleMode bool `name:"compatible-mode" help:"Enable old mongodb-exporter compatible metrics"` @@ -117,6 +118,7 @@ func buildExporter(opts GlobalFlags) (*exporter.Exporter, error) { DisableDiagnosticData: opts.DisableDiagnosticData, DisableReplicasetStatus: opts.DisableReplicasetStatus, DirectConnect: opts.DirectConnect, + DisableDBStats: opts.DisableDBStats, } e, err := exporter.New(exporterOpts) From 24ae0d111e6784617618ddb4c07ce5853824f5bd Mon Sep 17 00:00:00 2001 From: Kanak Agrawal Date: Wed, 11 Aug 2021 06:29:10 +0000 Subject: [PATCH 2/4] Run make format --- exporter/dbstats_collector.go | 1 - 1 file changed, 1 deletion(-) diff --git a/exporter/dbstats_collector.go b/exporter/dbstats_collector.go index 03b846706..c71697b5b 100644 --- a/exporter/dbstats_collector.go +++ b/exporter/dbstats_collector.go @@ -38,7 +38,6 @@ func (d *dbstatsCollector) Describe(ch chan<- *prometheus.Desc) { } func (d *dbstatsCollector) Collect(ch chan<- prometheus.Metric) { - // List all databases names dbNames, err := d.client.ListDatabaseNames(d.ctx, bson.M{}) if err != nil { From c483d61506dbca6f5382be133bd49ed5a9b503c9 Mon Sep 17 00:00:00 2001 From: Kanak Agrawal Date: Fri, 20 Aug 2021 12:58:20 +0000 Subject: [PATCH 3/4] Add dbstats_ prefix --- exporter/dbstats_collector.go | 2 +- exporter/dbstats_collector_test.go | 72 +++++++++++++++--------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/exporter/dbstats_collector.go b/exporter/dbstats_collector.go index c71697b5b..9e6faa6e9 100644 --- a/exporter/dbstats_collector.go +++ b/exporter/dbstats_collector.go @@ -61,7 +61,7 @@ func (d *dbstatsCollector) Collect(ch chan<- prometheus.Metric) { // Since all dbstats will have the same fields, we need to use a metric prefix (db) // to differentiate metrics between different databases. Labels are being set only to make it easier // to filter - prefix := db + prefix := "dbstats_" + db labels := d.topologyInfo.baseLabels() labels["database"] = db diff --git a/exporter/dbstats_collector_test.go b/exporter/dbstats_collector_test.go index 9db83ae5e..5e6d2fea3 100644 --- a/exporter/dbstats_collector_test.go +++ b/exporter/dbstats_collector_test.go @@ -63,33 +63,33 @@ func TestDBStatsCollector(t *testing.T) { // The last \n at the end of this string is important expected := strings.NewReader(` -# HELP mongodb_testdb_avgObjSize testdb. -# TYPE mongodb_testdb_avgObjSize untyped -mongodb_testdb_avgObjSize 40 -# HELP mongodb_testdb_collections testdb. -# TYPE mongodb_testdb_collections untyped -mongodb_testdb_collections 3 -# HELP mongodb_testdb_dataSize testdb. -# TYPE mongodb_testdb_dataSize untyped -mongodb_testdb_dataSize 1200 -# HELP mongodb_testdb_indexSize testdb. -# TYPE mongodb_testdb_indexSize untyped -mongodb_testdb_indexSize 12288 -# HELP mongodb_testdb_indexes testdb. -# TYPE mongodb_testdb_indexes untyped -mongodb_testdb_indexes 3 -# HELP mongodb_testdb_objects testdb. -# TYPE mongodb_testdb_objects untyped -mongodb_testdb_objects 30 -# HELP mongodb_testdb_ok testdb. -# TYPE mongodb_testdb_ok untyped -mongodb_testdb_ok 1 -# HELP mongodb_testdb_storageSize testdb. -# TYPE mongodb_testdb_storageSize untyped -mongodb_testdb_storageSize 12288 -# HELP mongodb_testdb_views testdb. -# TYPE mongodb_testdb_views untyped -mongodb_testdb_views 0` + +# HELP mongodb_dbstats_testdb_avgObjSize dbstats_testdb. +# TYPE mongodb_dbstats_testdb_avgObjSize untyped +mongodb_dbstats_testdb_avgObjSize 40 +# HELP mongodb_dbstats_testdb_collections dbstats_testdb. +# TYPE mongodb_dbstats_testdb_collections untyped +mongodb_dbstats_testdb_collections 3 +# HELP mongodb_dbstats_testdb_dataSize dbstats_testdb. +# TYPE mongodb_dbstats_testdb_dataSize untyped +mongodb_dbstats_testdb_dataSize 1200 +# HELP mongodb_dbstats_testdb_indexSize dbstats_testdb. +# TYPE mongodb_dbstats_testdb_indexSize untyped +mongodb_dbstats_testdb_indexSize 12288 +# HELP mongodb_dbstats_testdb_indexes dbstats_testdb. +# TYPE mongodb_dbstats_testdb_indexes untyped +mongodb_dbstats_testdb_indexes 3 +# HELP mongodb_dbstats_testdb_objects dbstats_testdb. +# TYPE mongodb_dbstats_testdb_objects untyped +mongodb_dbstats_testdb_objects 30 +# HELP mongodb_dbstats_testdb_ok dbstats_testdb. +# TYPE mongodb_dbstats_testdb_ok untyped +mongodb_dbstats_testdb_ok 1 +# HELP mongodb_dbstats_testdb_storageSize dbstats_testdb. +# TYPE mongodb_dbstats_testdb_storageSize untyped +mongodb_dbstats_testdb_storageSize 12288 +# HELP mongodb_dbstats_testdb_views dbstats_testdb. +# TYPE mongodb_dbstats_testdb_views untyped +mongodb_dbstats_testdb_views 0` + "\n") // Filter metrics for 2 reasons: @@ -97,15 +97,15 @@ mongodb_testdb_views 0` + // 2. We need to check against know values. Don't use metrics that return counters like uptime // or counters like the number of transactions because they won't return a known value to compare filter := []string{ - "mongodb_testdb_avgObjSize", - "mongodb_testdb_collections", - "mongodb_testdb_dataSize", - "mongodb_testdb_indexSize", - "mongodb_testdb_indexes", - "mongodb_testdb_objects", - "mongodb_testdb_views", - "mongodb_testdb_storageSize", - "mongodb_testdb_ok", + "mongodb_dbstats_testdb_avgObjSize", + "mongodb_dbstats_testdb_collections", + "mongodb_dbstats_testdb_dataSize", + "mongodb_dbstats_testdb_indexSize", + "mongodb_dbstats_testdb_indexes", + "mongodb_dbstats_testdb_objects", + "mongodb_dbstats_testdb_views", + "mongodb_dbstats_testdb_storageSize", + "mongodb_dbstats_testdb_ok", } err := testutil.CollectAndCompare(c, expected, filter...) assert.NoError(t, err) From 714abd090280665fa52e10284026fcd66c6a02f7 Mon Sep 17 00:00:00 2001 From: Kanak Agrawal Date: Sat, 21 Aug 2021 10:30:30 +0000 Subject: [PATCH 4/4] Update Changelog --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index c3093e21b..2cee4a7ff 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ Release 0.20.7 released 2021-08-10 Improvement : Upgraded dependencies Fixed error PMM-6877: replsetGetStatus error on mongos +Improvement PMM-7424: Add dbStats metric Release 0.20.6 released 2021-06-26