diff --git a/exporter/collstats_collector.go b/exporter/collstats_collector.go index 1636f896..aae7f22d 100644 --- a/exporter/collstats_collector.go +++ b/exporter/collstats_collector.go @@ -85,6 +85,7 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { } } + reservedNames := GetAllIndexesForCollections(d.ctx, client, collections) for _, dbCollection := range collections { parts := strings.Split(dbCollection, ".") if len(parts) < 2 { //nolint:gomnd @@ -151,7 +152,7 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { labels["shard"] = shard } - for _, metric := range makeMetrics(prefix, metrics, labels, d.compatibleMode) { + for _, metric := range makeMetrics(reservedNames, prefix, metrics, labels, d.compatibleMode) { ch <- metric } } diff --git a/exporter/collstats_collector_test.go b/exporter/collstats_collector_test.go index b098418a..a285a47c 100644 --- a/exporter/collstats_collector_test.go +++ b/exporter/collstats_collector_test.go @@ -25,7 +25,10 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/promslog" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" "github.com/percona/mongodb_exporter/internal/tu" ) @@ -58,11 +61,6 @@ func TestCollStatsCollector(t *testing.T) { // The last \n at the end of this string is important expected := strings.NewReader(` -# HELP mongodb_collstats_latencyStats_commands_latency collstats.latencyStats.commands.latency -# TYPE mongodb_collstats_latencyStats_commands_latency untyped -mongodb_collstats_latencyStats_commands_latency{collection="testcol_00",database="testdb"} 0 -mongodb_collstats_latencyStats_commands_latency{collection="testcol_01",database="testdb"} 0 -mongodb_collstats_latencyStats_commands_latency{collection="testcol_02",database="testdb"} 0 # HELP mongodb_collstats_latencyStats_transactions_ops collstats.latencyStats.transactions.ops # TYPE mongodb_collstats_latencyStats_transactions_ops untyped mongodb_collstats_latencyStats_transactions_ops{collection="testcol_00",database="testdb"} 0 @@ -85,7 +83,6 @@ mongodb_collstats_storageStats_capped{collection="testcol_02",database="testdb"} // 2. We need to check against know values. Don't use metrics that return counters like uptime // or counters like the number of transactions because they won't return a known value to compare filter := []string{ - "mongodb_collstats_latencyStats_commands_latency", "mongodb_collstats_storageStats_capped", "mongodb_collstats_storageStats_indexSizes", "mongodb_collstats_latencyStats_transactions_ops", @@ -93,3 +90,61 @@ mongodb_collstats_storageStats_capped{collection="testcol_02",database="testdb"} err := testutil.CollectAndCompare(c, expected, filter...) assert.NoError(t, err) } + +func TestCollStatsForFakeCountType(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + + client := tu.DefaultTestClient(ctx, t) + + database := client.Database("testdb") + database.Drop(ctx) //nolint + + defer func() { + err := database.Drop(ctx) + require.NoError(t, err) + }() + + collName := "test_collection_account" + coll := database.Collection(collName) + + _, err := coll.InsertOne(ctx, bson.M{"account_id": 1, "count": 10}) + require.NoError(t, err) + _, err = coll.InsertOne(ctx, bson.M{"account_id": 2, "count": 20}) + require.NoError(t, err) + + indexModel := mongo.IndexModel{ + Keys: bson.D{{Key: "account_id", Value: 1}}, + Options: options.Index().SetName("test_index_account"), + } + _, err = coll.Indexes().CreateOne(ctx, indexModel) + require.NoError(t, err) + + indexModel = mongo.IndexModel{ + Keys: bson.D{{Key: "count", Value: 1}}, + Options: options.Index().SetName("test_index_count"), + } + _, err = coll.Indexes().CreateOne(ctx, indexModel) + require.NoError(t, err) + + ti := labelsGetterMock{} + + collection := []string{"testdb.test_collection_account"} + logger := promslog.New(&promslog.Config{}) + c := newCollectionStatsCollector(ctx, client, logger, false, ti, collection, false) + + expected := strings.NewReader(` + # HELP mongodb_collstats_storageStats_indexSizes collstats.storageStats.indexSizes + # TYPE mongodb_collstats_storageStats_indexSizes untyped + mongodb_collstats_storageStats_indexSizes{collection="test_collection_account",database="testdb",index_name="_id_"} 4096 + mongodb_collstats_storageStats_indexSizes{collection="test_collection_account",database="testdb",index_name="test_index_account"} 20480 + mongodb_collstats_storageStats_indexSizes{collection="test_collection_account",database="testdb",index_name="test_index_count"} 20480 + `) + + filter := []string{ + "mongodb_collstats_storageStats_indexSizes", + } + err = testutil.CollectAndCompare(c, expected, filter...) + require.NoError(t, err) +} diff --git a/exporter/dbstats_collector.go b/exporter/dbstats_collector.go index 136ad975..1dd96c9a 100644 --- a/exporter/dbstats_collector.go +++ b/exporter/dbstats_collector.go @@ -100,7 +100,7 @@ func (d *dbstatsCollector) collect(ch chan<- prometheus.Metric) { // to differentiate metrics between different databases. labels["database"] = db - newMetrics := makeMetrics(prefix, dbStats, labels, d.compatibleMode) + newMetrics := makeMetrics(nil, prefix, dbStats, labels, d.compatibleMode) for _, metric := range newMetrics { ch <- metric } diff --git a/exporter/diagnostic_data_collector.go b/exporter/diagnostic_data_collector.go index 147e19ae..5ad0a9ac 100644 --- a/exporter/diagnostic_data_collector.go +++ b/exporter/diagnostic_data_collector.go @@ -126,7 +126,7 @@ func (d *diagnosticDataCollector) collect(ch chan<- prometheus.Metric) { m = b } - metrics = makeMetrics("", m, d.topologyInfo.baseLabels(), d.compatibleMode) + metrics = makeMetrics(nil, "", m, d.topologyInfo.baseLabels(), d.compatibleMode) metrics = append(metrics, locksMetrics(logger, m)...) securityMetric, err := d.getSecurityMetricFromLineOptions(client) diff --git a/exporter/indexstats_collector.go b/exporter/indexstats_collector.go index e2e6f0ab..c18cd373 100644 --- a/exporter/indexstats_collector.go +++ b/exporter/indexstats_collector.go @@ -85,6 +85,7 @@ func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) { } } + reservedNames := GetAllIndexesForCollections(d.ctx, client, collections) for _, dbCollection := range collections { parts := strings.Split(dbCollection, ".") if len(parts) < 2 { //nolint:gomnd @@ -137,7 +138,7 @@ func (d *indexstatsCollector) collect(ch chan<- prometheus.Metric) { labels["key_name"] = indexName metrics := sanitizeMetrics(metric) - for _, metric := range makeMetrics(prefix, metrics, labels, false) { + for _, metric := range makeMetrics(reservedNames, prefix, metrics, labels, false) { ch <- metric } } diff --git a/exporter/metrics.go b/exporter/metrics.go index d4e28dc1..d3095872 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -16,7 +16,9 @@ package exporter import ( + "context" "regexp" + "slices" "strings" "sync" "time" @@ -25,6 +27,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" ) const ( @@ -206,7 +209,7 @@ func nameAndLabel(prefix, name string) (string, string) { // makeRawMetric creates a Prometheus metric based on the parameters we collected by // traversing the MongoDB structures returned by the collector functions. -func makeRawMetric(prefix, name string, value interface{}, labels map[string]string) (*rawMetric, error) { +func makeRawMetric(reservedNames []string, prefix, name string, value interface{}, labels map[string]string) (*rawMetric, error) { f, err := asFloat64(value) if err != nil { return nil, err @@ -220,8 +223,10 @@ func makeRawMetric(prefix, name string, value interface{}, labels map[string]str fqName, label := nameAndLabel(prefix, name) metricType := prometheus.UntypedValue - if strings.HasSuffix(strings.ToLower(name), "count") { - metricType = prometheus.CounterValue + if !slices.Contains(reservedNames, name) { + if strings.HasSuffix(strings.ToLower(name), "count") { + metricType = prometheus.CounterValue + } } rm := &rawMetric{ @@ -301,7 +306,39 @@ func metricHelp(prefix, name string) string { return name } -func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMode bool) []prometheus.Metric { +// GetAllIndexesForCollections returns all index names for the given list of "db.collection" strings. +func GetAllIndexesForCollections(ctx context.Context, client *mongo.Client, collections []string) []string { + var indexNames []string + for _, dbCollection := range collections { + parts := strings.SplitN(dbCollection, ".", 2) //nolint:mnd + if len(parts) != 2 { + continue // skip invalid format + } + dbName := parts[0] + collName := parts[1] + + coll := client.Database(dbName).Collection(collName) + cursor, err := coll.Indexes().List(ctx) + if err != nil { + continue // skip collections where indexes cannot be listed (e.g., views, system collections) + } + + for cursor.Next(ctx) { + var indexDoc struct { + Name string `bson:"name"` + } + if err := cursor.Decode(&indexDoc); err != nil { + continue + } + indexNames = append(indexNames, indexDoc.Name) + } + cursor.Close(ctx) //nolint:errcheck + } + + return indexNames +} + +func makeMetrics(reservedNames []string, prefix string, m bson.M, labels map[string]string, compatibleMode bool) []prometheus.Metric { var res []prometheus.Metric if prefix != "" { @@ -325,48 +362,50 @@ func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMo } else { l = labels } - switch v := val.(type) { - case bson.M: - res = append(res, makeMetrics(nextPrefix, v, l, compatibleMode)...) - case map[string]interface{}: - res = append(res, makeMetrics(nextPrefix, v, l, compatibleMode)...) - case primitive.A: - res = append(res, processSlice(nextPrefix, v, l, compatibleMode)...) - case []interface{}: - continue - default: - rm, err := makeRawMetric(prefix, k, v, l) + res = append(res, handleMetricSwitch(reservedNames, prefix, nextPrefix, k, val, l, compatibleMode)...) + } + return res +} + +func handleMetricSwitch(reservedNames []string, prefix, nextPrefix, k string, val interface{}, l map[string]string, compatibleMode bool) []prometheus.Metric { + var res []prometheus.Metric + switch v := val.(type) { + case bson.M: + res = append(res, makeMetrics(reservedNames, nextPrefix, v, l, compatibleMode)...) + case map[string]interface{}: + res = append(res, makeMetrics(reservedNames, nextPrefix, v, l, compatibleMode)...) + case primitive.A: + res = append(res, processSlice(reservedNames, nextPrefix, v, l, compatibleMode)...) + case []interface{}: + // skip + default: + rm, err := makeRawMetric(reservedNames, prefix, k, v, l) + if err != nil { + invalidMetric := prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err) + res = append(res, invalidMetric) + return res + } + + // makeRawMetric returns a nil metric for some data types like strings + // because we cannot extract data from all types + if rm == nil { + return res + } + metrics := []*rawMetric{rm} + if renamedMetrics := metricRenameAndLabel(rm, specialConversions); renamedMetrics != nil { + metrics = renamedMetrics + } + + for _, m := range metrics { + metric, err := rawToPrometheusMetric(m) if err != nil { invalidMetric := prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err) res = append(res, invalidMetric) continue } - - // makeRawMetric returns a nil metric for some data types like strings - // because we cannot extract data from all types - if rm == nil { - continue - } - - metrics := []*rawMetric{rm} - - if renamedMetrics := metricRenameAndLabel(rm, specialConversions); renamedMetrics != nil { - metrics = renamedMetrics - } - - for _, m := range metrics { - metric, err := rawToPrometheusMetric(m) - if err != nil { - invalidMetric := prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err) - res = append(res, invalidMetric) - continue - } - - res = append(res, metric) - - if compatibleMode { - res = appendCompatibleMetric(res, m) - } + res = append(res, metric) + if compatibleMode { + res = appendCompatibleMetric(res, m) } } } @@ -376,7 +415,7 @@ func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMo // Extract maps from arrays. Only some structures like replicasets have arrays of members // and each member is represented by a map[string]interface{}. -func processSlice(prefix string, v []interface{}, commonLabels map[string]string, compatibleMode bool) []prometheus.Metric { +func processSlice(reservedNames []string, prefix string, v []interface{}, commonLabels map[string]string, compatibleMode bool) []prometheus.Metric { metrics := make([]prometheus.Metric, 0) labels := make(map[string]string) for name, value := range commonLabels { @@ -406,7 +445,7 @@ func processSlice(prefix string, v []interface{}, commonLabels map[string]string labels["member_idx"] = host } - metrics = append(metrics, makeMetrics(prefix, s, labels, compatibleMode)...) + metrics = append(metrics, makeMetrics(reservedNames, prefix, s, labels, compatibleMode)...) } return metrics diff --git a/exporter/metrics_test.go b/exporter/metrics_test.go index 1a20591b..ee35e01d 100644 --- a/exporter/metrics_test.go +++ b/exporter/metrics_test.go @@ -166,7 +166,7 @@ func TestMakeRawMetric(t *testing.T) { } } - m, err := makeRawMetric(prefix, name, tc.value, nil) + m, err := makeRawMetric(nil, prefix, name, tc.value, nil) assert.NoError(t, err) assert.Equal(t, want, m) diff --git a/exporter/profile_status_collector.go b/exporter/profile_status_collector.go index 37325f3f..f332c021 100644 --- a/exporter/profile_status_collector.go +++ b/exporter/profile_status_collector.go @@ -88,7 +88,7 @@ func (d *profileCollector) collect(ch chan<- prometheus.Metric) { logger.Debug("profile response from MongoDB:") debugResult(logger, primitive.M{db: m}) - for _, metric := range makeMetrics("profile_slow_query", m, labels, d.compatibleMode) { + for _, metric := range makeMetrics(nil, "profile_slow_query", m, labels, d.compatibleMode) { ch <- metric } } diff --git a/exporter/replset_config_collector.go b/exporter/replset_config_collector.go index 561e4522..c6d13938 100644 --- a/exporter/replset_config_collector.go +++ b/exporter/replset_config_collector.go @@ -86,7 +86,7 @@ func (d *replSetGetConfigCollector) collect(ch chan<- prometheus.Metric) { logger.Debug("replSetGetConfig result:") debugResult(logger, m) - for _, metric := range makeMetrics("rs_cfg", m, d.topologyInfo.baseLabels(), d.compatibleMode) { + for _, metric := range makeMetrics(nil, "rs_cfg", m, d.topologyInfo.baseLabels(), d.compatibleMode) { ch <- metric } } diff --git a/exporter/replset_status_collector.go b/exporter/replset_status_collector.go index 0387b8d2..34057cd5 100644 --- a/exporter/replset_status_collector.go +++ b/exporter/replset_status_collector.go @@ -81,7 +81,7 @@ func (d *replSetGetStatusCollector) collect(ch chan<- prometheus.Metric) { logger.Debug("replSetGetStatus result:") debugResult(logger, m) - for _, metric := range makeMetrics("", m, d.topologyInfo.baseLabels(), d.compatibleMode) { + for _, metric := range makeMetrics(nil, "", m, d.topologyInfo.baseLabels(), d.compatibleMode) { ch <- metric } } diff --git a/exporter/shards_collector.go b/exporter/shards_collector.go index e204baa2..bb4fd930 100644 --- a/exporter/shards_collector.go +++ b/exporter/shards_collector.go @@ -82,6 +82,18 @@ func (d *shardsCollector) collect(ch chan<- prometheus.Metric) { if err != nil { logger.Error("cannot get database names", "error", err) } + + collections := make([]string, 0, len(databaseNames)) + for _, database := range databaseNames { + colls := d.getCollectionsForDBName(database) + for _, coll := range colls { + if id, ok := coll["_id"].(string); ok { + _, c := splitNamespace(id) + collections = append(collections, c) + } + } + } + reservedNames := GetAllIndexesForCollections(d.ctx, client, collections) for _, database := range databaseNames { collections := d.getCollectionsForDBName(database) for _, row := range collections { @@ -104,7 +116,7 @@ func (d *shardsCollector) collect(ch chan<- prometheus.Metric) { if !success { continue } - for _, metric := range makeMetrics(prefix, primitive.M{"count": chunks}, labels, d.compatible) { + for _, metric := range makeMetrics(reservedNames, prefix, primitive.M{"count": chunks}, labels, d.compatible) { ch <- metric } } diff --git a/exporter/top_collector.go b/exporter/top_collector.go index 6d8f2945..c0c4c35f 100644 --- a/exporter/top_collector.go +++ b/exporter/top_collector.go @@ -139,6 +139,12 @@ func (d *topCollector) collect(ch chan<- prometheus.Metric) { and pass the namespace as a label to the makeMetrics function. */ + collections := make([]string, 0, len(totals)) + for namespace := range totals { + _, coll := splitNamespace(namespace) + collections = append(collections, coll) + } + reservedNames := GetAllIndexesForCollections(d.ctx, client, collections) for namespace, metrics := range totals { labels := d.topologyInfo.baseLabels() db, coll := splitNamespace(namespace) @@ -150,7 +156,7 @@ func (d *topCollector) collect(ch chan<- prometheus.Metric) { continue } - for _, metric := range makeMetrics("top", mm, labels, d.compatibleMode) { + for _, metric := range makeMetrics(reservedNames, "top", mm, labels, d.compatibleMode) { ch <- metric } }