From 645185892935ecf98b9d0a45d50fc45c730f4a0b Mon Sep 17 00:00:00 2001 From: Vilius Okockis <228085+DeathBorn@users.noreply.github.com> Date: Wed, 29 Nov 2017 18:30:09 +0200 Subject: [PATCH] adding database and collection stats collection --- collector/mongod/collections_status.go | 128 +++++++++++++++++++++++++ collector/mongod/database_status.go | 106 ++++++++++++++++++++ collector/mongodb_collector.go | 34 +++++++ collector/mongos/collections_status.go | 128 +++++++++++++++++++++++++ collector/mongos/database_status.go | 127 ++++++++++++++++++++++++ mongodb_exporter.go | 4 + 6 files changed, 527 insertions(+) create mode 100644 collector/mongod/collections_status.go create mode 100644 collector/mongod/database_status.go create mode 100644 collector/mongos/collections_status.go create mode 100644 collector/mongos/database_status.go diff --git a/collector/mongod/collections_status.go b/collector/mongod/collections_status.go new file mode 100644 index 000000000..26827e552 --- /dev/null +++ b/collector/mongod/collections_status.go @@ -0,0 +1,128 @@ +package collector_mongod + +import ( + "github.com/prometheus/common/log" + "github.com/prometheus/client_golang/prometheus" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" +) + +var ( + collectionSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "size", + Help: "The total size in memory of all records in a collection", + }, []string{"db","coll"}) + collectionObjectCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "count", + Help: "The number of objects or documents in this collection", + }, []string{"db","coll"}) + collectionAvgObjSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "avgobjsize", + Help: "The average size of an object in the collection (plus any padding)", + }, []string{"db","coll"}) + collectionStorageSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "storage_size", + Help: "The total amount of storage allocated to this collection for document storage", + }, []string{"db","coll"}) + collectionIndexes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "indexes", + Help: "The number of indexes on the collection", + }, []string{"db","coll"}) + collectionIndexesSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "indexes_size", + Help: "The total size of all indexes", + }, []string{"db","coll"}) +) + +// CollectionStatList contains stats from all collections +type CollectionStatList struct { + Members []CollectionStatus +} + +// CollectionStatus represents stats about a collection in database (mongod and raw from mongos) +type CollectionStatus struct { + Database string + Name string + Size int `bson:"size,omitempty"` + Count int `bson:"count,omitempty"` + AvgObjSize int `bson:"avgObjSize,omitempty"` + StorageSize int `bson:"storageSize,omitempty"` + Indexes int `bson:"indexSizes,omitempty"` + IndexesSize int `bson:"totalIndexSize,omitempty"` +} + + +// Export exports database stats to prometheus +func (collStatList *CollectionStatList) Export(ch chan<- prometheus.Metric) { + for _, member := range collStatList.Members { + ls := prometheus.Labels{ + "db": member.Database, + "coll": member.Name, + } + collectionSize.With(ls).Set(float64(member.Size)) + collectionObjectCount.With(ls).Set(float64(member.Count)) + collectionAvgObjSize.With(ls).Set(float64(member.AvgObjSize)) + collectionStorageSize.With(ls).Set(float64(member.StorageSize)) + collectionIndexes.With(ls).Set(float64(member.Indexes)) + collectionIndexesSize.With(ls).Set(float64(member.IndexesSize)) + } + collectionSize.Collect(ch) + collectionObjectCount.Collect(ch) + collectionAvgObjSize.Collect(ch) + collectionStorageSize.Collect(ch) + collectionIndexes.Collect(ch) + collectionIndexesSize.Collect(ch) +} + +// Describe describes database stats for prometheus +func (collStatList *CollectionStatList) Describe(ch chan<- *prometheus.Desc) { + collectionSize.Describe(ch) + collectionObjectCount.Describe(ch) + collectionAvgObjSize.Describe(ch) + collectionStorageSize.Describe(ch) + collectionIndexes.Describe(ch) + collectionIndexesSize.Describe(ch) +} + +// GetDatabaseStatus returns stats for a given database +func GetCollectionStatList(session *mgo.Session) *CollectionStatList { + collectionStatList := &CollectionStatList{} + database_names, err := session.DatabaseNames() + if err != nil { + log.Error("Failed to get database names") + return nil + } + for _, db := range database_names { + if db == "admin" || db == "test" || db == "local" { continue } + collection_names, err := session.DB(db).CollectionNames() + if err != nil { + log.Error("Failed to get collection names for db=" + db) + return nil + } + for _, collection_name := range collection_names { + collStatus := CollectionStatus{} + err := session.DB(db).Run(bson.D{{"collStats", collection_name}, {"scale", 1}}, &collStatus) + collStatus.Database = db + collStatus.Name = collection_name + if err != nil { + log.Error("Failed to get collection status.") + return nil + } + collectionStatList.Members = append(collectionStatList.Members,collStatus) + } + } + + return collectionStatList +} diff --git a/collector/mongod/database_status.go b/collector/mongod/database_status.go new file mode 100644 index 000000000..53a9ad70d --- /dev/null +++ b/collector/mongod/database_status.go @@ -0,0 +1,106 @@ +package collector_mongod + +import ( + "github.com/prometheus/common/log" + "github.com/prometheus/client_golang/prometheus" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" +) + +var ( + indexSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "index_size_bytes", + Help: "The total size in bytes of all indexes created on this database", + }, []string{"db"}) + dataSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "data_size_bytes", + Help: "The total size in bytes of the uncompressed data held in this database", + }, []string{"db"}) + collectionsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "collections_total", + Help: "Contains a count of the number of collections in that database", + }, []string{"db"}) + indexesTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "indexes_total", + Help: "Contains a count of the total number of indexes across all collections in the database", + }, []string{"db"}) + objectsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "objects_total", + Help: "Contains a count of the number of objects (i.e. documents) in the database across all collections", + }, []string{"db"}) +) + + +// DatabaseStatList contains stats from all databases +type DatabaseStatList struct { + Members []DatabaseStatus +} + +// DatabaseStatus represents stats about a database (mongod and raw from mongos) +type DatabaseStatus struct { + Name string `bson:"db,omitempty"` + IndexSize int `bson:"indexSize,omitempty"` + DataSize int `bson:"dataSize,omitempty"` + Collections int `bson:"collections,omitempty"` + Objects int `bson:"objects,omitempty"` + Indexes int `bson:"indexes,omitempty"` +} + +// Export exports database stats to prometheus +func (dbStatList *DatabaseStatList) Export(ch chan<- prometheus.Metric) { + for _, member := range dbStatList.Members { + ls := prometheus.Labels{ "db": member.Name } + indexSize.With(ls).Set(float64(member.IndexSize)) + dataSize.With(ls).Set(float64(member.DataSize)) + collectionsTotal.With(ls).Set(float64(member.Collections)) + indexesTotal.With(ls).Set(float64(member.Indexes)) + objectsTotal.With(ls).Set(float64(member.Objects)) + } + indexSize.Collect(ch) + dataSize.Collect(ch) + collectionsTotal.Collect(ch) + indexesTotal.Collect(ch) + objectsTotal.Collect(ch) + +} + +// Describe describes database stats for prometheus +func (dbStatList *DatabaseStatList) Describe(ch chan<- *prometheus.Desc) { + indexSize.Describe(ch) + dataSize.Describe(ch) + collectionsTotal.Describe(ch) + indexesTotal.Describe(ch) + objectsTotal.Describe(ch) +} + +// GetDatabaseStatList returns stats for all databases +func GetDatabaseStatList(session *mgo.Session) *DatabaseStatList { + dbStatList := &DatabaseStatList{} + database_names, err := session.DatabaseNames() + if err != nil { + log.Error("Failed to get database names") + return nil + } + for _, db := range database_names { + if db == "admin" || db == "test" || db == "local" { continue } + dbStatus := DatabaseStatus{} + err := session.DB(db).Run(bson.D{{"dbStats", 1}, {"scale", 1}}, &dbStatus) + if err != nil { + log.Error("Failed to get database status.") + return nil + } + dbStatList.Members = append(dbStatList.Members,dbStatus) + } + + return dbStatList +} diff --git a/collector/mongodb_collector.go b/collector/mongodb_collector.go index e7f90ad13..31812e2f8 100644 --- a/collector/mongodb_collector.go +++ b/collector/mongodb_collector.go @@ -38,6 +38,8 @@ type MongodbCollectorOpts struct { TLSPrivateKeyFile string TLSCaFile string TLSHostnameValidation bool + CollectDatabaseMetrics bool + CollectCollectionMetrics bool } func (in MongodbCollectorOpts) toSessionOps() shared.MongoSessionOpts { @@ -189,6 +191,22 @@ func (exporter *MongodbCollector) collectMongos(session *mgo.Session, ch chan<- if shardingStatus != nil { shardingStatus.Export(ch) } + + if exporter.Opts.CollectDatabaseMetrics { + log.Debug("Collecting Database Status From Mongos") + dbStatList := collector_mongos.GetDatabaseStatList(session) + if dbStatList != nil { + dbStatList.Export(ch) + } + } + + if exporter.Opts.CollectCollectionMetrics { + log.Debug("Collecting Collection Status From Mongos") + collStatList := collector_mongos.GetCollectionStatList(session) + if collStatList != nil { + collStatList.Export(ch) + } + } } func (exporter *MongodbCollector) collectMongod(session *mgo.Session, ch chan<- prometheus.Metric) { @@ -197,6 +215,22 @@ func (exporter *MongodbCollector) collectMongod(session *mgo.Session, ch chan<- if serverStatus != nil { serverStatus.Export(ch) } + + if exporter.Opts.CollectDatabaseMetrics { + log.Debug("Collecting Database Status From Mongod") + dbStatList := collector_mongod.GetDatabaseStatList(session) + if dbStatList != nil { + dbStatList.Export(ch) + } + } + + if exporter.Opts.CollectCollectionMetrics { + log.Debug("Collecting Collection Status From Mongod") + collStatList := collector_mongod.GetCollectionStatList(session) + if collStatList != nil { + collStatList.Export(ch) + } + } } func (exporter *MongodbCollector) collectMongodReplSet(session *mgo.Session, ch chan<- prometheus.Metric) { diff --git a/collector/mongos/collections_status.go b/collector/mongos/collections_status.go new file mode 100644 index 000000000..c6ffb2e91 --- /dev/null +++ b/collector/mongos/collections_status.go @@ -0,0 +1,128 @@ +package collector_mongos + +import ( + "github.com/prometheus/common/log" + "github.com/prometheus/client_golang/prometheus" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" +) + +var ( + collectionSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "size", + Help: "The total size in memory of all records in a collection", + }, []string{"db", "coll"}) + collectionObjectCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "count", + Help: "The number of objects or documents in this collection", + }, []string{"db", "coll"}) + collectionAvgObjSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "avgobjsize", + Help: "The average size of an object in the collection (plus any padding)", + }, []string{"db", "coll"}) + collectionStorageSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "storage_size", + Help: "The total amount of storage allocated to this collection for document storage", + }, []string{"db", "coll"}) + collectionIndexes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "indexes", + Help: "The number of indexes on the collection", + }, []string{"db", "coll"}) + collectionIndexesSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "indexes_size", + Help: "The total size of all indexes", + }, []string{"db", "coll"}) +) + + +// CollectionStatList contains stats from all collections +type CollectionStatList struct { + Members []CollectionStatus +} + +// CollectionStatus represents stats about a collection in database (mongod and raw from mongos) +type CollectionStatus struct { + Database string + Name string + Size int `bson:"size,omitempty"` + Count int `bson:"count,omitempty"` + AvgObjSize int `bson:"avgObjSize,omitempty"` + StorageSize int `bson:"storageSize,omitempty"` + Indexes int `bson:"indexSizes,omitempty"` + IndexesSize int `bson:"totalIndexSize,omitempty"` +} + +// Export exports database stats to prometheus +func (collStatList *CollectionStatList) Export(ch chan<- prometheus.Metric) { + for _, member := range collStatList.Members { + ls := prometheus.Labels{ + "db": member.Database, + "coll": member.Name, + } + collectionSize.With(ls).Set(float64(member.Size)) + collectionObjectCount.With(ls).Set(float64(member.Count)) + collectionAvgObjSize.With(ls).Set(float64(member.AvgObjSize)) + collectionStorageSize.With(ls).Set(float64(member.StorageSize)) + collectionIndexes.With(ls).Set(float64(member.Indexes)) + collectionIndexesSize.With(ls).Set(float64(member.IndexesSize)) + } + collectionSize.Collect(ch) + collectionObjectCount.Collect(ch) + collectionAvgObjSize.Collect(ch) + collectionStorageSize.Collect(ch) + collectionIndexes.Collect(ch) + collectionIndexesSize.Collect(ch) +} + +// Describe describes database stats for prometheus +func (collStatList *CollectionStatList) Describe(ch chan<- *prometheus.Desc) { + collectionSize.Describe(ch) + collectionObjectCount.Describe(ch) + collectionAvgObjSize.Describe(ch) + collectionStorageSize.Describe(ch) + collectionIndexes.Describe(ch) + collectionIndexesSize.Describe(ch) +} + +// GetDatabaseStatus returns stats for a given database +func GetCollectionStatList(session *mgo.Session) *CollectionStatList { + collectionStatList := &CollectionStatList{} + database_names, err := session.DatabaseNames() + if err != nil { + log.Error("Failed to get database names") + return nil + } + for _, db := range database_names { + if db == "admin" || db == "test" || db == "local" { continue } + collection_names, err := session.DB(db).CollectionNames() + if err != nil { + log.Error("Failed to get collection names for db=" + db) + return nil + } + for _, collection_name := range collection_names { + collStatus := CollectionStatus{} + err := session.DB(db).Run(bson.D{{"collStats", collection_name}, {"scale", 1}}, &collStatus) + collStatus.Database = db + collStatus.Name = collection_name + if err != nil { + log.Error("Failed to get collection status.") + return nil + } + collectionStatList.Members = append(collectionStatList.Members,collStatus) + } + } + + return collectionStatList +} \ No newline at end of file diff --git a/collector/mongos/database_status.go b/collector/mongos/database_status.go new file mode 100644 index 000000000..dd2e35089 --- /dev/null +++ b/collector/mongos/database_status.go @@ -0,0 +1,127 @@ +package collector_mongos + +import ( + "strings" + + "github.com/prometheus/common/log" + "github.com/prometheus/client_golang/prometheus" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" +) + +var ( + indexSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "index_size_bytes", + Help: "The total size in bytes of all indexes created on this database", + }, []string{"db", "shard"}) + dataSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "data_size_bytes", + Help: "The total size in bytes of the uncompressed data held in this database", + }, []string{"db", "shard"}) + collectionsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "collections_total", + Help: "Contains a count of the number of collections in that database", + }, []string{"db", "shard"}) + indexesTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "indexes_total", + Help: "Contains a count of the total number of indexes across all collections in the database", + }, []string{"db", "shard"}) + objectsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "objects_total", + Help: "Contains a count of the number of objects (i.e. documents) in the database across all collections", + }, []string{"db", "shard"}) +) + + +// DatabaseStatList contains stats from all databases +type DatabaseStatList struct { + Members []DatabaseStatus +} + +// DatabaseStatus represents stats about a database (mongod and raw from mongos) +type DatabaseStatus struct { + RawStatus // embed to collect top-level attributes + Shards map[string]*RawStatus `bson:"raw,omitempty"` +} + +// RawStatus represents stats about a database from Mongos side +type RawStatus struct { + Name string `bson:"db,omitempty"` + IndexSize int `bson:"indexSize,omitempty"` + DataSize int `bson:"dataSize,omitempty"` + Collections int `bson:"collections,omitempty"` + Objects int `bson:"objects,omitempty"` + Indexes int `bson:"indexes,omitempty"` +} + +// Export exports database stats to prometheus +func (dbStatList *DatabaseStatList) Export(ch chan<- prometheus.Metric) { + for _, member := range dbStatList.Members { + if len(member.Shards) > 0 { + for shard, stats := range member.Shards { + ls := prometheus.Labels{ + "db": stats.Name, + "shard": strings.Split(shard, "/")[0], + } + indexSize.With(ls).Set(float64(stats.IndexSize)) + dataSize.With(ls).Set(float64(stats.DataSize)) + collectionsTotal.With(ls).Set(float64(stats.Collections)) + indexesTotal.With(ls).Set(float64(stats.Indexes)) + objectsTotal.With(ls).Set(float64(stats.Objects)) + } + } + } + + indexSize.Collect(ch) + dataSize.Collect(ch) + collectionsTotal.Collect(ch) + indexesTotal.Collect(ch) + objectsTotal.Collect(ch) + + indexSize.Reset() + dataSize.Reset() + collectionsTotal.Reset() + indexesTotal.Reset() + objectsTotal.Reset() +} + +// Describe describes database stats for prometheus +func (dbStatList *DatabaseStatList) Describe(ch chan<- *prometheus.Desc) { + indexSize.Describe(ch) + dataSize.Describe(ch) + collectionsTotal.Describe(ch) + indexesTotal.Describe(ch) + objectsTotal.Describe(ch) +} + +// GetDatabaseStatList returns stats for all databases +func GetDatabaseStatList(session *mgo.Session) *DatabaseStatList { + dbStatList := &DatabaseStatList{} + database_names, err := session.DatabaseNames() + if err != nil { + log.Error("Failed to get database names") + return nil + } + for _, db := range database_names { + if db == "admin" || db == "test" || db == "local" { continue } + dbStatus := DatabaseStatus{} + err := session.DB(db).Run(bson.D{{"dbStats", 1}, {"scale", 1}}, &dbStatus) + if err != nil { + log.Error("Failed to get database status.") + return nil + } + dbStatList.Members = append(dbStatList.Members,dbStatus) + } + + return dbStatList +} diff --git a/mongodb_exporter.go b/mongodb_exporter.go index 018f2fdf6..20dc79cc2 100644 --- a/mongodb_exporter.go +++ b/mongodb_exporter.go @@ -52,6 +52,8 @@ var ( sslCertFileF = flag.String("web.ssl-cert-file", "", "Path to SSL certificate file.") sslKeyFileF = flag.String("web.ssl-key-file", "", "Path to SSL key file.") + DbF = flag.Bool("collect.database", false, "Enable collection of Database metrics") + CollF = flag.Bool("collect.collection", false, "Enable collection of Collection metrics") uriF = flag.String("mongodb.uri", mongodbDefaultURI(), "MongoDB URI, format: [mongodb://][user:pass@]host1[:port1][,host2[:port2],...][/database][?options]") tlsF = flag.Bool("mongodb.tls", false, "Enable tls connection with mongo server") tlsCertF = flag.String("mongodb.tls-cert", "", "Path to PEM file that contains the certificate (and optionally also the private key in PEM format).\n"+ @@ -218,6 +220,8 @@ func registerCollector() { TLSPrivateKeyFile: *tlsPrivateKeyF, TLSCaFile: *tlsCAF, TLSHostnameValidation: !(*tlsDisableHostnameValidationF), + CollectDatabaseMetrics: *DbF, + CollectCollectionMetrics: *CollF, }) prometheus.MustRegister(mongodbCollector) }