adding database and collection stats collection

collector/mongod/collections_status.go

@@ -0,0 +1,128 @@
+package collector_mongod
+
+import (
+	"github.com/prometheus/common/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"gopkg.in/mgo.v2"
+	"gopkg.in/mgo.v2/bson"
+)
+
+var (
+	collectionSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "size",
+		Help:      "The total size in memory of all records in a collection",
+	}, []string{"db","coll"})
+	collectionObjectCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "count",
+		Help:      "The number of objects or documents in this collection",
+	}, []string{"db","coll"})
+	collectionAvgObjSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "avgobjsize",
+		Help:      "The average size of an object in the collection (plus any padding)",
+	}, []string{"db","coll"})
+	collectionStorageSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "storage_size",
+		Help:      "The total amount of storage allocated to this collection for document storage",
+	}, []string{"db","coll"})
+	collectionIndexes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "indexes",
+		Help:      "The number of indexes on the collection",
+	}, []string{"db","coll"})
+	collectionIndexesSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "indexes_size",
+		Help:      "The total size of all indexes",
+	}, []string{"db","coll"})
+)
+
+// CollectionStatList contains stats from all collections
+type CollectionStatList struct {
+	Members []CollectionStatus
+}
+
+// CollectionStatus represents stats about a collection in database (mongod and raw from mongos)
+type CollectionStatus struct {
+	Database    string 
+	Name        string
+	Size        int    `bson:"size,omitempty"`
+	Count       int    `bson:"count,omitempty"`
+	AvgObjSize  int    `bson:"avgObjSize,omitempty"`
+	StorageSize int    `bson:"storageSize,omitempty"`
+	Indexes     int    `bson:"indexSizes,omitempty"`
+	IndexesSize int    `bson:"totalIndexSize,omitempty"`
+}
+
+
+// Export exports database stats to prometheus
+func (collStatList *CollectionStatList) Export(ch chan<- prometheus.Metric) {
+	for _, member := range collStatList.Members {
+		ls := prometheus.Labels{ 
+			"db": member.Database,
+			"coll": member.Name,
+		}
+		collectionSize.With(ls).Set(float64(member.Size))
+		collectionObjectCount.With(ls).Set(float64(member.Count))
+		collectionAvgObjSize.With(ls).Set(float64(member.AvgObjSize))
+		collectionStorageSize.With(ls).Set(float64(member.StorageSize))
+		collectionIndexes.With(ls).Set(float64(member.Indexes))
+		collectionIndexesSize.With(ls).Set(float64(member.IndexesSize))
+	}
+	collectionSize.Collect(ch)
+	collectionObjectCount.Collect(ch)
+	collectionAvgObjSize.Collect(ch)
+	collectionStorageSize.Collect(ch)
+	collectionIndexes.Collect(ch)
+	collectionIndexesSize.Collect(ch)
+}
+
+// Describe describes database stats for prometheus
+func (collStatList *CollectionStatList) Describe(ch chan<- *prometheus.Desc) {
+	collectionSize.Describe(ch)
+	collectionObjectCount.Describe(ch)
+	collectionAvgObjSize.Describe(ch)
+	collectionStorageSize.Describe(ch)
+	collectionIndexes.Describe(ch)
+	collectionIndexesSize.Describe(ch)
+}
+
+// GetDatabaseStatus returns stats for a given database
+func GetCollectionStatList(session *mgo.Session) *CollectionStatList {
+	collectionStatList := &CollectionStatList{}
+	database_names, err := session.DatabaseNames()
+	if err != nil {
+		log.Error("Failed to get database names")
+		return nil
+	}
+	for _, db := range database_names {
+		if db == "admin" || db == "test" || db == "local" { continue }
+		collection_names, err := session.DB(db).CollectionNames()
+		if err != nil {
+			log.Error("Failed to get collection names for db=" + db)
+			return nil
+		}
+		for _, collection_name := range collection_names {
+			collStatus := CollectionStatus{}
+			err := session.DB(db).Run(bson.D{{"collStats", collection_name}, {"scale", 1}}, &collStatus)
+			collStatus.Database = db
+			collStatus.Name = collection_name
+			if err != nil {
+				log.Error("Failed to get collection status.")
+				return nil
+			}
+			collectionStatList.Members = append(collectionStatList.Members,collStatus)
+		}
+	}
+
+	return collectionStatList
+}

collector/mongod/database_status.go

@@ -0,0 +1,106 @@
+package collector_mongod
+
+import (
+	"github.com/prometheus/common/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"gopkg.in/mgo.v2"
+	"gopkg.in/mgo.v2/bson"
+)
+
+var (
+	indexSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db",
+		Name:      "index_size_bytes",
+		Help:      "The total size in bytes of all indexes created on this database",
+	}, []string{"db"})
+	dataSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db",
+		Name:      "data_size_bytes",
+		Help:      "The total size in bytes of the uncompressed data held in this database",
+	}, []string{"db"})
+	collectionsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db",
+		Name:      "collections_total",
+		Help:      "Contains a count of the number of collections in that database",
+	}, []string{"db"})
+	indexesTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db",
+		Name:      "indexes_total",
+		Help:      "Contains a count of the total number of indexes across all collections in the database",
+	}, []string{"db"})
+	objectsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db",
+		Name:      "objects_total",
+		Help:      "Contains a count of the number of objects (i.e. documents) in the database across all collections",
+	}, []string{"db"})
+)
+
+
+// DatabaseStatList contains stats from all databases
+type DatabaseStatList struct {
+	Members []DatabaseStatus
+}
+
+// DatabaseStatus represents stats about a database (mongod and raw from mongos)
+type DatabaseStatus struct {
+	Name        string `bson:"db,omitempty"`
+	IndexSize   int    `bson:"indexSize,omitempty"`
+	DataSize    int    `bson:"dataSize,omitempty"`
+	Collections int    `bson:"collections,omitempty"`
+	Objects     int    `bson:"objects,omitempty"`
+	Indexes     int    `bson:"indexes,omitempty"`
+}
+
+// Export exports database stats to prometheus
+func (dbStatList *DatabaseStatList) Export(ch chan<- prometheus.Metric) {
+	for _, member := range dbStatList.Members {
+		ls := prometheus.Labels{ "db": member.Name }
+		indexSize.With(ls).Set(float64(member.IndexSize))
+		dataSize.With(ls).Set(float64(member.DataSize))
+		collectionsTotal.With(ls).Set(float64(member.Collections))
+		indexesTotal.With(ls).Set(float64(member.Indexes))
+		objectsTotal.With(ls).Set(float64(member.Objects))
+	}
+	indexSize.Collect(ch)
+	dataSize.Collect(ch)
+	collectionsTotal.Collect(ch)
+	indexesTotal.Collect(ch)
+	objectsTotal.Collect(ch)
+
+}
+
+// Describe describes database stats for prometheus
+func (dbStatList *DatabaseStatList) Describe(ch chan<- *prometheus.Desc) {
+	indexSize.Describe(ch)
+	dataSize.Describe(ch)
+	collectionsTotal.Describe(ch)
+	indexesTotal.Describe(ch)
+	objectsTotal.Describe(ch)
+}
+
+// GetDatabaseStatList returns stats for all databases
+func GetDatabaseStatList(session *mgo.Session) *DatabaseStatList {
+	dbStatList := &DatabaseStatList{}
+	database_names, err := session.DatabaseNames()
+	if err != nil {
+		log.Error("Failed to get database names")
+		return nil
+	}
+	for _, db := range database_names {
+		if db == "admin" || db == "test" || db == "local" { continue }
+		dbStatus := DatabaseStatus{}
+		err := session.DB(db).Run(bson.D{{"dbStats", 1}, {"scale", 1}}, &dbStatus)
+		if err != nil {
+			log.Error("Failed to get database status.")
+			return nil
+		}
+		dbStatList.Members = append(dbStatList.Members,dbStatus)
+	}
+
+	return dbStatList
+}

collector/mongodb_collector.go

@@ -38,6 +38,8 @@ type MongodbCollectorOpts struct {
 	TLSPrivateKeyFile     string
 	TLSCaFile             string
 	TLSHostnameValidation bool
+	CollectDatabaseMetrics   bool
+	CollectCollectionMetrics bool
 	DBPoolLimit           int
 }
 
@@ -221,6 +223,22 @@ func (exporter *MongodbCollector) collectMongos(session *mgo.Session, ch chan<-
 	if shardingStatus != nil {
 		shardingStatus.Export(ch)
 	}
+
+	if exporter.Opts.CollectDatabaseMetrics {
+		log.Debug("Collecting Database Status From Mongos")
+		dbStatList := collector_mongos.GetDatabaseStatList(session)
+		if dbStatList != nil {
+			dbStatList.Export(ch)
+		}
+	}
+
+	if exporter.Opts.CollectCollectionMetrics {
+		log.Debug("Collecting Collection Status From Mongos")
+		collStatList := collector_mongos.GetCollectionStatList(session)
+		if collStatList != nil {
+			collStatList.Export(ch)
+		}
+	}
 }
 
 func (exporter *MongodbCollector) collectMongod(session *mgo.Session, ch chan<- prometheus.Metric) {
@@ -229,6 +247,22 @@ func (exporter *MongodbCollector) collectMongod(session *mgo.Session, ch chan<-
 	if serverStatus != nil {
 		serverStatus.Export(ch)
 	}
+
+	if exporter.Opts.CollectDatabaseMetrics {
+		log.Debug("Collecting Database Status From Mongod")
+		dbStatList := collector_mongod.GetDatabaseStatList(session)
+		if dbStatList != nil {
+			dbStatList.Export(ch)
+		}
+	}
+
+	if exporter.Opts.CollectCollectionMetrics {
+		log.Debug("Collecting Collection Status From Mongod")
+		collStatList := collector_mongod.GetCollectionStatList(session)
+		if collStatList != nil {
+			collStatList.Export(ch)
+		}
+	}
 }
 
 func (exporter *MongodbCollector) collectMongodReplSet(session *mgo.Session, ch chan<- prometheus.Metric) {