PMM-1764 Various small improvements

.travis.yml

@@ -14,6 +14,7 @@ env:
   - MONGODB_IMAGE=mongo:3.4
   - MONGODB_IMAGE=mongo:3.6
   - MONGODB_IMAGE=percona/percona-server-mongodb:3.4
+  - MONGODB_IMAGE=perconalab/percona-server-mongodb:3.6
 
 services:
   - docker

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## v0.4.0 (not released yet)
 
+* New flags `-collect.database` and `-collect.collection` can be used to enable collection of database and collection
+  metrics. They are disabled by default.
+* MongoDB connections are now kept between the scrapes. New flag `-mongodb.max-connections` (with the default value `1`)
+  controls the maximum number of established connections.
+* Add standard metrics:
+  * `mongodb_scrape_errors_total`
+  * `mongodb_up`
+* Some queries now contain [cursor comments](https://www.percona.com/blog/2017/06/21/tracing-mongodb-queries-to-code-with-cursor-comments/)
+  with source code locations.
 * Go vendoring switched to [dep](https://github.com/golang/dep).
 
 ## v0.3.1 (2017-09-08)

README.md

@@ -64,6 +64,12 @@ db.getSiblingDB("admin").createUser({
 export MONGODB_URL=mongodb://mongodb_exporter:s3cr3tpassw0rd@localhost:27017
 ```
 
+If you use [x.509 Certificates to Authenticate Clients](https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/), pass in username and `authMechanism` via [connection options](https://docs.mongodb.com/manual/reference/connection-string/#connections-connection-options) to the MongoDB uri. Eg:
+
+```
+mongodb://CN=myName,OU=myOrgUnit,O=myOrg,L=myLocality,ST=myState,C=myCountry@localhost:27017/?authMechanism=MONGODB-X509
+```
+
 ## Note about how this works
 
 Point the process to any mongo port and it will detect if it is a mongos, replicaset member, or stand alone mongod and return the appropriate metrics for that type of node. This was done to preent the need to an exporter per type of process.

collector/mongod/collections_status.go

@@ -0,0 +1,126 @@
+package collector_mongod
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
+	"gopkg.in/mgo.v2"
+	"gopkg.in/mgo.v2/bson"
+)
+
+var (
+	collectionSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "size",
+		Help:      "The total size in memory of all records in a collection",
+	}, []string{"db", "coll"})
+	collectionObjectCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "count",
+		Help:      "The number of objects or documents in this collection",
+	}, []string{"db", "coll"})
+	collectionAvgObjSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "avgobjsize",
+		Help:      "The average size of an object in the collection (plus any padding)",
+	}, []string{"db", "coll"})
+	collectionStorageSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "storage_size",
+		Help:      "The total amount of storage allocated to this collection for document storage",
+	}, []string{"db", "coll"})
+	collectionIndexes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "indexes",
+		Help:      "The number of indexes on the collection",
+	}, []string{"db", "coll"})
+	collectionIndexesSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db_coll",
+		Name:      "indexes_size",
+		Help:      "The total size of all indexes",
+	}, []string{"db", "coll"})
+)
+
+// CollectionStatList contains stats from all collections
+type CollectionStatList struct {
+	Members []CollectionStatus
+}
+
+// CollectionStatus represents stats about a collection in database (mongod and raw from mongos)
+type CollectionStatus struct {
+	Database    string
+	Name        string
+	Size        int `bson:"size,omitempty"`
+	Count       int `bson:"count,omitempty"`
+	AvgObjSize  int `bson:"avgObjSize,omitempty"`
+	StorageSize int `bson:"storageSize,omitempty"`
+	Indexes     int `bson:"indexSizes,omitempty"`
+	IndexesSize int `bson:"totalIndexSize,omitempty"`
+}
+
+// Export exports database stats to prometheus
+func (collStatList *CollectionStatList) Export(ch chan<- prometheus.Metric) {
+	for _, member := range collStatList.Members {
+		ls := prometheus.Labels{
+			"db":   member.Database,
+			"coll": member.Name,
+		}
+		collectionSize.With(ls).Set(float64(member.Size))
+		collectionObjectCount.With(ls).Set(float64(member.Count))
+		collectionAvgObjSize.With(ls).Set(float64(member.AvgObjSize))
+		collectionStorageSize.With(ls).Set(float64(member.StorageSize))
+		collectionIndexes.With(ls).Set(float64(member.Indexes))
+		collectionIndexesSize.With(ls).Set(float64(member.IndexesSize))
+	}
+	collectionSize.Collect(ch)
+	collectionObjectCount.Collect(ch)
+	collectionAvgObjSize.Collect(ch)
+	collectionStorageSize.Collect(ch)
+	collectionIndexes.Collect(ch)
+	collectionIndexesSize.Collect(ch)
+}
+
+// Describe describes database stats for prometheus
+func (collStatList *CollectionStatList) Describe(ch chan<- *prometheus.Desc) {
+	collectionSize.Describe(ch)
+	collectionObjectCount.Describe(ch)
+	collectionAvgObjSize.Describe(ch)
+	collectionStorageSize.Describe(ch)
+	collectionIndexes.Describe(ch)
+	collectionIndexesSize.Describe(ch)
+}
+
+// GetDatabaseStatus returns stats for a given database
+func GetCollectionStatList(session *mgo.Session) *CollectionStatList {
+	collectionStatList := &CollectionStatList{}
+	database_names, err := session.DatabaseNames()
+	if err != nil {
+		log.Error("Failed to get database names")
+		return nil
+	}
+	for _, db := range database_names {
+		collection_names, err := session.DB(db).CollectionNames()
+		if err != nil {
+			log.Error("Failed to get collection names for db=" + db)
+			return nil
+		}
+		for _, collection_name := range collection_names {
+			collStatus := CollectionStatus{}
+			err := session.DB(db).Run(bson.D{{"collStats", collection_name}, {"scale", 1}}, &collStatus)
+			collStatus.Database = db
+			collStatus.Name = collection_name
+			if err != nil {
+				log.Error("Failed to get collection status.")
+				return nil
+			}
+			collectionStatList.Members = append(collectionStatList.Members, collStatus)
+		}
+	}
+
+	return collectionStatList
+}

collector/mongod/database_status.go

@@ -0,0 +1,104 @@
+package collector_mongod
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
+	"gopkg.in/mgo.v2"
+	"gopkg.in/mgo.v2/bson"
+)
+
+var (
+	indexSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db",
+		Name:      "index_size_bytes",
+		Help:      "The total size in bytes of all indexes created on this database",
+	}, []string{"db"})
+	dataSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db",
+		Name:      "data_size_bytes",
+		Help:      "The total size in bytes of the uncompressed data held in this database",
+	}, []string{"db"})
+	collectionsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db",
+		Name:      "collections_total",
+		Help:      "Contains a count of the number of collections in that database",
+	}, []string{"db"})
+	indexesTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db",
+		Name:      "indexes_total",
+		Help:      "Contains a count of the total number of indexes across all collections in the database",
+	}, []string{"db"})
+	objectsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: "db",
+		Name:      "objects_total",
+		Help:      "Contains a count of the number of objects (i.e. documents) in the database across all collections",
+	}, []string{"db"})
+)
+
+// DatabaseStatList contains stats from all databases
+type DatabaseStatList struct {
+	Members []DatabaseStatus
+}
+
+// DatabaseStatus represents stats about a database (mongod and raw from mongos)
+type DatabaseStatus struct {
+	Name        string `bson:"db,omitempty"`
+	IndexSize   int    `bson:"indexSize,omitempty"`
+	DataSize    int    `bson:"dataSize,omitempty"`
+	Collections int    `bson:"collections,omitempty"`
+	Objects     int    `bson:"objects,omitempty"`
+	Indexes     int    `bson:"indexes,omitempty"`
+}
+
+// Export exports database stats to prometheus
+func (dbStatList *DatabaseStatList) Export(ch chan<- prometheus.Metric) {
+	for _, member := range dbStatList.Members {
+		ls := prometheus.Labels{"db": member.Name}
+		indexSize.With(ls).Set(float64(member.IndexSize))
+		dataSize.With(ls).Set(float64(member.DataSize))
+		collectionsTotal.With(ls).Set(float64(member.Collections))
+		indexesTotal.With(ls).Set(float64(member.Indexes))
+		objectsTotal.With(ls).Set(float64(member.Objects))
+	}
+	indexSize.Collect(ch)
+	dataSize.Collect(ch)
+	collectionsTotal.Collect(ch)
+	indexesTotal.Collect(ch)
+	objectsTotal.Collect(ch)
+
+}
+
+// Describe describes database stats for prometheus
+func (dbStatList *DatabaseStatList) Describe(ch chan<- *prometheus.Desc) {
+	indexSize.Describe(ch)
+	dataSize.Describe(ch)
+	collectionsTotal.Describe(ch)
+	indexesTotal.Describe(ch)
+	objectsTotal.Describe(ch)
+}
+
+// GetDatabaseStatList returns stats for all databases
+func GetDatabaseStatList(session *mgo.Session) *DatabaseStatList {
+	dbStatList := &DatabaseStatList{}
+	database_names, err := session.DatabaseNames()
+	if err != nil {
+		log.Error("Failed to get database names")
+		return nil
+	}
+	for _, db := range database_names {
+		dbStatus := DatabaseStatus{}
+		err := session.DB(db).Run(bson.D{{"dbStats", 1}, {"scale", 1}}, &dbStatus)
+		if err != nil {
+			log.Error("Failed to get database status.")
+			return nil
+		}
+		dbStatList.Members = append(dbStatList.Members, dbStatus)
+	}
+
+	return dbStatList
+}

collector/mongod/global_lock.go

@@ -31,12 +31,6 @@ var (
 		Name:      "total",
 		Help:      "The value of totalTime represents the time, in microseconds, since the database last started and creation of the globalLock. This is roughly equivalent to total server uptime",
 	})
-	globalLockLockTotal = prometheus.NewCounter(prometheus.CounterOpts{
-		Namespace: Namespace,
-		Subsystem: "global_lock",
-		Name:      "lock_total",
-		Help:      "The value of lockTime represents the time, in microseconds, since the database last started, that the globalLock has been held",
-	})
 )
 var (
 	globalLockCurrentQueue = prometheus.NewGaugeVec(prometheus.GaugeOpts{

collector/mongod/oplog_status.go

@@ -19,9 +19,13 @@ import (
 	"github.com/prometheus/common/log"
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
+
+	"github.com/percona/mongodb_exporter/shared"
 )
 
 var (
+	oplogDb          = "local"
+	oplogCollection  = "oplog.rs"
 	oplogStatusCount = prometheus.NewGauge(prometheus.GaugeOpts{
 		Namespace: Namespace,
 		Subsystem: "replset_oplog",
@@ -69,44 +73,34 @@ func BsonMongoTimestampToUnix(timestamp bson.MongoTimestamp) float64 {
 	return float64(timestamp >> 32)
 }
 
-func GetOplogTimestamps(session *mgo.Session) (*OplogTimestamps, error) {
-	oplogTimestamps := &OplogTimestamps{}
-	var err error
-
-	// retry once if there is an error
-	var tries int64 = 0
-	var head_result struct {
+func getOplogTailOrHeadTimestamp(session *mgo.Session, returnHead bool) (float64, error) {
+	var result struct {
 		Timestamp bson.MongoTimestamp `bson:"ts"`
 	}
-	for tries < 2 {
-		err = session.DB("local").C("oplog.rs").Find(nil).Sort("-$natural").Limit(1).One(&head_result)
-		if err == nil {
-			break
-		}
-		tries += 1
-	}
-	if err != nil {
-		return oplogTimestamps, err
-	}
 
-	// retry once if there is an error
-	tries = 0
-	var tail_result struct {
-		Timestamp bson.MongoTimestamp `bson:"ts"`
+	var sortCond string = "$natural"
+	if returnHead {
+		sortCond = "-$natural"
 	}
-	for tries < 2 {
-		err = session.DB("local").C("oplog.rs").Find(nil).Sort("$natural").Limit(1).One(&tail_result)
-		if err == nil {
-			break
-		}
-		tries += 1
+
+	findQuery := session.DB(oplogDb).C(oplogCollection).Find(nil).Sort(sortCond).Limit(1)
+	err := shared.AddCodeCommentToQuery(findQuery).One(&result)
+	return BsonMongoTimestampToUnix(result.Timestamp), err
+}
+
+func GetOplogTimestamps(session *mgo.Session) (*OplogTimestamps, error) {
+	headTs, err := getOplogTailOrHeadTimestamp(session, true)
+	if err != nil {
+		return nil, err
 	}
+	tailTs, err := getOplogTailOrHeadTimestamp(session, false)
 	if err != nil {
-		return oplogTimestamps, err
+		return nil, err
+	}
+	oplogTimestamps := &OplogTimestamps{
+		Head: headTs,
+		Tail: tailTs,
 	}
-
-	oplogTimestamps.Tail = BsonMongoTimestampToUnix(tail_result.Timestamp)
-	oplogTimestamps.Head = BsonMongoTimestampToUnix(head_result.Timestamp)
 	return oplogTimestamps, err
 }
 

collector/mongod/replset_status.go

@@ -115,12 +115,6 @@ var (
 		Name:      "member_config_version",
 		Help:      "The configVersion value is the replica set configuration version.",
 	}, []string{"set", "name", "state"})
-	memberOptime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: Namespace,
-		Subsystem: subsystem,
-		Name:      "member_optime",
-		Help:      "Information regarding the last operation from the operation log that this member has applied.",
-	}, []string{"set", "name", "state"})
 )
 
 // ReplSetStatus keeps the data returned by the GetReplSetStatus method