From b836caac04426afce8d7b1307bb0935a0a804e19 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 4 Jan 2018 12:57:56 +0100 Subject: [PATCH 01/17] Disable pre-fetching, limit conn pool and reuse db connection (#79) * Disable pre-fetching (not needed on exporter queries), limit conn pool size and re-use database connection to MongoDB * Remove .Ping() on exporter.mongoSess, mgo.Session will recreate the conn for us * avoid race condition with sync.Mutex * Use lock on close of mgo session also * Use lock on close of mgo session also #2 * make pool limit a flag instead of hard-code * default to 1 connection, it seems to work fine even at high volume * Must return .Copy() of mgo.Session to avoid mgo deadlocking under concurrent calls. Moved session SocketTimeout to no longer be forever * Must defer .Close() on copied mgo.Session from .getSession * must check if != nil a 2nd time in getSession() --- collector/mongodb_collector.go | 35 +++++++++++++++++++++++++++++++--- mongodb_exporter.go | 10 +++++++--- shared/connection.go | 5 ++++- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/collector/mongodb_collector.go b/collector/mongodb_collector.go index e7f90ad13..79f1f726b 100644 --- a/collector/mongodb_collector.go +++ b/collector/mongodb_collector.go @@ -15,8 +15,8 @@ package collector import ( - "errors" "fmt" + "sync" "time" "github.com/prometheus/client_golang/prometheus" @@ -38,6 +38,7 @@ type MongodbCollectorOpts struct { TLSPrivateKeyFile string TLSCaFile string TLSHostnameValidation bool + DbPoolLimit int } func (in MongodbCollectorOpts) toSessionOps() shared.MongoSessionOpts { @@ -48,6 +49,7 @@ func (in MongodbCollectorOpts) toSessionOps() shared.MongoSessionOpts { TLSPrivateKeyFile: in.TLSPrivateKeyFile, TLSCaFile: in.TLSCaFile, TLSHostnameValidation: in.TLSHostnameValidation, + PoolLimit: in.DbPoolLimit, } } @@ -57,6 +59,8 @@ type MongodbCollector struct { scrapesTotal prometheus.Counter lastScrapeError prometheus.Gauge lastScrapeDurationSeconds prometheus.Gauge + mongoSess *mgo.Session + mongoSessLock sync.Mutex } // NewMongodbCollector returns a new instance of a MongodbCollector. @@ -87,6 +91,31 @@ func NewMongodbCollector(opts MongodbCollectorOpts) *MongodbCollector { return exporter } +// getSession returns the cached *mgo.Session or creates a new session and returns it. +// Use sync.Mutex to avoid race condition around session creation. +func (exporter *MongodbCollector) getSession() *mgo.Session { + exporter.mongoSessLock.Lock() + defer exporter.mongoSessLock.Unlock() + + if exporter.mongoSess != nil { + return exporter.mongoSess.Copy() + } + exporter.mongoSess = shared.MongoSession(exporter.Opts.toSessionOps()) + if exporter.mongoSess != nil { + return exporter.mongoSess.Copy() + } + return nil +} + +// Close cleanly closes the mongo session if it exists. +func (exporter *MongodbCollector) Close() { + if exporter.mongoSess != nil { + exporter.mongoSessLock.Lock() + exporter.mongoSess.Close() + exporter.mongoSessLock.Unlock() + } +} + // Describe sends the super-set of all possible descriptors of metrics collected by this Collector // to the provided channel and returns once the last descriptor has been sent. // Part of prometheus.Collector interface. @@ -139,9 +168,9 @@ func (exporter *MongodbCollector) scrape(ch chan<- prometheus.Metric) { } }(time.Now()) - mongoSess := shared.MongoSession(exporter.Opts.toSessionOps()) + mongoSess := exporter.getSession() if mongoSess == nil { - err = errors.New("can't create mongo session") + log.Errorf("can't create mongo session to %s", exporter.Opts.URI) return } defer mongoSess.Close() diff --git a/mongodb_exporter.go b/mongodb_exporter.go index 8f1c44e26..109677dc1 100644 --- a/mongodb_exporter.go +++ b/mongodb_exporter.go @@ -63,6 +63,8 @@ var ( " \tIf not provided: System default CAs are used.") tlsDisableHostnameValidationF = flag.Bool("mongodb.tls-disable-hostname-validation", false, "Do hostname validation for server connection.") + dbPoolLimit = flag.Int("mongodb.max-connections", 1, "Max number of pooled connections to the database.") + // FIXME currently ignored enabledGroupsFlag = flag.String("groups.enabled", "asserts,durability,background_flushing,connections,extra_info,global_lock,index_counters,network,op_counters,op_counters_repl,memory,locks,metrics", "Comma-separated list of groups to use, for more info see: docs.mongodb.org/manual/reference/command/serverStatus/") ) @@ -154,8 +156,8 @@ func startWebServer() { } handler := prometheusHandler() - - registerCollector() + collector := registerCollector() + defer collector.Close() if (*sslCertFileF == "") != (*sslKeyFileF == "") { log.Fatal("One of the flags -web.ssl-cert-file or -web.ssl-key-file is missing to enable HTTPS/TLS") @@ -210,7 +212,7 @@ func startWebServer() { } } -func registerCollector() { +func registerCollector() *collector.MongodbCollector { mongodbCollector := collector.NewMongodbCollector(collector.MongodbCollectorOpts{ URI: *uriF, TLSConnection: *tlsF, @@ -218,8 +220,10 @@ func registerCollector() { TLSPrivateKeyFile: *tlsPrivateKeyF, TLSCaFile: *tlsCAF, TLSHostnameValidation: !(*tlsDisableHostnameValidationF), + DbPoolLimit: *dbPoolLimit, }) prometheus.MustRegister(mongodbCollector) + return mongodbCollector } func main() { diff --git a/shared/connection.go b/shared/connection.go index 1aaa5acde..ccb8fd96d 100644 --- a/shared/connection.go +++ b/shared/connection.go @@ -52,6 +52,7 @@ type MongoSessionOpts struct { TLSPrivateKeyFile string TLSCaFile string TLSHostnameValidation bool + PoolLimit int } func MongoSession(opts MongoSessionOpts) *mgo.Session { @@ -76,8 +77,10 @@ func MongoSession(opts MongoSessionOpts) *mgo.Session { return nil } session.SetMode(mgo.Eventual, true) + session.SetPoolLimit(opts.PoolLimit) + session.SetPrefetch(0.00) session.SetSyncTimeout(syncMongodbTimeout) - session.SetSocketTimeout(0) + session.SetSocketTimeout(dialMongodbTimeout) return session } From b6b8e6edabc1b39ddf6ca4a885a852f5349946ec Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 4 Jan 2018 15:30:19 +0300 Subject: [PATCH 02/17] Minor refactoring. * Use mutex hat. * Protect Close() with lock too. * Count getSession() errors. --- collector/mongodb_collector.go | 31 +++++++++++++++++-------------- mongodb_exporter.go | 2 +- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/collector/mongodb_collector.go b/collector/mongodb_collector.go index 79f1f726b..0077803a2 100644 --- a/collector/mongodb_collector.go +++ b/collector/mongodb_collector.go @@ -38,7 +38,7 @@ type MongodbCollectorOpts struct { TLSPrivateKeyFile string TLSCaFile string TLSHostnameValidation bool - DbPoolLimit int + DBPoolLimit int } func (in MongodbCollectorOpts) toSessionOps() shared.MongoSessionOpts { @@ -49,18 +49,20 @@ func (in MongodbCollectorOpts) toSessionOps() shared.MongoSessionOpts { TLSPrivateKeyFile: in.TLSPrivateKeyFile, TLSCaFile: in.TLSCaFile, TLSHostnameValidation: in.TLSHostnameValidation, - PoolLimit: in.DbPoolLimit, + PoolLimit: in.DBPoolLimit, } } // MongodbCollector is in charge of collecting mongodb's metrics. type MongodbCollector struct { - Opts MongodbCollectorOpts + Opts MongodbCollectorOpts + scrapesTotal prometheus.Counter lastScrapeError prometheus.Gauge lastScrapeDurationSeconds prometheus.Gauge - mongoSess *mgo.Session - mongoSessLock sync.Mutex + + mongoSessLock sync.Mutex + mongoSess *mgo.Session } // NewMongodbCollector returns a new instance of a MongodbCollector. @@ -97,22 +99,22 @@ func (exporter *MongodbCollector) getSession() *mgo.Session { exporter.mongoSessLock.Lock() defer exporter.mongoSessLock.Unlock() - if exporter.mongoSess != nil { - return exporter.mongoSess.Copy() + if exporter.mongoSess == nil { + exporter.mongoSess = shared.MongoSession(exporter.Opts.toSessionOps()) } - exporter.mongoSess = shared.MongoSession(exporter.Opts.toSessionOps()) - if exporter.mongoSess != nil { - return exporter.mongoSess.Copy() + if exporter.mongoSess == nil { + return nil } - return nil + return exporter.mongoSess.Copy() } // Close cleanly closes the mongo session if it exists. func (exporter *MongodbCollector) Close() { + exporter.mongoSessLock.Lock() + defer exporter.mongoSessLock.Unlock() + if exporter.mongoSess != nil { - exporter.mongoSessLock.Lock() exporter.mongoSess.Close() - exporter.mongoSessLock.Unlock() } } @@ -170,7 +172,8 @@ func (exporter *MongodbCollector) scrape(ch chan<- prometheus.Metric) { mongoSess := exporter.getSession() if mongoSess == nil { - log.Errorf("can't create mongo session to %s", exporter.Opts.URI) + err = fmt.Errorf("Can't create mongo session to %s", exporter.Opts.URI) + log.Error(err) return } defer mongoSess.Close() diff --git a/mongodb_exporter.go b/mongodb_exporter.go index 109677dc1..10112aef7 100644 --- a/mongodb_exporter.go +++ b/mongodb_exporter.go @@ -220,7 +220,7 @@ func registerCollector() *collector.MongodbCollector { TLSPrivateKeyFile: *tlsPrivateKeyF, TLSCaFile: *tlsCAF, TLSHostnameValidation: !(*tlsDisableHostnameValidationF), - DbPoolLimit: *dbPoolLimit, + DBPoolLimit: *dbPoolLimit, }) prometheus.MustRegister(mongodbCollector) return mongodbCollector From f08e2e2ddeafbc0e8b935c79b08ff0a0ee6569c8 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 4 Jan 2018 13:43:06 +0100 Subject: [PATCH 03/17] Add MongoDB Query-comments to .Find() operations (#80) * Added query comments to .Find() queries in oplog_status.go, created shared/utils.go function to add comments, cleaned up duplicated code I made long ago * shorten comment --- collector/mongod/oplog_status.go | 53 +++++++++++++++++--------------- shared/utils.go | 9 ++++++ 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/collector/mongod/oplog_status.go b/collector/mongod/oplog_status.go index 92609486f..f006fc027 100644 --- a/collector/mongod/oplog_status.go +++ b/collector/mongod/oplog_status.go @@ -15,6 +15,7 @@ package collector_mongod import ( + "github.com/percona/mongodb_exporter/shared" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/log" "gopkg.in/mgo.v2" @@ -22,6 +23,8 @@ import ( ) var ( + oplogDb = "local" + oplogCollection = "oplog.rs" oplogStatusCount = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "replset_oplog", @@ -69,44 +72,44 @@ func BsonMongoTimestampToUnix(timestamp bson.MongoTimestamp) float64 { return float64(timestamp >> 32) } -func GetOplogTimestamps(session *mgo.Session) (*OplogTimestamps, error) { - oplogTimestamps := &OplogTimestamps{} - var err error - - // retry once if there is an error - var tries int64 = 0 - var head_result struct { +func getOplogTailOrHeadTimestamp(session *mgo.Session, returnHead bool) (float64, error) { + var result struct { Timestamp bson.MongoTimestamp `bson:"ts"` } - for tries < 2 { - err = session.DB("local").C("oplog.rs").Find(nil).Sort("-$natural").Limit(1).One(&head_result) - if err == nil { - break - } - tries += 1 - } - if err != nil { - return oplogTimestamps, err + + var sortCond string = "$natural" + if returnHead { + sortCond = "-$natural" } // retry once if there is an error - tries = 0 - var tail_result struct { - Timestamp bson.MongoTimestamp `bson:"ts"` - } + var err error + var tries int64 = 0 for tries < 2 { - err = session.DB("local").C("oplog.rs").Find(nil).Sort("$natural").Limit(1).One(&tail_result) + findQuery := session.DB(oplogDb).C(oplogCollection).Find(nil).Sort(sortCond).Limit(1) + err = shared.AddCodeCommentToQuery(findQuery).One(&result) if err == nil { break } tries += 1 } + + return BsonMongoTimestampToUnix(result.Timestamp), err +} + +func GetOplogTimestamps(session *mgo.Session) (*OplogTimestamps, error) { + headTs, err := getOplogTailOrHeadTimestamp(session, true) + if err != nil { + return nil, err + } + tailTs, err := getOplogTailOrHeadTimestamp(session, false) if err != nil { - return oplogTimestamps, err + return nil, err + } + oplogTimestamps := &OplogTimestamps{ + Head: headTs, + Tail: tailTs, } - - oplogTimestamps.Tail = BsonMongoTimestampToUnix(tail_result.Timestamp) - oplogTimestamps.Head = BsonMongoTimestampToUnix(head_result.Timestamp) return oplogTimestamps, err } diff --git a/shared/utils.go b/shared/utils.go index 4c8c71ecd..372a456ee 100644 --- a/shared/utils.go +++ b/shared/utils.go @@ -18,6 +18,10 @@ import ( "crypto/tls" "crypto/x509" "io/ioutil" + "runtime" + "strconv" + + "gopkg.in/mgo.v2" ) func LoadCaFrom(pemFile string) (*x509.CertPool, error) { @@ -37,3 +41,8 @@ func LoadKeyPairFrom(pemFile string, privateKeyPemFile string) (tls.Certificate, } return tls.LoadX509KeyPair(pemFile, targetPrivateKeyPemFile) } + +func AddCodeCommentToQuery(query *mgo.Query) *mgo.Query { + _, fileName, lineNum, _ := runtime.Caller(1) + return query.Comment(fileName + ":" + strconv.Itoa(lineNum)) +} From 7c439c19a8245ad32ef9d2bf0012e308d61fdf3b Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 4 Jan 2018 15:48:19 +0300 Subject: [PATCH 04/17] Tiny tweaks. --- collector/mongod/oplog_status.go | 3 ++- shared/utils.go | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/collector/mongod/oplog_status.go b/collector/mongod/oplog_status.go index f006fc027..c6bacaf07 100644 --- a/collector/mongod/oplog_status.go +++ b/collector/mongod/oplog_status.go @@ -15,11 +15,12 @@ package collector_mongod import ( - "github.com/percona/mongodb_exporter/shared" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/log" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" + + "github.com/percona/mongodb_exporter/shared" ) var ( diff --git a/shared/utils.go b/shared/utils.go index 372a456ee..1316ce278 100644 --- a/shared/utils.go +++ b/shared/utils.go @@ -43,6 +43,9 @@ func LoadKeyPairFrom(pemFile string, privateKeyPemFile string) (tls.Certificate, } func AddCodeCommentToQuery(query *mgo.Query) *mgo.Query { - _, fileName, lineNum, _ := runtime.Caller(1) + _, fileName, lineNum, ok := runtime.Caller(1) + if !ok { + return query + } return query.Comment(fileName + ":" + strconv.Itoa(lineNum)) } From 24a1cbcfa5483b918f3c211beeead4402ddcb6de Mon Sep 17 00:00:00 2001 From: Adam Ward Date: Wed, 6 Sep 2017 10:26:23 +1000 Subject: [PATCH 05/17] Document MONGODB-X509 auth --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index b57cb21c4..d1cfb60f1 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,12 @@ db.getSiblingDB("admin").createUser({ export MONGODB_URL=mongodb://mongodb_exporter:s3cr3tpassw0rd@localhost:27017 ``` +If you use [x.509 Certificates to Authenticate Clients](https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/), pass in username and `authMechanism` via [connection options](https://docs.mongodb.com/manual/reference/connection-string/#connections-connection-options) to the MongoDB uri. Eg: + +``` +mongodb://CN=myName,OU=myOrgUnit,O=myOrg,L=myLocality,ST=myState,C=myCountry@localhost:27017/?authMechanism=MONGODB-X509 +``` + ## Note about how this works Point the process to any mongo port and it will detect if it is a mongos, replicaset member, or stand alone mongod and return the appropriate metrics for that type of node. This was done to preent the need to an exporter per type of process. From 2e2b7e9da8bb179f9667b4c515ff7aec594de8b1 Mon Sep 17 00:00:00 2001 From: Vilius Okockis <228085+DeathBorn@users.noreply.github.com> Date: Wed, 29 Nov 2017 18:30:09 +0200 Subject: [PATCH 06/17] adding database and collection stats collection --- collector/mongod/collections_status.go | 128 +++++++++++++++++++++++++ collector/mongod/database_status.go | 106 ++++++++++++++++++++ collector/mongodb_collector.go | 48 ++++++++-- collector/mongos/collections_status.go | 128 +++++++++++++++++++++++++ collector/mongos/database_status.go | 127 ++++++++++++++++++++++++ mongodb_exporter.go | 18 ++-- 6 files changed, 541 insertions(+), 14 deletions(-) create mode 100644 collector/mongod/collections_status.go create mode 100644 collector/mongod/database_status.go create mode 100644 collector/mongos/collections_status.go create mode 100644 collector/mongos/database_status.go diff --git a/collector/mongod/collections_status.go b/collector/mongod/collections_status.go new file mode 100644 index 000000000..26827e552 --- /dev/null +++ b/collector/mongod/collections_status.go @@ -0,0 +1,128 @@ +package collector_mongod + +import ( + "github.com/prometheus/common/log" + "github.com/prometheus/client_golang/prometheus" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" +) + +var ( + collectionSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "size", + Help: "The total size in memory of all records in a collection", + }, []string{"db","coll"}) + collectionObjectCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "count", + Help: "The number of objects or documents in this collection", + }, []string{"db","coll"}) + collectionAvgObjSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "avgobjsize", + Help: "The average size of an object in the collection (plus any padding)", + }, []string{"db","coll"}) + collectionStorageSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "storage_size", + Help: "The total amount of storage allocated to this collection for document storage", + }, []string{"db","coll"}) + collectionIndexes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "indexes", + Help: "The number of indexes on the collection", + }, []string{"db","coll"}) + collectionIndexesSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "indexes_size", + Help: "The total size of all indexes", + }, []string{"db","coll"}) +) + +// CollectionStatList contains stats from all collections +type CollectionStatList struct { + Members []CollectionStatus +} + +// CollectionStatus represents stats about a collection in database (mongod and raw from mongos) +type CollectionStatus struct { + Database string + Name string + Size int `bson:"size,omitempty"` + Count int `bson:"count,omitempty"` + AvgObjSize int `bson:"avgObjSize,omitempty"` + StorageSize int `bson:"storageSize,omitempty"` + Indexes int `bson:"indexSizes,omitempty"` + IndexesSize int `bson:"totalIndexSize,omitempty"` +} + + +// Export exports database stats to prometheus +func (collStatList *CollectionStatList) Export(ch chan<- prometheus.Metric) { + for _, member := range collStatList.Members { + ls := prometheus.Labels{ + "db": member.Database, + "coll": member.Name, + } + collectionSize.With(ls).Set(float64(member.Size)) + collectionObjectCount.With(ls).Set(float64(member.Count)) + collectionAvgObjSize.With(ls).Set(float64(member.AvgObjSize)) + collectionStorageSize.With(ls).Set(float64(member.StorageSize)) + collectionIndexes.With(ls).Set(float64(member.Indexes)) + collectionIndexesSize.With(ls).Set(float64(member.IndexesSize)) + } + collectionSize.Collect(ch) + collectionObjectCount.Collect(ch) + collectionAvgObjSize.Collect(ch) + collectionStorageSize.Collect(ch) + collectionIndexes.Collect(ch) + collectionIndexesSize.Collect(ch) +} + +// Describe describes database stats for prometheus +func (collStatList *CollectionStatList) Describe(ch chan<- *prometheus.Desc) { + collectionSize.Describe(ch) + collectionObjectCount.Describe(ch) + collectionAvgObjSize.Describe(ch) + collectionStorageSize.Describe(ch) + collectionIndexes.Describe(ch) + collectionIndexesSize.Describe(ch) +} + +// GetDatabaseStatus returns stats for a given database +func GetCollectionStatList(session *mgo.Session) *CollectionStatList { + collectionStatList := &CollectionStatList{} + database_names, err := session.DatabaseNames() + if err != nil { + log.Error("Failed to get database names") + return nil + } + for _, db := range database_names { + if db == "admin" || db == "test" || db == "local" { continue } + collection_names, err := session.DB(db).CollectionNames() + if err != nil { + log.Error("Failed to get collection names for db=" + db) + return nil + } + for _, collection_name := range collection_names { + collStatus := CollectionStatus{} + err := session.DB(db).Run(bson.D{{"collStats", collection_name}, {"scale", 1}}, &collStatus) + collStatus.Database = db + collStatus.Name = collection_name + if err != nil { + log.Error("Failed to get collection status.") + return nil + } + collectionStatList.Members = append(collectionStatList.Members,collStatus) + } + } + + return collectionStatList +} diff --git a/collector/mongod/database_status.go b/collector/mongod/database_status.go new file mode 100644 index 000000000..53a9ad70d --- /dev/null +++ b/collector/mongod/database_status.go @@ -0,0 +1,106 @@ +package collector_mongod + +import ( + "github.com/prometheus/common/log" + "github.com/prometheus/client_golang/prometheus" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" +) + +var ( + indexSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "index_size_bytes", + Help: "The total size in bytes of all indexes created on this database", + }, []string{"db"}) + dataSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "data_size_bytes", + Help: "The total size in bytes of the uncompressed data held in this database", + }, []string{"db"}) + collectionsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "collections_total", + Help: "Contains a count of the number of collections in that database", + }, []string{"db"}) + indexesTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "indexes_total", + Help: "Contains a count of the total number of indexes across all collections in the database", + }, []string{"db"}) + objectsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "objects_total", + Help: "Contains a count of the number of objects (i.e. documents) in the database across all collections", + }, []string{"db"}) +) + + +// DatabaseStatList contains stats from all databases +type DatabaseStatList struct { + Members []DatabaseStatus +} + +// DatabaseStatus represents stats about a database (mongod and raw from mongos) +type DatabaseStatus struct { + Name string `bson:"db,omitempty"` + IndexSize int `bson:"indexSize,omitempty"` + DataSize int `bson:"dataSize,omitempty"` + Collections int `bson:"collections,omitempty"` + Objects int `bson:"objects,omitempty"` + Indexes int `bson:"indexes,omitempty"` +} + +// Export exports database stats to prometheus +func (dbStatList *DatabaseStatList) Export(ch chan<- prometheus.Metric) { + for _, member := range dbStatList.Members { + ls := prometheus.Labels{ "db": member.Name } + indexSize.With(ls).Set(float64(member.IndexSize)) + dataSize.With(ls).Set(float64(member.DataSize)) + collectionsTotal.With(ls).Set(float64(member.Collections)) + indexesTotal.With(ls).Set(float64(member.Indexes)) + objectsTotal.With(ls).Set(float64(member.Objects)) + } + indexSize.Collect(ch) + dataSize.Collect(ch) + collectionsTotal.Collect(ch) + indexesTotal.Collect(ch) + objectsTotal.Collect(ch) + +} + +// Describe describes database stats for prometheus +func (dbStatList *DatabaseStatList) Describe(ch chan<- *prometheus.Desc) { + indexSize.Describe(ch) + dataSize.Describe(ch) + collectionsTotal.Describe(ch) + indexesTotal.Describe(ch) + objectsTotal.Describe(ch) +} + +// GetDatabaseStatList returns stats for all databases +func GetDatabaseStatList(session *mgo.Session) *DatabaseStatList { + dbStatList := &DatabaseStatList{} + database_names, err := session.DatabaseNames() + if err != nil { + log.Error("Failed to get database names") + return nil + } + for _, db := range database_names { + if db == "admin" || db == "test" || db == "local" { continue } + dbStatus := DatabaseStatus{} + err := session.DB(db).Run(bson.D{{"dbStats", 1}, {"scale", 1}}, &dbStatus) + if err != nil { + log.Error("Failed to get database status.") + return nil + } + dbStatList.Members = append(dbStatList.Members,dbStatus) + } + + return dbStatList +} diff --git a/collector/mongodb_collector.go b/collector/mongodb_collector.go index 0077803a2..cec3a4b72 100644 --- a/collector/mongodb_collector.go +++ b/collector/mongodb_collector.go @@ -32,13 +32,15 @@ const namespace = "mongodb" // MongodbCollectorOpts is the options of the mongodb collector. type MongodbCollectorOpts struct { - URI string - TLSConnection bool - TLSCertificateFile string - TLSPrivateKeyFile string - TLSCaFile string - TLSHostnameValidation bool - DBPoolLimit int + URI string + TLSConnection bool + TLSCertificateFile string + TLSPrivateKeyFile string + TLSCaFile string + TLSHostnameValidation bool + DBPoolLimit int + CollectDatabaseMetrics bool + CollectCollectionMetrics bool } func (in MongodbCollectorOpts) toSessionOps() shared.MongoSessionOpts { @@ -221,6 +223,22 @@ func (exporter *MongodbCollector) collectMongos(session *mgo.Session, ch chan<- if shardingStatus != nil { shardingStatus.Export(ch) } + + if exporter.Opts.CollectDatabaseMetrics { + log.Debug("Collecting Database Status From Mongos") + dbStatList := collector_mongos.GetDatabaseStatList(session) + if dbStatList != nil { + dbStatList.Export(ch) + } + } + + if exporter.Opts.CollectCollectionMetrics { + log.Debug("Collecting Collection Status From Mongos") + collStatList := collector_mongos.GetCollectionStatList(session) + if collStatList != nil { + collStatList.Export(ch) + } + } } func (exporter *MongodbCollector) collectMongod(session *mgo.Session, ch chan<- prometheus.Metric) { @@ -229,6 +247,22 @@ func (exporter *MongodbCollector) collectMongod(session *mgo.Session, ch chan<- if serverStatus != nil { serverStatus.Export(ch) } + + if exporter.Opts.CollectDatabaseMetrics { + log.Debug("Collecting Database Status From Mongod") + dbStatList := collector_mongod.GetDatabaseStatList(session) + if dbStatList != nil { + dbStatList.Export(ch) + } + } + + if exporter.Opts.CollectCollectionMetrics { + log.Debug("Collecting Collection Status From Mongod") + collStatList := collector_mongod.GetCollectionStatList(session) + if collStatList != nil { + collStatList.Export(ch) + } + } } func (exporter *MongodbCollector) collectMongodReplSet(session *mgo.Session, ch chan<- prometheus.Metric) { diff --git a/collector/mongos/collections_status.go b/collector/mongos/collections_status.go new file mode 100644 index 000000000..c6ffb2e91 --- /dev/null +++ b/collector/mongos/collections_status.go @@ -0,0 +1,128 @@ +package collector_mongos + +import ( + "github.com/prometheus/common/log" + "github.com/prometheus/client_golang/prometheus" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" +) + +var ( + collectionSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "size", + Help: "The total size in memory of all records in a collection", + }, []string{"db", "coll"}) + collectionObjectCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "count", + Help: "The number of objects or documents in this collection", + }, []string{"db", "coll"}) + collectionAvgObjSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "avgobjsize", + Help: "The average size of an object in the collection (plus any padding)", + }, []string{"db", "coll"}) + collectionStorageSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "storage_size", + Help: "The total amount of storage allocated to this collection for document storage", + }, []string{"db", "coll"}) + collectionIndexes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "indexes", + Help: "The number of indexes on the collection", + }, []string{"db", "coll"}) + collectionIndexesSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db_coll", + Name: "indexes_size", + Help: "The total size of all indexes", + }, []string{"db", "coll"}) +) + + +// CollectionStatList contains stats from all collections +type CollectionStatList struct { + Members []CollectionStatus +} + +// CollectionStatus represents stats about a collection in database (mongod and raw from mongos) +type CollectionStatus struct { + Database string + Name string + Size int `bson:"size,omitempty"` + Count int `bson:"count,omitempty"` + AvgObjSize int `bson:"avgObjSize,omitempty"` + StorageSize int `bson:"storageSize,omitempty"` + Indexes int `bson:"indexSizes,omitempty"` + IndexesSize int `bson:"totalIndexSize,omitempty"` +} + +// Export exports database stats to prometheus +func (collStatList *CollectionStatList) Export(ch chan<- prometheus.Metric) { + for _, member := range collStatList.Members { + ls := prometheus.Labels{ + "db": member.Database, + "coll": member.Name, + } + collectionSize.With(ls).Set(float64(member.Size)) + collectionObjectCount.With(ls).Set(float64(member.Count)) + collectionAvgObjSize.With(ls).Set(float64(member.AvgObjSize)) + collectionStorageSize.With(ls).Set(float64(member.StorageSize)) + collectionIndexes.With(ls).Set(float64(member.Indexes)) + collectionIndexesSize.With(ls).Set(float64(member.IndexesSize)) + } + collectionSize.Collect(ch) + collectionObjectCount.Collect(ch) + collectionAvgObjSize.Collect(ch) + collectionStorageSize.Collect(ch) + collectionIndexes.Collect(ch) + collectionIndexesSize.Collect(ch) +} + +// Describe describes database stats for prometheus +func (collStatList *CollectionStatList) Describe(ch chan<- *prometheus.Desc) { + collectionSize.Describe(ch) + collectionObjectCount.Describe(ch) + collectionAvgObjSize.Describe(ch) + collectionStorageSize.Describe(ch) + collectionIndexes.Describe(ch) + collectionIndexesSize.Describe(ch) +} + +// GetDatabaseStatus returns stats for a given database +func GetCollectionStatList(session *mgo.Session) *CollectionStatList { + collectionStatList := &CollectionStatList{} + database_names, err := session.DatabaseNames() + if err != nil { + log.Error("Failed to get database names") + return nil + } + for _, db := range database_names { + if db == "admin" || db == "test" || db == "local" { continue } + collection_names, err := session.DB(db).CollectionNames() + if err != nil { + log.Error("Failed to get collection names for db=" + db) + return nil + } + for _, collection_name := range collection_names { + collStatus := CollectionStatus{} + err := session.DB(db).Run(bson.D{{"collStats", collection_name}, {"scale", 1}}, &collStatus) + collStatus.Database = db + collStatus.Name = collection_name + if err != nil { + log.Error("Failed to get collection status.") + return nil + } + collectionStatList.Members = append(collectionStatList.Members,collStatus) + } + } + + return collectionStatList +} \ No newline at end of file diff --git a/collector/mongos/database_status.go b/collector/mongos/database_status.go new file mode 100644 index 000000000..dd2e35089 --- /dev/null +++ b/collector/mongos/database_status.go @@ -0,0 +1,127 @@ +package collector_mongos + +import ( + "strings" + + "github.com/prometheus/common/log" + "github.com/prometheus/client_golang/prometheus" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" +) + +var ( + indexSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "index_size_bytes", + Help: "The total size in bytes of all indexes created on this database", + }, []string{"db", "shard"}) + dataSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "data_size_bytes", + Help: "The total size in bytes of the uncompressed data held in this database", + }, []string{"db", "shard"}) + collectionsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "collections_total", + Help: "Contains a count of the number of collections in that database", + }, []string{"db", "shard"}) + indexesTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "indexes_total", + Help: "Contains a count of the total number of indexes across all collections in the database", + }, []string{"db", "shard"}) + objectsTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "db", + Name: "objects_total", + Help: "Contains a count of the number of objects (i.e. documents) in the database across all collections", + }, []string{"db", "shard"}) +) + + +// DatabaseStatList contains stats from all databases +type DatabaseStatList struct { + Members []DatabaseStatus +} + +// DatabaseStatus represents stats about a database (mongod and raw from mongos) +type DatabaseStatus struct { + RawStatus // embed to collect top-level attributes + Shards map[string]*RawStatus `bson:"raw,omitempty"` +} + +// RawStatus represents stats about a database from Mongos side +type RawStatus struct { + Name string `bson:"db,omitempty"` + IndexSize int `bson:"indexSize,omitempty"` + DataSize int `bson:"dataSize,omitempty"` + Collections int `bson:"collections,omitempty"` + Objects int `bson:"objects,omitempty"` + Indexes int `bson:"indexes,omitempty"` +} + +// Export exports database stats to prometheus +func (dbStatList *DatabaseStatList) Export(ch chan<- prometheus.Metric) { + for _, member := range dbStatList.Members { + if len(member.Shards) > 0 { + for shard, stats := range member.Shards { + ls := prometheus.Labels{ + "db": stats.Name, + "shard": strings.Split(shard, "/")[0], + } + indexSize.With(ls).Set(float64(stats.IndexSize)) + dataSize.With(ls).Set(float64(stats.DataSize)) + collectionsTotal.With(ls).Set(float64(stats.Collections)) + indexesTotal.With(ls).Set(float64(stats.Indexes)) + objectsTotal.With(ls).Set(float64(stats.Objects)) + } + } + } + + indexSize.Collect(ch) + dataSize.Collect(ch) + collectionsTotal.Collect(ch) + indexesTotal.Collect(ch) + objectsTotal.Collect(ch) + + indexSize.Reset() + dataSize.Reset() + collectionsTotal.Reset() + indexesTotal.Reset() + objectsTotal.Reset() +} + +// Describe describes database stats for prometheus +func (dbStatList *DatabaseStatList) Describe(ch chan<- *prometheus.Desc) { + indexSize.Describe(ch) + dataSize.Describe(ch) + collectionsTotal.Describe(ch) + indexesTotal.Describe(ch) + objectsTotal.Describe(ch) +} + +// GetDatabaseStatList returns stats for all databases +func GetDatabaseStatList(session *mgo.Session) *DatabaseStatList { + dbStatList := &DatabaseStatList{} + database_names, err := session.DatabaseNames() + if err != nil { + log.Error("Failed to get database names") + return nil + } + for _, db := range database_names { + if db == "admin" || db == "test" || db == "local" { continue } + dbStatus := DatabaseStatus{} + err := session.DB(db).Run(bson.D{{"dbStats", 1}, {"scale", 1}}, &dbStatus) + if err != nil { + log.Error("Failed to get database status.") + return nil + } + dbStatList.Members = append(dbStatList.Members,dbStatus) + } + + return dbStatList +} diff --git a/mongodb_exporter.go b/mongodb_exporter.go index 10112aef7..017c22c08 100644 --- a/mongodb_exporter.go +++ b/mongodb_exporter.go @@ -52,6 +52,8 @@ var ( sslCertFileF = flag.String("web.ssl-cert-file", "", "Path to SSL certificate file.") sslKeyFileF = flag.String("web.ssl-key-file", "", "Path to SSL key file.") + DbF = flag.Bool("collect.database", false, "Enable collection of Database metrics") + CollF = flag.Bool("collect.collection", false, "Enable collection of Collection metrics") uriF = flag.String("mongodb.uri", mongodbDefaultURI(), "MongoDB URI, format: [mongodb://][user:pass@]host1[:port1][,host2[:port2],...][/database][?options]") tlsF = flag.Bool("mongodb.tls", false, "Enable tls connection with mongo server") tlsCertF = flag.String("mongodb.tls-cert", "", "Path to PEM file that contains the certificate (and optionally also the decrypted private key in PEM format).\n"+ @@ -214,13 +216,15 @@ func startWebServer() { func registerCollector() *collector.MongodbCollector { mongodbCollector := collector.NewMongodbCollector(collector.MongodbCollectorOpts{ - URI: *uriF, - TLSConnection: *tlsF, - TLSCertificateFile: *tlsCertF, - TLSPrivateKeyFile: *tlsPrivateKeyF, - TLSCaFile: *tlsCAF, - TLSHostnameValidation: !(*tlsDisableHostnameValidationF), - DBPoolLimit: *dbPoolLimit, + URI: *uriF, + TLSConnection: *tlsF, + TLSCertificateFile: *tlsCertF, + TLSPrivateKeyFile: *tlsPrivateKeyF, + TLSCaFile: *tlsCAF, + TLSHostnameValidation: !(*tlsDisableHostnameValidationF), + DBPoolLimit: *dbPoolLimit, + CollectDatabaseMetrics: *DbF, + CollectCollectionMetrics: *CollF, }) prometheus.MustRegister(mongodbCollector) return mongodbCollector From dfde4020099cdd4b2194d3cabf70d3719a484be1 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 4 Jan 2018 16:34:53 +0300 Subject: [PATCH 07/17] Unify flags style. --- mongodb_exporter.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mongodb_exporter.go b/mongodb_exporter.go index 017c22c08..84b7eb5d4 100644 --- a/mongodb_exporter.go +++ b/mongodb_exporter.go @@ -52,8 +52,9 @@ var ( sslCertFileF = flag.String("web.ssl-cert-file", "", "Path to SSL certificate file.") sslKeyFileF = flag.String("web.ssl-key-file", "", "Path to SSL key file.") - DbF = flag.Bool("collect.database", false, "Enable collection of Database metrics") - CollF = flag.Bool("collect.collection", false, "Enable collection of Collection metrics") + collectDatabaseF = flag.Bool("collect.database", false, "Enable collection of Database metrics") + collectCollectionF = flag.Bool("collect.collection", false, "Enable collection of Collection metrics") + uriF = flag.String("mongodb.uri", mongodbDefaultURI(), "MongoDB URI, format: [mongodb://][user:pass@]host1[:port1][,host2[:port2],...][/database][?options]") tlsF = flag.Bool("mongodb.tls", false, "Enable tls connection with mongo server") tlsCertF = flag.String("mongodb.tls-cert", "", "Path to PEM file that contains the certificate (and optionally also the decrypted private key in PEM format).\n"+ @@ -64,8 +65,7 @@ var ( " \tIf provided: MongoDB servers connecting to should present a certificate signed by one of this CAs.\n"+ " \tIf not provided: System default CAs are used.") tlsDisableHostnameValidationF = flag.Bool("mongodb.tls-disable-hostname-validation", false, "Do hostname validation for server connection.") - - dbPoolLimit = flag.Int("mongodb.max-connections", 1, "Max number of pooled connections to the database.") + maxConnectionsF = flag.Int("mongodb.max-connections", 1, "Max number of pooled connections to the database.") // FIXME currently ignored enabledGroupsFlag = flag.String("groups.enabled", "asserts,durability,background_flushing,connections,extra_info,global_lock,index_counters,network,op_counters,op_counters_repl,memory,locks,metrics", "Comma-separated list of groups to use, for more info see: docs.mongodb.org/manual/reference/command/serverStatus/") @@ -222,9 +222,9 @@ func registerCollector() *collector.MongodbCollector { TLSPrivateKeyFile: *tlsPrivateKeyF, TLSCaFile: *tlsCAF, TLSHostnameValidation: !(*tlsDisableHostnameValidationF), - DBPoolLimit: *dbPoolLimit, - CollectDatabaseMetrics: *DbF, - CollectCollectionMetrics: *CollF, + DBPoolLimit: *maxConnectionsF, + CollectDatabaseMetrics: *collectDatabaseF, + CollectCollectionMetrics: *collectCollectionF, }) prometheus.MustRegister(mongodbCollector) return mongodbCollector From e08ab38b29be7db637f84884780cafaf13921cd3 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 4 Jan 2018 16:35:50 +0300 Subject: [PATCH 08/17] Apply `gofmt -s`. --- collector/mongod/collections_status.go | 39 +++++++++++++------------- collector/mongod/database_status.go | 11 ++++---- collector/mongos/collections_status.go | 29 ++++++++++--------- collector/mongos/database_status.go | 13 +++++---- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/collector/mongod/collections_status.go b/collector/mongod/collections_status.go index 26827e552..ac4b3f30a 100644 --- a/collector/mongod/collections_status.go +++ b/collector/mongod/collections_status.go @@ -1,8 +1,8 @@ package collector_mongod import ( - "github.com/prometheus/common/log" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/log" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" ) @@ -13,37 +13,37 @@ var ( Subsystem: "db_coll", Name: "size", Help: "The total size in memory of all records in a collection", - }, []string{"db","coll"}) + }, []string{"db", "coll"}) collectionObjectCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "db_coll", Name: "count", Help: "The number of objects or documents in this collection", - }, []string{"db","coll"}) + }, []string{"db", "coll"}) collectionAvgObjSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "db_coll", Name: "avgobjsize", Help: "The average size of an object in the collection (plus any padding)", - }, []string{"db","coll"}) + }, []string{"db", "coll"}) collectionStorageSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "db_coll", Name: "storage_size", Help: "The total amount of storage allocated to this collection for document storage", - }, []string{"db","coll"}) + }, []string{"db", "coll"}) collectionIndexes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "db_coll", Name: "indexes", Help: "The number of indexes on the collection", - }, []string{"db","coll"}) + }, []string{"db", "coll"}) collectionIndexesSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: Namespace, Subsystem: "db_coll", Name: "indexes_size", Help: "The total size of all indexes", - }, []string{"db","coll"}) + }, []string{"db", "coll"}) ) // CollectionStatList contains stats from all collections @@ -53,22 +53,21 @@ type CollectionStatList struct { // CollectionStatus represents stats about a collection in database (mongod and raw from mongos) type CollectionStatus struct { - Database string + Database string Name string - Size int `bson:"size,omitempty"` - Count int `bson:"count,omitempty"` - AvgObjSize int `bson:"avgObjSize,omitempty"` - StorageSize int `bson:"storageSize,omitempty"` - Indexes int `bson:"indexSizes,omitempty"` - IndexesSize int `bson:"totalIndexSize,omitempty"` + Size int `bson:"size,omitempty"` + Count int `bson:"count,omitempty"` + AvgObjSize int `bson:"avgObjSize,omitempty"` + StorageSize int `bson:"storageSize,omitempty"` + Indexes int `bson:"indexSizes,omitempty"` + IndexesSize int `bson:"totalIndexSize,omitempty"` } - // Export exports database stats to prometheus func (collStatList *CollectionStatList) Export(ch chan<- prometheus.Metric) { for _, member := range collStatList.Members { - ls := prometheus.Labels{ - "db": member.Database, + ls := prometheus.Labels{ + "db": member.Database, "coll": member.Name, } collectionSize.With(ls).Set(float64(member.Size)) @@ -105,7 +104,9 @@ func GetCollectionStatList(session *mgo.Session) *CollectionStatList { return nil } for _, db := range database_names { - if db == "admin" || db == "test" || db == "local" { continue } + if db == "admin" || db == "test" || db == "local" { + continue + } collection_names, err := session.DB(db).CollectionNames() if err != nil { log.Error("Failed to get collection names for db=" + db) @@ -120,7 +121,7 @@ func GetCollectionStatList(session *mgo.Session) *CollectionStatList { log.Error("Failed to get collection status.") return nil } - collectionStatList.Members = append(collectionStatList.Members,collStatus) + collectionStatList.Members = append(collectionStatList.Members, collStatus) } } diff --git a/collector/mongod/database_status.go b/collector/mongod/database_status.go index 53a9ad70d..44d22e9cc 100644 --- a/collector/mongod/database_status.go +++ b/collector/mongod/database_status.go @@ -1,8 +1,8 @@ package collector_mongod import ( - "github.com/prometheus/common/log" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/log" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" ) @@ -40,7 +40,6 @@ var ( }, []string{"db"}) ) - // DatabaseStatList contains stats from all databases type DatabaseStatList struct { Members []DatabaseStatus @@ -59,7 +58,7 @@ type DatabaseStatus struct { // Export exports database stats to prometheus func (dbStatList *DatabaseStatList) Export(ch chan<- prometheus.Metric) { for _, member := range dbStatList.Members { - ls := prometheus.Labels{ "db": member.Name } + ls := prometheus.Labels{"db": member.Name} indexSize.With(ls).Set(float64(member.IndexSize)) dataSize.With(ls).Set(float64(member.DataSize)) collectionsTotal.With(ls).Set(float64(member.Collections)) @@ -92,14 +91,16 @@ func GetDatabaseStatList(session *mgo.Session) *DatabaseStatList { return nil } for _, db := range database_names { - if db == "admin" || db == "test" || db == "local" { continue } + if db == "admin" || db == "test" || db == "local" { + continue + } dbStatus := DatabaseStatus{} err := session.DB(db).Run(bson.D{{"dbStats", 1}, {"scale", 1}}, &dbStatus) if err != nil { log.Error("Failed to get database status.") return nil } - dbStatList.Members = append(dbStatList.Members,dbStatus) + dbStatList.Members = append(dbStatList.Members, dbStatus) } return dbStatList diff --git a/collector/mongos/collections_status.go b/collector/mongos/collections_status.go index c6ffb2e91..87744e88f 100644 --- a/collector/mongos/collections_status.go +++ b/collector/mongos/collections_status.go @@ -1,8 +1,8 @@ package collector_mongos import ( - "github.com/prometheus/common/log" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/log" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" ) @@ -46,7 +46,6 @@ var ( }, []string{"db", "coll"}) ) - // CollectionStatList contains stats from all collections type CollectionStatList struct { Members []CollectionStatus @@ -54,21 +53,21 @@ type CollectionStatList struct { // CollectionStatus represents stats about a collection in database (mongod and raw from mongos) type CollectionStatus struct { - Database string + Database string Name string - Size int `bson:"size,omitempty"` - Count int `bson:"count,omitempty"` - AvgObjSize int `bson:"avgObjSize,omitempty"` - StorageSize int `bson:"storageSize,omitempty"` - Indexes int `bson:"indexSizes,omitempty"` - IndexesSize int `bson:"totalIndexSize,omitempty"` + Size int `bson:"size,omitempty"` + Count int `bson:"count,omitempty"` + AvgObjSize int `bson:"avgObjSize,omitempty"` + StorageSize int `bson:"storageSize,omitempty"` + Indexes int `bson:"indexSizes,omitempty"` + IndexesSize int `bson:"totalIndexSize,omitempty"` } // Export exports database stats to prometheus func (collStatList *CollectionStatList) Export(ch chan<- prometheus.Metric) { for _, member := range collStatList.Members { - ls := prometheus.Labels{ - "db": member.Database, + ls := prometheus.Labels{ + "db": member.Database, "coll": member.Name, } collectionSize.With(ls).Set(float64(member.Size)) @@ -105,7 +104,9 @@ func GetCollectionStatList(session *mgo.Session) *CollectionStatList { return nil } for _, db := range database_names { - if db == "admin" || db == "test" || db == "local" { continue } + if db == "admin" || db == "test" || db == "local" { + continue + } collection_names, err := session.DB(db).CollectionNames() if err != nil { log.Error("Failed to get collection names for db=" + db) @@ -120,9 +121,9 @@ func GetCollectionStatList(session *mgo.Session) *CollectionStatList { log.Error("Failed to get collection status.") return nil } - collectionStatList.Members = append(collectionStatList.Members,collStatus) + collectionStatList.Members = append(collectionStatList.Members, collStatus) } } return collectionStatList -} \ No newline at end of file +} diff --git a/collector/mongos/database_status.go b/collector/mongos/database_status.go index dd2e35089..eb31c7293 100644 --- a/collector/mongos/database_status.go +++ b/collector/mongos/database_status.go @@ -3,8 +3,8 @@ package collector_mongos import ( "strings" - "github.com/prometheus/common/log" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/log" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" ) @@ -42,7 +42,6 @@ var ( }, []string{"db", "shard"}) ) - // DatabaseStatList contains stats from all databases type DatabaseStatList struct { Members []DatabaseStatus @@ -69,8 +68,8 @@ func (dbStatList *DatabaseStatList) Export(ch chan<- prometheus.Metric) { for _, member := range dbStatList.Members { if len(member.Shards) > 0 { for shard, stats := range member.Shards { - ls := prometheus.Labels{ - "db": stats.Name, + ls := prometheus.Labels{ + "db": stats.Name, "shard": strings.Split(shard, "/")[0], } indexSize.With(ls).Set(float64(stats.IndexSize)) @@ -113,14 +112,16 @@ func GetDatabaseStatList(session *mgo.Session) *DatabaseStatList { return nil } for _, db := range database_names { - if db == "admin" || db == "test" || db == "local" { continue } + if db == "admin" || db == "test" || db == "local" { + continue + } dbStatus := DatabaseStatus{} err := session.DB(db).Run(bson.D{{"dbStats", 1}, {"scale", 1}}, &dbStatus) if err != nil { log.Error("Failed to get database status.") return nil } - dbStatList.Members = append(dbStatList.Members,dbStatus) + dbStatList.Members = append(dbStatList.Members, dbStatus) } return dbStatList From 73540ad7db149cac2d87a49f1787c69ed701f785 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 4 Jan 2018 16:40:25 +0300 Subject: [PATCH 09/17] Remove unused metrics. --- collector/mongod/global_lock.go | 6 ------ collector/mongod/replset_status.go | 6 ------ 2 files changed, 12 deletions(-) diff --git a/collector/mongod/global_lock.go b/collector/mongod/global_lock.go index 7934006c1..f30ead94c 100644 --- a/collector/mongod/global_lock.go +++ b/collector/mongod/global_lock.go @@ -31,12 +31,6 @@ var ( Name: "total", Help: "The value of totalTime represents the time, in microseconds, since the database last started and creation of the globalLock. This is roughly equivalent to total server uptime", }) - globalLockLockTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "global_lock", - Name: "lock_total", - Help: "The value of lockTime represents the time, in microseconds, since the database last started, that the globalLock has been held", - }) ) var ( globalLockCurrentQueue = prometheus.NewGaugeVec(prometheus.GaugeOpts{ diff --git a/collector/mongod/replset_status.go b/collector/mongod/replset_status.go index 677cb0da6..57ade8453 100644 --- a/collector/mongod/replset_status.go +++ b/collector/mongod/replset_status.go @@ -115,12 +115,6 @@ var ( Name: "member_config_version", Help: "The configVersion value is the replica set configuration version.", }, []string{"set", "name", "state"}) - memberOptime = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: Namespace, - Subsystem: subsystem, - Name: "member_optime", - Help: "Information regarding the last operation from the operation log that this member has applied.", - }, []string{"set", "name", "state"}) ) // ReplSetStatus keeps the data returned by the GetReplSetStatus method From 9a29c00c6772ba73d8e8fd4cd7070b260c08b046 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 4 Jan 2018 16:40:49 +0300 Subject: [PATCH 10/17] Simplify boolean comparisons. --- collector/mongos/sharding_status.go | 2 +- collector/mongos/sharding_topology.go | 2 +- shared/connection.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/collector/mongos/sharding_status.go b/collector/mongos/sharding_status.go index 2737b177e..b7daeb6f2 100644 --- a/collector/mongos/sharding_status.go +++ b/collector/mongos/sharding_status.go @@ -114,7 +114,7 @@ func IsBalancerEnabled(session *mgo.Session) float64 { if err != nil { return 1 } - if balancerConfig.Stopped == true { + if balancerConfig.Stopped { return 0 } return 1 diff --git a/collector/mongos/sharding_topology.go b/collector/mongos/sharding_topology.go index b458d762a..5f93865c7 100644 --- a/collector/mongos/sharding_topology.go +++ b/collector/mongos/sharding_topology.go @@ -132,7 +132,7 @@ func (status *ShardingTopoStats) Export(ch chan<- prometheus.Metric) { if status.Shards != nil { var drainingShards float64 = 0 for _, shard := range *status.Shards { - if shard.Draining == true { + if shard.Draining { drainingShards = drainingShards + 1 } } diff --git a/shared/connection.go b/shared/connection.go index ccb8fd96d..235f22311 100644 --- a/shared/connection.go +++ b/shared/connection.go @@ -85,7 +85,7 @@ func MongoSession(opts MongoSessionOpts) *mgo.Session { } func (opts MongoSessionOpts) configureDialInfoIfRequired(dialInfo *mgo.DialInfo) error { - if opts.TLSConnection == true { + if opts.TLSConnection { config := &tls.Config{ InsecureSkipVerify: !opts.TLSHostnameValidation, } From b912f8c749345b14b0e745aa2bed062c4a1df906 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 4 Jan 2018 17:01:39 +0300 Subject: [PATCH 11/17] =?UTF-8?q?PMM-1586=20Add=20=C2=ABup=C2=BB=20and=20?= =?UTF-8?q?=C2=ABscrape=5Ferrors=5Ftotal=C2=BB.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- collector/mongodb_collector.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/collector/mongodb_collector.go b/collector/mongodb_collector.go index cec3a4b72..a24d33120 100644 --- a/collector/mongodb_collector.go +++ b/collector/mongodb_collector.go @@ -60,8 +60,10 @@ type MongodbCollector struct { Opts MongodbCollectorOpts scrapesTotal prometheus.Counter + scrapeErrorsTotal prometheus.Counter lastScrapeError prometheus.Gauge lastScrapeDurationSeconds prometheus.Gauge + mongoUp prometheus.Gauge mongoSessLock sync.Mutex mongoSess *mgo.Session @@ -78,6 +80,12 @@ func NewMongodbCollector(opts MongodbCollectorOpts) *MongodbCollector { Name: "scrapes_total", Help: "Total number of times MongoDB was scraped for metrics.", }), + scrapeErrorsTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "exporter", + Name: "scrape_errors_total", + Help: "Total number of times an error occurred scraping a MongoDB.", + }), lastScrapeError: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: "exporter", @@ -90,6 +98,11 @@ func NewMongodbCollector(opts MongodbCollectorOpts) *MongodbCollector { Name: "last_scrape_duration_seconds", Help: "Duration of the last scrape of metrics from MongoDB.", }), + mongoUp: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "up", + Help: "Whether MongoDB is up.", + }), } return exporter @@ -168,6 +181,7 @@ func (exporter *MongodbCollector) scrape(ch chan<- prometheus.Metric) { if err == nil { exporter.lastScrapeError.Set(0) } else { + exporter.scrapeErrorsTotal.Inc() exporter.lastScrapeError.Set(1) } }(time.Now()) @@ -176,9 +190,11 @@ func (exporter *MongodbCollector) scrape(ch chan<- prometheus.Metric) { if mongoSess == nil { err = fmt.Errorf("Can't create mongo session to %s", exporter.Opts.URI) log.Error(err) + exporter.mongoUp.Set(0) return } defer mongoSess.Close() + exporter.mongoUp.Set(1) var serverVersion string serverVersion, err = shared.MongoSessionServerVersion(mongoSess) From 7da484443612c3b93a6ca51f81227882b5ecd462 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 4 Jan 2018 17:02:01 +0300 Subject: [PATCH 12/17] Update changelog. --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbef27928..d99174f30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## v0.4.0 (not released yet) +* New flags `-collect.database` and `-collect.collection` can be used to enable collection of database and collection + metrics. They are disabled by default. +* MongoDB connections are now kept between the scrapes. New flag `-mongodb.max-connections` (with the default value `1`) + controls the maximum number of established connections. +* Add standard metrics: + * `mongodb_scrape_errors_total` + * `mongodb_up` +* Some queries now contain [cursor comments](https://www.percona.com/blog/2017/06/21/tracing-mongodb-queries-to-code-with-cursor-comments/) + with source code locations. * Go vendoring switched to [dep](https://github.com/golang/dep). ## v0.3.1 (2017-09-08) From fbe1f0cfea2aae93b3b010cefab60983ada445a2 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 4 Jan 2018 17:17:51 +0300 Subject: [PATCH 13/17] =?UTF-8?q?PMM-1586=20Fix=20=C2=ABup=C2=BB=20metric?= =?UTF-8?q?=20to=20be=20accurate.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- collector/mongodb_collector.go | 5 ++++- shared/connection.go | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/collector/mongodb_collector.go b/collector/mongodb_collector.go index a24d33120..5d5918624 100644 --- a/collector/mongodb_collector.go +++ b/collector/mongodb_collector.go @@ -169,8 +169,10 @@ func (exporter *MongodbCollector) Collect(ch chan<- prometheus.Metric) { exporter.scrape(ch) exporter.scrapesTotal.Collect(ch) + exporter.scrapeErrorsTotal.Collect(ch) exporter.lastScrapeError.Collect(ch) exporter.lastScrapeDurationSeconds.Collect(ch) + exporter.mongoUp.Collect(ch) } func (exporter *MongodbCollector) scrape(ch chan<- prometheus.Metric) { @@ -194,14 +196,15 @@ func (exporter *MongodbCollector) scrape(ch chan<- prometheus.Metric) { return } defer mongoSess.Close() - exporter.mongoUp.Set(1) var serverVersion string serverVersion, err = shared.MongoSessionServerVersion(mongoSess) if err != nil { log.Errorf("Problem gathering the mongo server version: %s", err) + exporter.mongoUp.Set(0) return } + exporter.mongoUp.Set(1) var nodeType string nodeType, err = shared.MongoSessionNodeType(mongoSess) diff --git a/shared/connection.go b/shared/connection.go index 235f22311..3f2c2f375 100644 --- a/shared/connection.go +++ b/shared/connection.go @@ -62,8 +62,10 @@ func MongoSession(opts MongoSessionOpts) *mgo.Session { return nil } - dialInfo.Direct = true // Force direct connection + // connect directly, fail faster, do not retry - for faster responses and accurate metrics, including mongoUp + dialInfo.Direct = true dialInfo.Timeout = dialMongodbTimeout + dialInfo.FailFast = true err = opts.configureDialInfoIfRequired(dialInfo) if err != nil { From 0cf5091241719b1db822d688c9072b12e12e8e4a Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 4 Jan 2018 19:19:38 +0300 Subject: [PATCH 14/17] Test with upcoming version of PSMDB 3.6. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 84d6193d2..43b7234d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ env: - MONGODB_IMAGE=mongo:3.4 - MONGODB_IMAGE=mongo:3.6 - MONGODB_IMAGE=percona/percona-server-mongodb:3.4 + - MONGODB_IMAGE=perconalab/percona-server-mongodb:3.6 services: - docker From eaf2f65517ff4e8c208198e37c5e8553ae5bda54 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 17 Jan 2018 12:01:22 +0300 Subject: [PATCH 15/17] PMM-1764 Do not skip standard databases. --- collector/mongod/collections_status.go | 3 --- collector/mongod/database_status.go | 3 --- collector/mongos/collections_status.go | 3 --- collector/mongos/database_status.go | 3 --- 4 files changed, 12 deletions(-) diff --git a/collector/mongod/collections_status.go b/collector/mongod/collections_status.go index ac4b3f30a..eb650d832 100644 --- a/collector/mongod/collections_status.go +++ b/collector/mongod/collections_status.go @@ -104,9 +104,6 @@ func GetCollectionStatList(session *mgo.Session) *CollectionStatList { return nil } for _, db := range database_names { - if db == "admin" || db == "test" || db == "local" { - continue - } collection_names, err := session.DB(db).CollectionNames() if err != nil { log.Error("Failed to get collection names for db=" + db) diff --git a/collector/mongod/database_status.go b/collector/mongod/database_status.go index 44d22e9cc..db1d54a18 100644 --- a/collector/mongod/database_status.go +++ b/collector/mongod/database_status.go @@ -91,9 +91,6 @@ func GetDatabaseStatList(session *mgo.Session) *DatabaseStatList { return nil } for _, db := range database_names { - if db == "admin" || db == "test" || db == "local" { - continue - } dbStatus := DatabaseStatus{} err := session.DB(db).Run(bson.D{{"dbStats", 1}, {"scale", 1}}, &dbStatus) if err != nil { diff --git a/collector/mongos/collections_status.go b/collector/mongos/collections_status.go index 87744e88f..49789f5d5 100644 --- a/collector/mongos/collections_status.go +++ b/collector/mongos/collections_status.go @@ -104,9 +104,6 @@ func GetCollectionStatList(session *mgo.Session) *CollectionStatList { return nil } for _, db := range database_names { - if db == "admin" || db == "test" || db == "local" { - continue - } collection_names, err := session.DB(db).CollectionNames() if err != nil { log.Error("Failed to get collection names for db=" + db) diff --git a/collector/mongos/database_status.go b/collector/mongos/database_status.go index eb31c7293..6cc74b5db 100644 --- a/collector/mongos/database_status.go +++ b/collector/mongos/database_status.go @@ -112,9 +112,6 @@ func GetDatabaseStatList(session *mgo.Session) *DatabaseStatList { return nil } for _, db := range database_names { - if db == "admin" || db == "test" || db == "local" { - continue - } dbStatus := DatabaseStatus{} err := session.DB(db).Run(bson.D{{"dbStats", 1}, {"scale", 1}}, &dbStatus) if err != nil { From a804b8e657bf482cb5afc2bdfda2a690bfb0d611 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 17 Jan 2018 12:01:34 +0300 Subject: [PATCH 16/17] PMM-1764 Add comment. --- shared/utils.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shared/utils.go b/shared/utils.go index 1316ce278..e3d87dd28 100644 --- a/shared/utils.go +++ b/shared/utils.go @@ -42,6 +42,8 @@ func LoadKeyPairFrom(pemFile string, privateKeyPemFile string) (tls.Certificate, return tls.LoadX509KeyPair(pemFile, targetPrivateKeyPemFile) } +// AddCodeCommentToQuery adds location of the caller in the source code (e.g. "oplog_status.go:91") +// to the given query as a comment. func AddCodeCommentToQuery(query *mgo.Query) *mgo.Query { _, fileName, lineNum, ok := runtime.Caller(1) if !ok { From f58598a052a1bce07f4a6dccb0266041c3f8aaab Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 17 Jan 2018 12:03:48 +0300 Subject: [PATCH 17/17] PMM-1764 Remove retries. --- collector/mongod/oplog_status.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/collector/mongod/oplog_status.go b/collector/mongod/oplog_status.go index c6bacaf07..68a15dc79 100644 --- a/collector/mongod/oplog_status.go +++ b/collector/mongod/oplog_status.go @@ -83,18 +83,8 @@ func getOplogTailOrHeadTimestamp(session *mgo.Session, returnHead bool) (float64 sortCond = "-$natural" } - // retry once if there is an error - var err error - var tries int64 = 0 - for tries < 2 { - findQuery := session.DB(oplogDb).C(oplogCollection).Find(nil).Sort(sortCond).Limit(1) - err = shared.AddCodeCommentToQuery(findQuery).One(&result) - if err == nil { - break - } - tries += 1 - } - + findQuery := session.DB(oplogDb).C(oplogCollection).Find(nil).Sort(sortCond).Limit(1) + err := shared.AddCodeCommentToQuery(findQuery).One(&result) return BsonMongoTimestampToUnix(result.Timestamp), err }