From 1dd99df6d67f2a4960d66b4d4ae8911a7841f67d Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Mon, 11 Apr 2016 11:34:05 +0200 Subject: [PATCH 1/6] added upstream's improved version of my direct-connection fix for replsets, removed broken go tests to match upstream, pulled in a few bugfixes --- collector/mongod/asserts_test.go | 17 - collector/mongod/background_flushing_test.go | 16 - collector/mongod/connections_test.go | 15 - collector/mongod/cursors_test.go | 11 - collector/mongod/durability_test.go | 13 - collector/mongod/extra_info_test.go | 11 - collector/mongod/global_lock_test.go | 14 - collector/mongod/index_counters_test.go | 11 - collector/mongod/locks_test.go | 16 - collector/mongod/memory_test.go | 11 - collector/mongod/metrics.go | 2 +- collector/mongod/metrics_test.go | 33 -- collector/mongod/network_test.go | 11 - collector/mongod/op_counters_test.go | 11 - collector/mongod/server_status_test.go | 80 ---- collector/mongodb_collector.go | 188 +++------ collector/mongos/asserts_test.go | 17 - collector/mongos/connections_test.go | 15 - collector/mongos/cursors_test.go | 11 - collector/mongos/extra_info_test.go | 11 - collector/mongos/memory_test.go | 11 - collector/mongos/metrics.go | 415 ++++++++++++++++++- collector/mongos/metrics_test.go | 15 - collector/mongos/network_test.go | 11 - collector/mongos/op_counters_test.go | 11 - collector/mongos/server_status_test.go | 60 --- collector/shared/version.go | 42 -- mongodb_exporter.go | 19 +- shared/connection.go | 65 +++ 29 files changed, 543 insertions(+), 620 deletions(-) delete mode 100644 collector/mongod/asserts_test.go delete mode 100644 collector/mongod/background_flushing_test.go delete mode 100644 collector/mongod/connections_test.go delete mode 100644 collector/mongod/cursors_test.go delete mode 100644 collector/mongod/durability_test.go delete mode 100644 collector/mongod/extra_info_test.go delete mode 100644 collector/mongod/global_lock_test.go delete mode 100644 collector/mongod/index_counters_test.go delete mode 100644 collector/mongod/locks_test.go delete mode 100644 collector/mongod/memory_test.go delete mode 100644 collector/mongod/metrics_test.go delete mode 100644 collector/mongod/network_test.go delete mode 100644 collector/mongod/op_counters_test.go delete mode 100644 collector/mongod/server_status_test.go delete mode 100644 collector/mongos/asserts_test.go delete mode 100644 collector/mongos/connections_test.go delete mode 100644 collector/mongos/cursors_test.go delete mode 100644 collector/mongos/extra_info_test.go delete mode 100644 collector/mongos/memory_test.go delete mode 100644 collector/mongos/metrics_test.go delete mode 100644 collector/mongos/network_test.go delete mode 100644 collector/mongos/op_counters_test.go delete mode 100644 collector/mongos/server_status_test.go delete mode 100644 collector/shared/version.go create mode 100644 shared/connection.go diff --git a/collector/mongod/asserts_test.go b/collector/mongod/asserts_test.go deleted file mode 100644 index 18f330c92..000000000 --- a/collector/mongod/asserts_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_AssertsCollectData(t *testing.T) { - asserts := &AssertsStats{ - Regular: 1, - Warning: 2, - Msg: 3, - User: 4, - Rollovers: 5, - } - - asserts.Export() -} diff --git a/collector/mongod/background_flushing_test.go b/collector/mongod/background_flushing_test.go deleted file mode 100644 index 67f27c2d8..000000000 --- a/collector/mongod/background_flushing_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_BackgroundFlushingCollectData(t *testing.T) { - stats := &FlushStats{ - Flushes: 1, - TotalMs: 2, - AverageMs: 3, - LastMs: 4, - } - - stats.Export() -} diff --git a/collector/mongod/connections_test.go b/collector/mongod/connections_test.go deleted file mode 100644 index 9bb231683..000000000 --- a/collector/mongod/connections_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_ConnectionsCollectData(t *testing.T) { - stats := &ConnectionStats{ - Current: 1, - Available: 2, - TotalCreated: 3, - } - - stats.Export() -} diff --git a/collector/mongod/cursors_test.go b/collector/mongod/cursors_test.go deleted file mode 100644 index 6f0c06148..000000000 --- a/collector/mongod/cursors_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_CursorsCollectData(t *testing.T) { - cursors := &Cursors{} - - cursors.Export() -} diff --git a/collector/mongod/durability_test.go b/collector/mongod/durability_test.go deleted file mode 100644 index fb92e0b36..000000000 --- a/collector/mongod/durability_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_DurabilityCollectData(t *testing.T) { - stats := &DurStats{ - TimeMs: DurTiming{}, - } - - stats.Export() -} diff --git a/collector/mongod/extra_info_test.go b/collector/mongod/extra_info_test.go deleted file mode 100644 index d8bcbf335..000000000 --- a/collector/mongod/extra_info_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_ExtraInfoCollectData(t *testing.T) { - stats := &ExtraInfo{} - - stats.Export() -} diff --git a/collector/mongod/global_lock_test.go b/collector/mongod/global_lock_test.go deleted file mode 100644 index d5533307b..000000000 --- a/collector/mongod/global_lock_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_GlobalLockCollectData(t *testing.T) { - stats := &GlobalLockStats{ - CurrentQueue: &QueueStats{}, - ActiveClients: &ClientStats{}, - } - - stats.Export() -} diff --git a/collector/mongod/index_counters_test.go b/collector/mongod/index_counters_test.go deleted file mode 100644 index a14567737..000000000 --- a/collector/mongod/index_counters_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_IndexCountersCollectData(t *testing.T) { - stats := &IndexCounterStats{} - - stats.Export() -} diff --git a/collector/mongod/locks_test.go b/collector/mongod/locks_test.go deleted file mode 100644 index 4464671aa..000000000 --- a/collector/mongod/locks_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_LocksCollectData(t *testing.T) { - stats := &LockStatsMap{ - ".": LockStats{ - TimeLockedMicros: ReadWriteLockTimes{}, - TimeAcquiringMicros: ReadWriteLockTimes{}, - }, - } - - stats.Export() -} diff --git a/collector/mongod/memory_test.go b/collector/mongod/memory_test.go deleted file mode 100644 index ffaf0a119..000000000 --- a/collector/mongod/memory_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_MemoryCollectData(t *testing.T) { - stats := &MemStats{} - - stats.Export() -} diff --git a/collector/mongod/metrics.go b/collector/mongod/metrics.go index 20587c50f..893521d0c 100644 --- a/collector/mongod/metrics.go +++ b/collector/mongod/metrics.go @@ -397,7 +397,7 @@ type CursorStats struct { // Export exports the cursor stats. func (cursorStats *CursorStats) Export(ch chan<- prometheus.Metric) { metricsCursorTimedOutTotal.Set(cursorStats.TimedOut) - metricsCursorOpen.WithLabelValues("noTimeout").Set(cursorStats.Open.NoTimeout) + metricsCursorOpen.WithLabelValues("timed_out").Set(cursorStats.Open.NoTimeout) metricsCursorOpen.WithLabelValues("pinned").Set(cursorStats.Open.Pinned) metricsCursorOpen.WithLabelValues("total").Set(cursorStats.Open.Total) } diff --git a/collector/mongod/metrics_test.go b/collector/mongod/metrics_test.go deleted file mode 100644 index 921971234..000000000 --- a/collector/mongod/metrics_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_MetricsCollectData(t *testing.T) { - stats := &MetricsStats{ - Document: &DocumentStats{}, - GetLastError: &GetLastErrorStats{ - Wtime: &BenchmarkStats{}, - }, - Operation: &OperationStats{}, - QueryExecutor: &QueryExecutorStats{}, - Record: &RecordStats{}, - Repl: &ReplStats{ - Apply: &ApplyStats{ - Batches: &BenchmarkStats{}, - }, - Buffer: &BufferStats{}, - Network: &MetricsNetworkStats{ - GetMores: &BenchmarkStats{}, - }, - PreloadStats: &PreloadStats{ - Docs: &BenchmarkStats{}, - Indexes: &BenchmarkStats{}, - }, - }, - Storage: &StorageStats{}, - } - - stats.Export() -} diff --git a/collector/mongod/network_test.go b/collector/mongod/network_test.go deleted file mode 100644 index a98bbd2a8..000000000 --- a/collector/mongod/network_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_NetworkCollectData(t *testing.T) { - stats := &NetworkStats{} - - stats.Export() -} diff --git a/collector/mongod/op_counters_test.go b/collector/mongod/op_counters_test.go deleted file mode 100644 index 53f6bbd11..000000000 --- a/collector/mongod/op_counters_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package collector_mongod - -import ( - "testing" -) - -func Test_OpCountersCollectData(t *testing.T) { - stats := &OpcountersStats{} - - stats.Export() -} diff --git a/collector/mongod/server_status_test.go b/collector/mongod/server_status_test.go deleted file mode 100644 index a4d50de62..000000000 --- a/collector/mongod/server_status_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package collector_mongod - -import ( - "gopkg.in/mgo.v2/bson" - "testing" -) - -func Test_ServerStatusCollectData(t *testing.T) { - data := LoadFixture("server_status.bson") - serverStatus := &ServerStatus{} - loadServerStatusFromBson(data, serverStatus) - - serverStatus.Export() -} - -func Test_ParserServerStatus(t *testing.T) { - data := LoadFixture("server_status.bson") - - serverStatus := &ServerStatus{} - loadServerStatusFromBson(data, serverStatus) - - if serverStatus.Asserts == nil { - t.Error("Asserts group was not loaded") - } - - if serverStatus.Dur == nil { - t.Error("Dur group was not loaded") - } - - if serverStatus.BackgroundFlushing == nil { - t.Error("BackgroundFlushing group was not loaded") - } - - if serverStatus.Connections == nil { - t.Error("Connections group was not loaded") - } - - if serverStatus.ExtraInfo == nil { - t.Error("ExtraInfo group was not loaded") - } - - if serverStatus.GlobalLock == nil { - t.Error("GlobalLock group was not loaded") - } - - if serverStatus.Network == nil { - t.Error("Network group was not loaded") - } - - if serverStatus.Opcounters == nil { - t.Error("Opcounters group was not loaded") - } - - if serverStatus.OpcountersRepl == nil { - t.Error("OpcountersRepl group was not loaded") - } - - if serverStatus.Mem == nil { - t.Error("Mem group was not loaded") - } - - if serverStatus.Connections == nil { - t.Error("Connections group was not loaded") - } - - if serverStatus.Locks == nil { - t.Error("Locks group was not loaded") - } - - if serverStatus.Metrics.Document.Deleted != 45726 { - t.Error("Metrics group was not loaded correctly") - } -} - -func loadServerStatusFromBson(data []byte, status *ServerStatus) { - err := bson.Unmarshal(data, status) - if err != nil { - panic(err) - } -} diff --git a/collector/mongodb_collector.go b/collector/mongodb_collector.go index 044094636..391d7d97e 100644 --- a/collector/mongodb_collector.go +++ b/collector/mongodb_collector.go @@ -1,16 +1,12 @@ package collector import ( - //"github.com/dcu/mongodb_exporter/shared" + "github.com/dcu/mongodb_exporter/shared" "github.com/dcu/mongodb_exporter/collector/mongod" "github.com/dcu/mongodb_exporter/collector/mongos" - "github.com/dcu/mongodb_exporter/collector/shared" "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" "gopkg.in/mgo.v2" - "fmt" - "os" - "regexp" ) var ( @@ -40,146 +36,80 @@ func NewMongodbCollector(opts MongodbCollectorOpts) *MongodbCollector { // Describe describes all mongodb's metrics. func (exporter *MongodbCollector) Describe(ch chan<- *prometheus.Desc) { glog.Info("Describing groups") - session, err := connectMongo(exporter.Opts.URI) - if err != nil{ - return - } - serverStatus := collector_mongos.GetServerStatus(session) - if serverStatus != nil { - serverStatus.Describe(ch) - } + session := shared.MongoSession(exporter.Opts.URI) defer session.Close() -} - -func connectMongo(uri string)(*mgo.Session, error) { - r, _ := regexp.Compile("connect=direct") - if r.MatchString(uri) != true { - r, _ := regexp.Compile("\\?") - if r.MatchString(uri) == true { - uri = uri + "&connect=direct" - } else { - uri = uri + "?connect=direct" - } - } - session, err := mgo.Dial(uri) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - glog.Errorf("Cannot connect to server using url: %s", uri) - return nil,err - } - - session.SetMode(mgo.Eventual, true) - session.SetSocketTimeout(0) - - err = nil - return session,err -} - -// GetNodeType checks if the connected Session is a mongos, standalone, or replset, -// by looking at the result of calling isMaster. -func GetNodeType(session *mgo.Session)(string, error) { - masterDoc := struct { - SetName interface{} `bson:"setName"` - Hosts interface{} `bson:"hosts"` - Msg string `bson:"msg"` - }{} - err := session.Run("isMaster", &masterDoc) - if err != nil { - glog.Info("Got unknown node type\n") - return "unknown", err + if session != nil { + serverStatus := collector_mongos.GetServerStatus(session) + if serverStatus != nil { + serverStatus.Describe(ch) + } } - - if masterDoc.SetName != nil || masterDoc.Hosts != nil { - glog.Info("Got replset node type") - return "replset", nil - } else if masterDoc.Msg == "isdbgrid" { - glog.Info("Got mongos node type\n") - // isdbgrid is always the msg value when calling isMaster on a mongos - // see http://docs.mongodb.org/manual/core/sharded-cluster-query-router/ - return "mongos", nil - } - glog.Info("defaulted to mongod node type\n") - return "mongod", nil } // Collect collects all mongodb's metrics. func (exporter *MongodbCollector) Collect(ch chan<- prometheus.Metric) { - glog.Info("Collecting Server Status from: ", exporter.Opts.URI) - session, err := connectMongo(exporter.Opts.URI) - if err != nil{ - glog.Error(fmt.Printf("We failed to connect to mongo with error of %s\n", err)) - } - - version := collector_shared.GetServerVersion(session) - glog.Info("Connected to: ", exporter.Opts.URI, ", server version: ", version) - - nodeType,err := GetNodeType(session) - if err != nil{ - glog.Error("We run had a node type error of %s\n", err) - } - //glog.Info("Passed nodeType with %s", nodeType) - switch { - case nodeType == "mongos": - serverStatus := collector_mongos.GetServerStatus(session) - if serverStatus != nil { - serverStatus.Export(ch) - } - balancerStatus := collector_mongos.GetBalancerStatus(session) - if balancerStatus != nil { - balancerStatus.Export(ch) - } - balancerTopoStatus := collector_mongos.GetBalancerTopoStatus(session) - if balancerTopoStatus != nil { - balancerTopoStatus.Export(ch) - } - balancerChangelogStatus := collector_mongos.GetBalancerChangelogStatus(session) - if balancerChangelogStatus != nil { - balancerChangelogStatus.Export(ch) - } - case nodeType == "mongod": - serverStatus := collector_mongod.GetServerStatus(session) - if serverStatus != nil { - serverStatus.Export(ch) - } - case nodeType == "replset": - serverStatus := collector_mongod.GetServerStatus(session) - if serverStatus != nil { - serverStatus.Export(ch) - } - oplogStatus := collector_mongod.GetOplogStatus(session) - if oplogStatus != nil { - oplogStatus.Export(ch) - } - replSetStatus := collector_mongod.GetReplSetStatus(session) - if replSetStatus != nil { - replSetStatus.Export(ch) - } - default: - glog.Info("No process for current node type no metrics printing!\n") - } - session.Close() - //exporter.collectMongodServerStatus(ch) - //exporter.collectMongosServerStatus(ch) + mongoSess := shared.MongoSession(exporter.Opts.URI) + defer mongoSess.Close() + if mongoSess != nil { + serverVersion, err := shared.MongoSessionServerVersion(mongoSess) + if err != nil { + glog.Errorf("Problem gathering the mongo server version: %s", err) + } + + nodeType, err := shared.MongoSessionNodeType(mongoSess) + if err != nil { + glog.Errorf("Problem gathering the mongo node type: %s", err) + } + + glog.Infof("Connected to: %s (node type: %s, server version: %s)", exporter.Opts.URI, nodeType, serverVersion) + switch { + case nodeType == "mongos": + exporter.collectMongos(mongoSess, ch) + case nodeType == "mongod": + exporter.collectMongod(mongoSess, ch) + case nodeType == "replset": + exporter.collectMongodReplSet(mongoSess, ch) + default: + glog.Infof("Unrecognized node type %s!", nodeType) + } + } } -/** -func (exporter *MongodbCollector) collectMongodServerStatus(ch chan<- prometheus.Metric) *collector_mongod.ServerStatus { - serverStatus := collector_mongod.GetServerStatus(exporter.Opts.URI) +func (exporter *MongodbCollector) collectMongos(session *mgo.Session, ch chan<- prometheus.Metric) { + glog.Info("Collecting Server Status") + serverStatus := collector_mongos.GetServerStatus(session) if serverStatus != nil { serverStatus.Export(ch) } - return serverStatus + glog.Info("Collecting Mongos Balancer Status") + balancerStatus := collector_mongos.GetBalancerStatus(session) + if balancerStatus != nil { + balancerStatus.Export(ch) + } } -func (exporter *MongodbCollector) collectMongosServerStatus(ch chan<- prometheus.Metric) *collector_mongos.ServerStatus { - serverStatus := collector_mongos.GetServerStatus(exporter.Opts.URI) - +func (exporter *MongodbCollector) collectMongod(session *mgo.Session, ch chan<- prometheus.Metric) { + glog.Info("Collecting Server Status") + serverStatus := collector_mongod.GetServerStatus(session) if serverStatus != nil { serverStatus.Export(ch) } +} + +func (exporter *MongodbCollector) collectMongodReplSet(session *mgo.Session, ch chan<- prometheus.Metric) { + exporter.collectMongod(session, ch) - return serverStatus + glog.Info("Collecting Replset Status") + oplogStatus := collector_mongod.GetOplogStatus(session) + if oplogStatus != nil { + oplogStatus.Export(ch) + } + + glog.Info("Collecting Replset Oplog Status") + replSetStatus := collector_mongod.GetReplSetStatus(session) + if replSetStatus != nil { + replSetStatus.Export(ch) + } } -**/ + diff --git a/collector/mongos/asserts_test.go b/collector/mongos/asserts_test.go deleted file mode 100644 index 43372624f..000000000 --- a/collector/mongos/asserts_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package collector_mongos - -import ( - "testing" -) - -func Test_AssertsCollectData(t *testing.T) { - asserts := &AssertsStats{ - Regular: 1, - Warning: 2, - Msg: 3, - User: 4, - Rollovers: 5, - } - - asserts.Export() -} diff --git a/collector/mongos/connections_test.go b/collector/mongos/connections_test.go deleted file mode 100644 index 572c4e2d5..000000000 --- a/collector/mongos/connections_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package collector_mongos - -import ( - "testing" -) - -func Test_ConnectionsCollectData(t *testing.T) { - stats := &ConnectionStats{ - Current: 1, - Available: 2, - TotalCreated: 3, - } - - stats.Export() -} diff --git a/collector/mongos/cursors_test.go b/collector/mongos/cursors_test.go deleted file mode 100644 index ae02e09c9..000000000 --- a/collector/mongos/cursors_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package collector_mongos - -import ( - "testing" -) - -func Test_CursorsCollectData(t *testing.T) { - cursors := &Cursors{} - - cursors.Export() -} diff --git a/collector/mongos/extra_info_test.go b/collector/mongos/extra_info_test.go deleted file mode 100644 index 70ceaf170..000000000 --- a/collector/mongos/extra_info_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package collector_mongos - -import ( - "testing" -) - -func Test_ExtraInfoCollectData(t *testing.T) { - stats := &ExtraInfo{} - - stats.Export() -} diff --git a/collector/mongos/memory_test.go b/collector/mongos/memory_test.go deleted file mode 100644 index e0f44309f..000000000 --- a/collector/mongos/memory_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package collector_mongos - -import ( - "testing" -) - -func Test_MemoryCollectData(t *testing.T) { - stats := &MemStats{} - - stats.Export() -} diff --git a/collector/mongos/metrics.go b/collector/mongos/metrics.go index 9ba1224d1..893521d0c 100644 --- a/collector/mongos/metrics.go +++ b/collector/mongos/metrics.go @@ -1,4 +1,4 @@ -package collector_mongos +package collector_mongod import ( "github.com/prometheus/client_golang/prometheus" @@ -19,6 +19,13 @@ var ( Help: "The open is an embedded document that contains data regarding open cursors", }, []string{"state"}) ) +var ( + metricsDocumentTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: Namespace, + Name: "metrics_document_total", + Help: "The document holds a document of that reflect document access and modification patterns and data use. Compare these values to the data in the opcounters document, which track total number of operations", + }, []string{"state"}) +) var ( metricsGetLastErrorWtimeNumTotal = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: Namespace, @@ -41,6 +48,191 @@ var ( Help: "wtimeouts reports the number of times that write concern operations have timed out as a result of the wtimeout threshold to getLastError.", }) ) +var ( + metricsOperationTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: Namespace, + Name: "metrics_operation_total", + Help: "operation is a sub-document that holds counters for several types of update and query operations that MongoDB handles using special operation types", + }, []string{"type"}) +) +var ( + metricsQueryExecutorTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: Namespace, + Name: "metrics_query_executor_total", + Help: "queryExecutor is a document that reports data from the query execution system", + }, []string{"state"}) +) +var ( + metricsRecordMovesTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_record", + Name: "moves_total", + Help: "moves reports the total number of times documents move within the on-disk representation of the MongoDB data set. Documents move as a result of operations that increase the size of the document beyond their allocated record size", + }) +) +var ( + metricsReplApplyBatchesNumTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_apply_batches", + Name: "num_total", + Help: "num reports the total number of batches applied across all databases", + }) + metricsReplApplyBatchesTotalMilliseconds = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_apply_batches", + Name: "total_milliseconds", + Help: "total_millis reports the total amount of time the mongod has spent applying operations from the oplog", + }) +) +var ( + metricsReplApplyOpsTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_apply", + Name: "ops_total", + Help: "ops reports the total number of oplog operations applied", + }) +) +var ( + metricsReplBufferCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_buffer", + Name: "count", + Help: "count reports the current number of operations in the oplog buffer", + }) + metricsReplBufferMaxSizeBytes = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_buffer", + Name: "max_size_bytes", + Help: "maxSizeBytes reports the maximum size of the buffer. This value is a constant setting in the mongod, and is not configurable", + }) + metricsReplBufferSizeBytes = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_buffer", + Name: "size_bytes", + Help: "sizeBytes reports the current size of the contents of the oplog buffer", + }) +) +var ( + metricsReplNetworkGetmoresNumTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_network_getmores", + Name: "num_total", + Help: "num reports the total number of getmore operations, which are operations that request an additional set of operations from the replication sync source.", + }) + metricsReplNetworkGetmoresTotalMilliseconds = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_network_getmores", + Name: "total_milliseconds", + Help: "total_millis reports the total amount of time required to collect data from getmore operations", + }) +) +var ( + metricsReplNetworkBytesTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_network", + Name: "bytes_total", + Help: "bytes reports the total amount of data read from the replication sync source", + }) + metricsReplNetworkOpsTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_network", + Name: "ops_total", + Help: "ops reports the total number of operations read from the replication source.", + }) + metricsReplNetworkReadersCreatedTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_network", + Name: "readers_created_total", + Help: "readersCreated reports the total number of oplog query processes created. MongoDB will create a new oplog query any time an error occurs in the connection, including a timeout, or a network operation. Furthermore, readersCreated will increment every time MongoDB selects a new source fore replication.", + }) +) +var ( + metricsReplOplogInsertNumTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_oplog_insert", + Name: "num_total", + Help: "num reports the total number of items inserted into the oplog.", + }) + metricsReplOplogInsertTotalMilliseconds = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_oplog_insert", + Name: "total_milliseconds", + Help: "total_millis reports the total amount of time spent for the mongod to insert data into the oplog.", + }) +) +var ( + metricsReplOplogInsertBytesTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_oplog", + Name: "insert_bytes_total", + Help: "insertBytes the total size of documents inserted into the oplog.", + }) +) +var ( + metricsReplPreloadDocsNumTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_preload_docs", + Name: "num_total", + Help: "num reports the total number of documents loaded during the pre-fetch stage of replication", + }) + metricsReplPreloadDocsTotalMilliseconds = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_preload_docs", + Name: "total_milliseconds", + Help: "total_millis reports the total amount of time spent loading documents as part of the pre-fetch stage of replication", + }) +) +var ( + metricsReplPreloadIndexesNumTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_preload_indexes", + Name: "num_total", + Help: "num reports the total number of index entries loaded by members before updating documents as part of the pre-fetch stage of replication", + }) + metricsReplPreloadIndexesTotalMilliseconds = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_repl_preload_indexes", + Name: "total_milliseconds", + Help: "total_millis reports the total amount of time spent loading index entries as part of the pre-fetch stage of replication", + }) +) +var ( + metricsStorageFreelistSearchTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: Namespace, + Name: "metrics_storage_freelist_search_total", + Help: "metrics about searching records in the database.", + }, []string{"type"}) +) +var ( + metricsTTLDeletedDocumentsTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_ttl", + Name: "deleted_documents_total", + Help: "deletedDocuments reports the total number of documents deleted from collections with a ttl index.", + }) + metricsTTLPassesTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "metrics_ttl", + Name: "passes_total", + Help: "passes reports the number of times the background process removes documents from collections with a ttl index", + }) +) + +// DocumentStats are the stats associated to a document. +type DocumentStats struct { + Deleted float64 `bson:"deleted"` + Inserted float64 `bson:"inserted"` + Returned float64 `bson:"returned"` + Updated float64 `bson:"updated"` +} + +// Export exposes the document stats to be consumed by the prometheus server. +func (documentStats *DocumentStats) Export(ch chan<- prometheus.Metric) { + metricsDocumentTotal.WithLabelValues("deleted").Set(documentStats.Deleted) + metricsDocumentTotal.WithLabelValues("inserted").Set(documentStats.Inserted) + metricsDocumentTotal.WithLabelValues("returned").Set(documentStats.Returned) + metricsDocumentTotal.WithLabelValues("updated").Set(documentStats.Updated) +} // BenchmarkStats is bechmark info about an operation. type BenchmarkStats struct { @@ -62,54 +254,255 @@ func (getLastErrorStats *GetLastErrorStats) Export(ch chan<- prometheus.Metric) metricsGetLastErrorWtimeoutsTotal.Set(getLastErrorStats.Wtimeouts) } +// OperationStats are the stats for some kind of operations. +type OperationStats struct { + Fastmod float64 `bson:"fastmod"` + Idhack float64 `bson:"idhack"` + ScanAndOrder float64 `bson:"scanAndOrder"` +} + +// Export exports the operation stats. +func (operationStats *OperationStats) Export(ch chan<- prometheus.Metric) { + metricsOperationTotal.WithLabelValues("fastmod").Set(operationStats.Fastmod) + metricsOperationTotal.WithLabelValues("idhack").Set(operationStats.Idhack) + metricsOperationTotal.WithLabelValues("scan_and_order").Set(operationStats.ScanAndOrder) +} + +// QueryExecutorStats are the stats associated with a query execution. +type QueryExecutorStats struct { + Scanned float64 `bson:"scanned"` + ScannedObjects float64 `bson:"scannedObjects"` +} + +// Export exports the query executor stats. +func (queryExecutorStats *QueryExecutorStats) Export(ch chan<- prometheus.Metric) { + metricsQueryExecutorTotal.WithLabelValues("scanned").Set(queryExecutorStats.Scanned) + metricsQueryExecutorTotal.WithLabelValues("scanned_objects").Set(queryExecutorStats.ScannedObjects) +} + +// RecordStats are stats associated with a record. +type RecordStats struct { + Moves float64 `bson:"moves"` +} + +// Export exposes the record stats. +func (recordStats *RecordStats) Export(ch chan<- prometheus.Metric) { + metricsRecordMovesTotal.Set(recordStats.Moves) +} + +// ApplyStats are the stats associated with the apply operation. +type ApplyStats struct { + Batches *BenchmarkStats `bson:"batches"` + Ops float64 `bson:"ops"` +} + +// Export exports the apply stats +func (applyStats *ApplyStats) Export(ch chan<- prometheus.Metric) { + metricsReplApplyOpsTotal.Set(applyStats.Ops) + + metricsReplApplyBatchesNumTotal.Set(applyStats.Batches.Num) + metricsReplApplyBatchesTotalMilliseconds.Set(applyStats.Batches.TotalMillis) +} + +// BufferStats are the stats associated with the buffer +type BufferStats struct { + Count float64 `bson:"count"` + MaxSizeBytes float64 `bson:"maxSizeBytes"` + SizeBytes float64 `bson:"sizeBytes"` +} + +// Export exports the buffer stats. +func (bufferStats *BufferStats) Export(ch chan<- prometheus.Metric) { + metricsReplBufferCount.Set(bufferStats.Count) + metricsReplBufferMaxSizeBytes.Set(bufferStats.MaxSizeBytes) + metricsReplBufferSizeBytes.Set(bufferStats.SizeBytes) +} + +// MetricsNetworkStats are the network stats. +type MetricsNetworkStats struct { + Bytes float64 `bson:"bytes"` + Ops float64 `bson:"ops"` + GetMores *BenchmarkStats `bson:"getmores"` + ReadersCreated float64 `bson:"readersCreated"` +} + +// Export exposes the network stats. +func (metricsNetworkStats *MetricsNetworkStats) Export(ch chan<- prometheus.Metric) { + metricsReplNetworkBytesTotal.Set(metricsNetworkStats.Bytes) + metricsReplNetworkOpsTotal.Set(metricsNetworkStats.Ops) + metricsReplNetworkReadersCreatedTotal.Set(metricsNetworkStats.ReadersCreated) + + metricsReplNetworkGetmoresNumTotal.Set(metricsNetworkStats.GetMores.Num) + metricsReplNetworkGetmoresTotalMilliseconds.Set(metricsNetworkStats.GetMores.TotalMillis) +} + +// ReplStats are the stats associated with the replication process. +type ReplStats struct { + Apply *ApplyStats `bson:"apply"` + Buffer *BufferStats `bson:"buffer"` + Network *MetricsNetworkStats `bson:"network"` + PreloadStats *PreloadStats `bson:"preload"` +} + +// Export exposes the replication stats. +func (replStats *ReplStats) Export(ch chan<- prometheus.Metric) { + replStats.Apply.Export(ch) + replStats.Buffer.Export(ch) + replStats.Network.Export(ch) + replStats.PreloadStats.Export(ch) +} + +// PreloadStats are the stats associated with preload operation. +type PreloadStats struct { + Docs *BenchmarkStats `bson:"docs"` + Indexes *BenchmarkStats `bson:"indexes"` +} + +// Export exposes the preload stats. +func (preloadStats *PreloadStats) Export(ch chan<- prometheus.Metric) { + metricsReplPreloadDocsNumTotal.Set(preloadStats.Docs.Num) + metricsReplPreloadDocsTotalMilliseconds.Set(preloadStats.Docs.TotalMillis) + + metricsReplPreloadIndexesNumTotal.Set(preloadStats.Indexes.Num) + metricsReplPreloadIndexesTotalMilliseconds.Set(preloadStats.Indexes.TotalMillis) +} + +// StorageStats are the stats associated with the storage. +type StorageStats struct { + BucketExhausted float64 `bson:"freelist.search.bucketExhausted"` + Requests float64 `bson:"freelist.search.requests"` + Scanned float64 `bson:"freelist.search.scanned"` +} + +// Export exports the storage stats. +func (storageStats *StorageStats) Export(ch chan<- prometheus.Metric) { + metricsStorageFreelistSearchTotal.WithLabelValues("bucket_exhausted").Set(storageStats.BucketExhausted) + metricsStorageFreelistSearchTotal.WithLabelValues("requests").Set(storageStats.Requests) + metricsStorageFreelistSearchTotal.WithLabelValues("scanned").Set(storageStats.Scanned) +} + // CursorStatsOpen are the stats for open cursors type CursorStatsOpen struct { - NoTimeout float64 `bson:"noTimeout"` - Pinned float64 `bson:"pinned"` - Total float64 `bson:"total"` + NoTimeout float64 `bson:"noTimeout"` + Pinned float64 `bson:"pinned"` + Total float64 `bson:"total"` } // CursorStats are the stats for cursors type CursorStats struct { - TimedOut float64 `bson:"timedOut"` - Open *CursorStatsOpen `bson:"open"` + TimedOut float64 `bson:"timedOut"` + Open *CursorStatsOpen `bson:"open"` } // Export exports the cursor stats. func (cursorStats *CursorStats) Export(ch chan<- prometheus.Metric) { - metricsCursorTimedOutTotal.Set(cursorStats.TimedOut) - metricsCursorOpen.WithLabelValues("noTimeout").Set(cursorStats.Open.NoTimeout) - metricsCursorOpen.WithLabelValues("pinned").Set(cursorStats.Open.Pinned) - metricsCursorOpen.WithLabelValues("total").Set(cursorStats.Open.Total) + metricsCursorTimedOutTotal.Set(cursorStats.TimedOut) + metricsCursorOpen.WithLabelValues("timed_out").Set(cursorStats.Open.NoTimeout) + metricsCursorOpen.WithLabelValues("pinned").Set(cursorStats.Open.Pinned) + metricsCursorOpen.WithLabelValues("total").Set(cursorStats.Open.Total) } // MetricsStats are all stats associated with metrics of the system type MetricsStats struct { + Document *DocumentStats `bson:"document"` GetLastError *GetLastErrorStats `bson:"getLastError"` - Cursor *CursorStats `bson:"cursor"` + Operation *OperationStats `bson:"operation"` + QueryExecutor *QueryExecutorStats `bson:"queryExecutor"` + Record *RecordStats `bson:"record"` + Repl *ReplStats `bson:"repl"` + Storage *StorageStats `bson:"storage"` + Cursor *CursorStats `bson:"cursor"` } // Export exports the metrics stats. func (metricsStats *MetricsStats) Export(ch chan<- prometheus.Metric) { + if metricsStats.Document != nil { + metricsStats.Document.Export(ch) + } if metricsStats.GetLastError != nil { metricsStats.GetLastError.Export(ch) } + if metricsStats.Operation != nil { + metricsStats.Operation.Export(ch) + } + if metricsStats.QueryExecutor != nil { + metricsStats.QueryExecutor.Export(ch) + } + if metricsStats.Record != nil { + metricsStats.Record.Export(ch) + } + if metricsStats.Repl != nil { + metricsStats.Repl.Export(ch) + } + if metricsStats.Storage != nil { + metricsStats.Storage.Export(ch) + } if metricsStats.Cursor != nil { metricsStats.Cursor.Export(ch) } metricsCursorTimedOutTotal.Collect(ch) metricsCursorOpen.Collect(ch) + metricsDocumentTotal.Collect(ch) metricsGetLastErrorWtimeNumTotal.Collect(ch) metricsGetLastErrorWtimeTotalMilliseconds.Collect(ch) metricsGetLastErrorWtimeoutsTotal.Collect(ch) + metricsOperationTotal.Collect(ch) + metricsQueryExecutorTotal.Collect(ch) + metricsRecordMovesTotal.Collect(ch) + metricsReplApplyBatchesNumTotal.Collect(ch) + metricsReplApplyBatchesTotalMilliseconds.Collect(ch) + metricsReplApplyOpsTotal.Collect(ch) + metricsReplBufferCount.Collect(ch) + metricsReplBufferMaxSizeBytes.Collect(ch) + metricsReplBufferSizeBytes.Collect(ch) + metricsReplNetworkGetmoresNumTotal.Collect(ch) + metricsReplNetworkGetmoresTotalMilliseconds.Collect(ch) + metricsReplNetworkBytesTotal.Collect(ch) + metricsReplNetworkOpsTotal.Collect(ch) + metricsReplNetworkReadersCreatedTotal.Collect(ch) + metricsReplOplogInsertNumTotal.Collect(ch) + metricsReplOplogInsertTotalMilliseconds.Collect(ch) + metricsReplOplogInsertBytesTotal.Collect(ch) + metricsReplPreloadDocsNumTotal.Collect(ch) + metricsReplPreloadDocsTotalMilliseconds.Collect(ch) + metricsReplPreloadIndexesNumTotal.Collect(ch) + metricsReplPreloadIndexesTotalMilliseconds.Collect(ch) + metricsStorageFreelistSearchTotal.Collect(ch) + metricsTTLDeletedDocumentsTotal.Collect(ch) + metricsTTLPassesTotal.Collect(ch) } // Describe describes the metrics for prometheus func (metricsStats *MetricsStats) Describe(ch chan<- *prometheus.Desc) { metricsCursorTimedOutTotal.Describe(ch) metricsCursorOpen.Describe(ch) + metricsDocumentTotal.Describe(ch) metricsGetLastErrorWtimeNumTotal.Describe(ch) metricsGetLastErrorWtimeTotalMilliseconds.Describe(ch) metricsGetLastErrorWtimeoutsTotal.Describe(ch) + metricsOperationTotal.Describe(ch) + metricsQueryExecutorTotal.Describe(ch) + metricsRecordMovesTotal.Describe(ch) + metricsReplApplyBatchesNumTotal.Describe(ch) + metricsReplApplyBatchesTotalMilliseconds.Describe(ch) + metricsReplApplyOpsTotal.Describe(ch) + metricsReplBufferCount.Describe(ch) + metricsReplBufferMaxSizeBytes.Describe(ch) + metricsReplBufferSizeBytes.Describe(ch) + metricsReplNetworkGetmoresNumTotal.Describe(ch) + metricsReplNetworkGetmoresTotalMilliseconds.Describe(ch) + metricsReplNetworkBytesTotal.Describe(ch) + metricsReplNetworkOpsTotal.Describe(ch) + metricsReplNetworkReadersCreatedTotal.Describe(ch) + metricsReplOplogInsertNumTotal.Describe(ch) + metricsReplOplogInsertTotalMilliseconds.Describe(ch) + metricsReplOplogInsertBytesTotal.Describe(ch) + metricsReplPreloadDocsNumTotal.Describe(ch) + metricsReplPreloadDocsTotalMilliseconds.Describe(ch) + metricsReplPreloadIndexesNumTotal.Describe(ch) + metricsReplPreloadIndexesTotalMilliseconds.Describe(ch) + metricsStorageFreelistSearchTotal.Describe(ch) + metricsTTLDeletedDocumentsTotal.Describe(ch) + metricsTTLPassesTotal.Describe(ch) } diff --git a/collector/mongos/metrics_test.go b/collector/mongos/metrics_test.go deleted file mode 100644 index 980ba595d..000000000 --- a/collector/mongos/metrics_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package collector_mongos - -import ( - "testing" -) - -func Test_MetricsCollectData(t *testing.T) { - stats := &MetricsStats{ - GetLastError: &GetLastErrorStats{ - Wtime: &BenchmarkStats{}, - }, - } - - stats.Export() -} diff --git a/collector/mongos/network_test.go b/collector/mongos/network_test.go deleted file mode 100644 index ef6f9e562..000000000 --- a/collector/mongos/network_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package collector_mongos - -import ( - "testing" -) - -func Test_NetworkCollectData(t *testing.T) { - stats := &NetworkStats{} - - stats.Export() -} diff --git a/collector/mongos/op_counters_test.go b/collector/mongos/op_counters_test.go deleted file mode 100644 index f7d1f6264..000000000 --- a/collector/mongos/op_counters_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package collector_mongos - -import ( - "testing" -) - -func Test_OpCountersCollectData(t *testing.T) { - stats := &OpcountersStats{} - - stats.Export() -} diff --git a/collector/mongos/server_status_test.go b/collector/mongos/server_status_test.go deleted file mode 100644 index 60f26ec6b..000000000 --- a/collector/mongos/server_status_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package collector_mongos - -import ( - "gopkg.in/mgo.v2/bson" - "testing" -) - -func Test_ServerStatusCollectData(t *testing.T) { - data := LoadFixture("server_status.bson") - serverStatus := &ServerStatus{} - loadServerStatusFromBson(data, serverStatus) - - serverStatus.Export() -} - -func Test_ParserServerStatus(t *testing.T) { - data := LoadFixture("server_status.bson") - - serverStatus := &ServerStatus{} - loadServerStatusFromBson(data, serverStatus) - - if serverStatus.Asserts == nil { - t.Error("Asserts group was not loaded") - } - - if serverStatus.Connections == nil { - t.Error("Connections group was not loaded") - } - - if serverStatus.ExtraInfo == nil { - t.Error("ExtraInfo group was not loaded") - } - - if serverStatus.Network == nil { - t.Error("Network group was not loaded") - } - - if serverStatus.Opcounters == nil { - t.Error("Opcounters group was not loaded") - } - - if serverStatus.Mem == nil { - t.Error("Mem group was not loaded") - } - - if serverStatus.Connections == nil { - t.Error("Connections group was not loaded") - } - - if serverStatus.Metrics.Document.Deleted != 45726 { - t.Error("Metrics group was not loaded correctly") - } -} - -func loadServerStatusFromBson(data []byte, status *ServerStatus) { - err := bson.Unmarshal(data, status) - if err != nil { - panic(err) - } -} diff --git a/collector/shared/version.go b/collector/shared/version.go deleted file mode 100644 index da2046f75..000000000 --- a/collector/shared/version.go +++ /dev/null @@ -1,42 +0,0 @@ -package collector_shared - -import( - "strings" - "strconv" - "github.com/golang/glog" - "gopkg.in/mgo.v2" -) - -var serverVersion string = "unknown" - -func GetServerVersion(session *mgo.Session) (string) { - if serverVersion == "unknown" { - buildInfo, err := session.BuildInfo() - if err != nil { - glog.Error("Could not get BuildInfo!") - return "unknown" - } - serverVersion = buildInfo.Version - return serverVersion - } else { - return serverVersion - } -} - -func IsVersionGreater(version string, major int, minor int, release int) (bool) { - split := strings.Split(version, ".") - cmp_major, _ := strconv.Atoi(split[0]) - cmp_minor, _ := strconv.Atoi(split[1]) - cmp_release, _ := strconv.Atoi(split[2]) - - if cmp_major >= major && cmp_minor >= minor && cmp_release >= release { - return true - } - - return false -} - -func IsMyVersionGreater(session *mgo.Session, major int, minor int, release int) (bool) { - version := GetServerVersion(session) - return IsVersionGreater(version, major, minor, release) -} diff --git a/mongodb_exporter.go b/mongodb_exporter.go index fc1f3f077..2c83bb985 100644 --- a/mongodb_exporter.go +++ b/mongodb_exporter.go @@ -3,21 +3,30 @@ package main import ( "flag" "fmt" + "net/http" + "os" + "github.com/dcu/mongodb_exporter/collector" "github.com/dcu/mongodb_exporter/shared" + "github.com/prometheus/client_golang/prometheus" - "net/http" ) +func mongodbDefaultUri() string { + if u := os.Getenv("MONGODB_URL"); u != "" { + return u + } + return "mongodb://localhost:27017" +} + var ( listenAddressFlag = flag.String("web.listen-address", ":9001", "Address on which to expose metrics and web interface.") metricsPathFlag = flag.String("web.metrics-path", "/metrics", "Path under which to expose metrics.") - mongodbURIFlag = flag.String("mongodb.uri", "mongodb://localhost:27017", "Mongodb URI, format: [mongodb://][user:pass@]host1[:port1][,host2[:port2],...][/database][?options]") + mongodbURIFlag = flag.String("mongodb.uri", mongodbDefaultUri(), "Mongodb URI, format: [mongodb://][user:pass@]host1[:port1][,host2[:port2],...][/database][?options]") enabledGroupsFlag = flag.String("groups.enabled", "asserts,durability,background_flushing,connections,extra_info,global_lock,index_counters,network,op_counters,op_counters_repl,memory,locks,metrics", "Comma-separated list of groups to use, for more info see: docs.mongodb.org/manual/reference/command/serverStatus/") - //printCollectors = flag.Bool("collectors.print", false, "If true, print available collectors and exit.") - authUserFlag = flag.String("auth.user", "", "Username for basic auth.") - authPassFlag = flag.String("auth.pass", "", "Password for basic auth.") + authUserFlag = flag.String("auth.user", "", "Username for basic auth.") + authPassFlag = flag.String("auth.pass", "", "Password for basic auth.") ) type basicAuthHandler struct { diff --git a/shared/connection.go b/shared/connection.go new file mode 100644 index 000000000..47684918e --- /dev/null +++ b/shared/connection.go @@ -0,0 +1,65 @@ +package shared + +import ( + "time" + + "github.com/golang/glog" + "gopkg.in/mgo.v2" +) + +const ( + dialMongodbTimeout = 10 * time.Second + syncMongodbTimeout = 1 * time.Minute +) + +func MongoSession(uri string) *mgo.Session { + dialInfo, err := mgo.ParseURL(uri) + if err != nil { + glog.Errorf("Cannot connect to server using url %s: %s", uri, err) + return nil + } + + dialInfo.Direct = true // Force direct connection + dialInfo.Timeout = dialMongodbTimeout + + session, err := mgo.DialWithInfo(dialInfo) + if err != nil { + glog.Errorf("Cannot connect to server using url %s: %s", uri, err) + return nil + } + session.SetMode(mgo.Eventual, true) + session.SetSyncTimeout(syncMongodbTimeout) + session.SetSocketTimeout(0) + return session +} + +func MongoSessionServerVersion(session *mgo.Session) (string, error) { + buildInfo, err := session.BuildInfo() + if err != nil { + glog.Errorf("Could not get MongoDB BuildInfo: %s!", err) + return "unknown", err + } + return buildInfo.Version, nil +} + +func MongoSessionNodeType(session *mgo.Session) (string, error) { + masterDoc := struct { + SetName interface{} `bson:"setName"` + Hosts interface{} `bson:"hosts"` + Msg string `bson:"msg"` + }{} + err := session.Run("isMaster", &masterDoc) + if err != nil { + glog.Errorf("Got unknown node type: %s", err) + return "unknown", err + } + + if masterDoc.SetName != nil || masterDoc.Hosts != nil { + return "replset", nil + } else if masterDoc.Msg == "isdbgrid" { + // isdbgrid is always the msg value when calling isMaster on a mongos + // see http://docs.mongodb.org/manual/core/sharded-cluster-query-router/ + return "mongos", nil + } + return "mongod", nil +} From 0affcf6473ef324478b39fd9ac28483294747611 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Mon, 11 Apr 2016 11:36:52 +0200 Subject: [PATCH 2/6] commiting the right file this time --- collector/mongos/metrics.go | 415 +----------------------------------- 1 file changed, 11 insertions(+), 404 deletions(-) diff --git a/collector/mongos/metrics.go b/collector/mongos/metrics.go index 893521d0c..70419a0b1 100644 --- a/collector/mongos/metrics.go +++ b/collector/mongos/metrics.go @@ -1,4 +1,4 @@ -package collector_mongod +package collector_mongos import ( "github.com/prometheus/client_golang/prometheus" @@ -19,13 +19,6 @@ var ( Help: "The open is an embedded document that contains data regarding open cursors", }, []string{"state"}) ) -var ( - metricsDocumentTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: Namespace, - Name: "metrics_document_total", - Help: "The document holds a document of that reflect document access and modification patterns and data use. Compare these values to the data in the opcounters document, which track total number of operations", - }, []string{"state"}) -) var ( metricsGetLastErrorWtimeNumTotal = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: Namespace, @@ -48,191 +41,6 @@ var ( Help: "wtimeouts reports the number of times that write concern operations have timed out as a result of the wtimeout threshold to getLastError.", }) ) -var ( - metricsOperationTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: Namespace, - Name: "metrics_operation_total", - Help: "operation is a sub-document that holds counters for several types of update and query operations that MongoDB handles using special operation types", - }, []string{"type"}) -) -var ( - metricsQueryExecutorTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: Namespace, - Name: "metrics_query_executor_total", - Help: "queryExecutor is a document that reports data from the query execution system", - }, []string{"state"}) -) -var ( - metricsRecordMovesTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_record", - Name: "moves_total", - Help: "moves reports the total number of times documents move within the on-disk representation of the MongoDB data set. Documents move as a result of operations that increase the size of the document beyond their allocated record size", - }) -) -var ( - metricsReplApplyBatchesNumTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_apply_batches", - Name: "num_total", - Help: "num reports the total number of batches applied across all databases", - }) - metricsReplApplyBatchesTotalMilliseconds = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_apply_batches", - Name: "total_milliseconds", - Help: "total_millis reports the total amount of time the mongod has spent applying operations from the oplog", - }) -) -var ( - metricsReplApplyOpsTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_apply", - Name: "ops_total", - Help: "ops reports the total number of oplog operations applied", - }) -) -var ( - metricsReplBufferCount = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_buffer", - Name: "count", - Help: "count reports the current number of operations in the oplog buffer", - }) - metricsReplBufferMaxSizeBytes = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_buffer", - Name: "max_size_bytes", - Help: "maxSizeBytes reports the maximum size of the buffer. This value is a constant setting in the mongod, and is not configurable", - }) - metricsReplBufferSizeBytes = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_buffer", - Name: "size_bytes", - Help: "sizeBytes reports the current size of the contents of the oplog buffer", - }) -) -var ( - metricsReplNetworkGetmoresNumTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_network_getmores", - Name: "num_total", - Help: "num reports the total number of getmore operations, which are operations that request an additional set of operations from the replication sync source.", - }) - metricsReplNetworkGetmoresTotalMilliseconds = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_network_getmores", - Name: "total_milliseconds", - Help: "total_millis reports the total amount of time required to collect data from getmore operations", - }) -) -var ( - metricsReplNetworkBytesTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_network", - Name: "bytes_total", - Help: "bytes reports the total amount of data read from the replication sync source", - }) - metricsReplNetworkOpsTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_network", - Name: "ops_total", - Help: "ops reports the total number of operations read from the replication source.", - }) - metricsReplNetworkReadersCreatedTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_network", - Name: "readers_created_total", - Help: "readersCreated reports the total number of oplog query processes created. MongoDB will create a new oplog query any time an error occurs in the connection, including a timeout, or a network operation. Furthermore, readersCreated will increment every time MongoDB selects a new source fore replication.", - }) -) -var ( - metricsReplOplogInsertNumTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_oplog_insert", - Name: "num_total", - Help: "num reports the total number of items inserted into the oplog.", - }) - metricsReplOplogInsertTotalMilliseconds = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_oplog_insert", - Name: "total_milliseconds", - Help: "total_millis reports the total amount of time spent for the mongod to insert data into the oplog.", - }) -) -var ( - metricsReplOplogInsertBytesTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_oplog", - Name: "insert_bytes_total", - Help: "insertBytes the total size of documents inserted into the oplog.", - }) -) -var ( - metricsReplPreloadDocsNumTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_preload_docs", - Name: "num_total", - Help: "num reports the total number of documents loaded during the pre-fetch stage of replication", - }) - metricsReplPreloadDocsTotalMilliseconds = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_preload_docs", - Name: "total_milliseconds", - Help: "total_millis reports the total amount of time spent loading documents as part of the pre-fetch stage of replication", - }) -) -var ( - metricsReplPreloadIndexesNumTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_preload_indexes", - Name: "num_total", - Help: "num reports the total number of index entries loaded by members before updating documents as part of the pre-fetch stage of replication", - }) - metricsReplPreloadIndexesTotalMilliseconds = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_repl_preload_indexes", - Name: "total_milliseconds", - Help: "total_millis reports the total amount of time spent loading index entries as part of the pre-fetch stage of replication", - }) -) -var ( - metricsStorageFreelistSearchTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: Namespace, - Name: "metrics_storage_freelist_search_total", - Help: "metrics about searching records in the database.", - }, []string{"type"}) -) -var ( - metricsTTLDeletedDocumentsTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_ttl", - Name: "deleted_documents_total", - Help: "deletedDocuments reports the total number of documents deleted from collections with a ttl index.", - }) - metricsTTLPassesTotal = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: Namespace, - Subsystem: "metrics_ttl", - Name: "passes_total", - Help: "passes reports the number of times the background process removes documents from collections with a ttl index", - }) -) - -// DocumentStats are the stats associated to a document. -type DocumentStats struct { - Deleted float64 `bson:"deleted"` - Inserted float64 `bson:"inserted"` - Returned float64 `bson:"returned"` - Updated float64 `bson:"updated"` -} - -// Export exposes the document stats to be consumed by the prometheus server. -func (documentStats *DocumentStats) Export(ch chan<- prometheus.Metric) { - metricsDocumentTotal.WithLabelValues("deleted").Set(documentStats.Deleted) - metricsDocumentTotal.WithLabelValues("inserted").Set(documentStats.Inserted) - metricsDocumentTotal.WithLabelValues("returned").Set(documentStats.Returned) - metricsDocumentTotal.WithLabelValues("updated").Set(documentStats.Updated) -} // BenchmarkStats is bechmark info about an operation. type BenchmarkStats struct { @@ -254,255 +62,54 @@ func (getLastErrorStats *GetLastErrorStats) Export(ch chan<- prometheus.Metric) metricsGetLastErrorWtimeoutsTotal.Set(getLastErrorStats.Wtimeouts) } -// OperationStats are the stats for some kind of operations. -type OperationStats struct { - Fastmod float64 `bson:"fastmod"` - Idhack float64 `bson:"idhack"` - ScanAndOrder float64 `bson:"scanAndOrder"` -} - -// Export exports the operation stats. -func (operationStats *OperationStats) Export(ch chan<- prometheus.Metric) { - metricsOperationTotal.WithLabelValues("fastmod").Set(operationStats.Fastmod) - metricsOperationTotal.WithLabelValues("idhack").Set(operationStats.Idhack) - metricsOperationTotal.WithLabelValues("scan_and_order").Set(operationStats.ScanAndOrder) -} - -// QueryExecutorStats are the stats associated with a query execution. -type QueryExecutorStats struct { - Scanned float64 `bson:"scanned"` - ScannedObjects float64 `bson:"scannedObjects"` -} - -// Export exports the query executor stats. -func (queryExecutorStats *QueryExecutorStats) Export(ch chan<- prometheus.Metric) { - metricsQueryExecutorTotal.WithLabelValues("scanned").Set(queryExecutorStats.Scanned) - metricsQueryExecutorTotal.WithLabelValues("scanned_objects").Set(queryExecutorStats.ScannedObjects) -} - -// RecordStats are stats associated with a record. -type RecordStats struct { - Moves float64 `bson:"moves"` -} - -// Export exposes the record stats. -func (recordStats *RecordStats) Export(ch chan<- prometheus.Metric) { - metricsRecordMovesTotal.Set(recordStats.Moves) -} - -// ApplyStats are the stats associated with the apply operation. -type ApplyStats struct { - Batches *BenchmarkStats `bson:"batches"` - Ops float64 `bson:"ops"` -} - -// Export exports the apply stats -func (applyStats *ApplyStats) Export(ch chan<- prometheus.Metric) { - metricsReplApplyOpsTotal.Set(applyStats.Ops) - - metricsReplApplyBatchesNumTotal.Set(applyStats.Batches.Num) - metricsReplApplyBatchesTotalMilliseconds.Set(applyStats.Batches.TotalMillis) -} - -// BufferStats are the stats associated with the buffer -type BufferStats struct { - Count float64 `bson:"count"` - MaxSizeBytes float64 `bson:"maxSizeBytes"` - SizeBytes float64 `bson:"sizeBytes"` -} - -// Export exports the buffer stats. -func (bufferStats *BufferStats) Export(ch chan<- prometheus.Metric) { - metricsReplBufferCount.Set(bufferStats.Count) - metricsReplBufferMaxSizeBytes.Set(bufferStats.MaxSizeBytes) - metricsReplBufferSizeBytes.Set(bufferStats.SizeBytes) -} - -// MetricsNetworkStats are the network stats. -type MetricsNetworkStats struct { - Bytes float64 `bson:"bytes"` - Ops float64 `bson:"ops"` - GetMores *BenchmarkStats `bson:"getmores"` - ReadersCreated float64 `bson:"readersCreated"` -} - -// Export exposes the network stats. -func (metricsNetworkStats *MetricsNetworkStats) Export(ch chan<- prometheus.Metric) { - metricsReplNetworkBytesTotal.Set(metricsNetworkStats.Bytes) - metricsReplNetworkOpsTotal.Set(metricsNetworkStats.Ops) - metricsReplNetworkReadersCreatedTotal.Set(metricsNetworkStats.ReadersCreated) - - metricsReplNetworkGetmoresNumTotal.Set(metricsNetworkStats.GetMores.Num) - metricsReplNetworkGetmoresTotalMilliseconds.Set(metricsNetworkStats.GetMores.TotalMillis) -} - -// ReplStats are the stats associated with the replication process. -type ReplStats struct { - Apply *ApplyStats `bson:"apply"` - Buffer *BufferStats `bson:"buffer"` - Network *MetricsNetworkStats `bson:"network"` - PreloadStats *PreloadStats `bson:"preload"` -} - -// Export exposes the replication stats. -func (replStats *ReplStats) Export(ch chan<- prometheus.Metric) { - replStats.Apply.Export(ch) - replStats.Buffer.Export(ch) - replStats.Network.Export(ch) - replStats.PreloadStats.Export(ch) -} - -// PreloadStats are the stats associated with preload operation. -type PreloadStats struct { - Docs *BenchmarkStats `bson:"docs"` - Indexes *BenchmarkStats `bson:"indexes"` -} - -// Export exposes the preload stats. -func (preloadStats *PreloadStats) Export(ch chan<- prometheus.Metric) { - metricsReplPreloadDocsNumTotal.Set(preloadStats.Docs.Num) - metricsReplPreloadDocsTotalMilliseconds.Set(preloadStats.Docs.TotalMillis) - - metricsReplPreloadIndexesNumTotal.Set(preloadStats.Indexes.Num) - metricsReplPreloadIndexesTotalMilliseconds.Set(preloadStats.Indexes.TotalMillis) -} - -// StorageStats are the stats associated with the storage. -type StorageStats struct { - BucketExhausted float64 `bson:"freelist.search.bucketExhausted"` - Requests float64 `bson:"freelist.search.requests"` - Scanned float64 `bson:"freelist.search.scanned"` -} - -// Export exports the storage stats. -func (storageStats *StorageStats) Export(ch chan<- prometheus.Metric) { - metricsStorageFreelistSearchTotal.WithLabelValues("bucket_exhausted").Set(storageStats.BucketExhausted) - metricsStorageFreelistSearchTotal.WithLabelValues("requests").Set(storageStats.Requests) - metricsStorageFreelistSearchTotal.WithLabelValues("scanned").Set(storageStats.Scanned) -} - // CursorStatsOpen are the stats for open cursors type CursorStatsOpen struct { - NoTimeout float64 `bson:"noTimeout"` - Pinned float64 `bson:"pinned"` - Total float64 `bson:"total"` + NoTimeout float64 `bson:"noTimeout"` + Pinned float64 `bson:"pinned"` + Total float64 `bson:"total"` } // CursorStats are the stats for cursors type CursorStats struct { - TimedOut float64 `bson:"timedOut"` - Open *CursorStatsOpen `bson:"open"` + TimedOut float64 `bson:"timedOut"` + Open *CursorStatsOpen `bson:"open"` } // Export exports the cursor stats. func (cursorStats *CursorStats) Export(ch chan<- prometheus.Metric) { - metricsCursorTimedOutTotal.Set(cursorStats.TimedOut) - metricsCursorOpen.WithLabelValues("timed_out").Set(cursorStats.Open.NoTimeout) - metricsCursorOpen.WithLabelValues("pinned").Set(cursorStats.Open.Pinned) - metricsCursorOpen.WithLabelValues("total").Set(cursorStats.Open.Total) + metricsCursorTimedOutTotal.Set(cursorStats.TimedOut) + metricsCursorOpen.WithLabelValues("timed_out").Set(cursorStats.Open.NoTimeout) + metricsCursorOpen.WithLabelValues("pinned").Set(cursorStats.Open.Pinned) + metricsCursorOpen.WithLabelValues("total").Set(cursorStats.Open.Total) } // MetricsStats are all stats associated with metrics of the system type MetricsStats struct { - Document *DocumentStats `bson:"document"` GetLastError *GetLastErrorStats `bson:"getLastError"` - Operation *OperationStats `bson:"operation"` - QueryExecutor *QueryExecutorStats `bson:"queryExecutor"` - Record *RecordStats `bson:"record"` - Repl *ReplStats `bson:"repl"` - Storage *StorageStats `bson:"storage"` - Cursor *CursorStats `bson:"cursor"` + Cursor *CursorStats `bson:"cursor"` } // Export exports the metrics stats. func (metricsStats *MetricsStats) Export(ch chan<- prometheus.Metric) { - if metricsStats.Document != nil { - metricsStats.Document.Export(ch) - } if metricsStats.GetLastError != nil { metricsStats.GetLastError.Export(ch) } - if metricsStats.Operation != nil { - metricsStats.Operation.Export(ch) - } - if metricsStats.QueryExecutor != nil { - metricsStats.QueryExecutor.Export(ch) - } - if metricsStats.Record != nil { - metricsStats.Record.Export(ch) - } - if metricsStats.Repl != nil { - metricsStats.Repl.Export(ch) - } - if metricsStats.Storage != nil { - metricsStats.Storage.Export(ch) - } if metricsStats.Cursor != nil { metricsStats.Cursor.Export(ch) } metricsCursorTimedOutTotal.Collect(ch) metricsCursorOpen.Collect(ch) - metricsDocumentTotal.Collect(ch) metricsGetLastErrorWtimeNumTotal.Collect(ch) metricsGetLastErrorWtimeTotalMilliseconds.Collect(ch) metricsGetLastErrorWtimeoutsTotal.Collect(ch) - metricsOperationTotal.Collect(ch) - metricsQueryExecutorTotal.Collect(ch) - metricsRecordMovesTotal.Collect(ch) - metricsReplApplyBatchesNumTotal.Collect(ch) - metricsReplApplyBatchesTotalMilliseconds.Collect(ch) - metricsReplApplyOpsTotal.Collect(ch) - metricsReplBufferCount.Collect(ch) - metricsReplBufferMaxSizeBytes.Collect(ch) - metricsReplBufferSizeBytes.Collect(ch) - metricsReplNetworkGetmoresNumTotal.Collect(ch) - metricsReplNetworkGetmoresTotalMilliseconds.Collect(ch) - metricsReplNetworkBytesTotal.Collect(ch) - metricsReplNetworkOpsTotal.Collect(ch) - metricsReplNetworkReadersCreatedTotal.Collect(ch) - metricsReplOplogInsertNumTotal.Collect(ch) - metricsReplOplogInsertTotalMilliseconds.Collect(ch) - metricsReplOplogInsertBytesTotal.Collect(ch) - metricsReplPreloadDocsNumTotal.Collect(ch) - metricsReplPreloadDocsTotalMilliseconds.Collect(ch) - metricsReplPreloadIndexesNumTotal.Collect(ch) - metricsReplPreloadIndexesTotalMilliseconds.Collect(ch) - metricsStorageFreelistSearchTotal.Collect(ch) - metricsTTLDeletedDocumentsTotal.Collect(ch) - metricsTTLPassesTotal.Collect(ch) } // Describe describes the metrics for prometheus func (metricsStats *MetricsStats) Describe(ch chan<- *prometheus.Desc) { metricsCursorTimedOutTotal.Describe(ch) metricsCursorOpen.Describe(ch) - metricsDocumentTotal.Describe(ch) metricsGetLastErrorWtimeNumTotal.Describe(ch) metricsGetLastErrorWtimeTotalMilliseconds.Describe(ch) metricsGetLastErrorWtimeoutsTotal.Describe(ch) - metricsOperationTotal.Describe(ch) - metricsQueryExecutorTotal.Describe(ch) - metricsRecordMovesTotal.Describe(ch) - metricsReplApplyBatchesNumTotal.Describe(ch) - metricsReplApplyBatchesTotalMilliseconds.Describe(ch) - metricsReplApplyOpsTotal.Describe(ch) - metricsReplBufferCount.Describe(ch) - metricsReplBufferMaxSizeBytes.Describe(ch) - metricsReplBufferSizeBytes.Describe(ch) - metricsReplNetworkGetmoresNumTotal.Describe(ch) - metricsReplNetworkGetmoresTotalMilliseconds.Describe(ch) - metricsReplNetworkBytesTotal.Describe(ch) - metricsReplNetworkOpsTotal.Describe(ch) - metricsReplNetworkReadersCreatedTotal.Describe(ch) - metricsReplOplogInsertNumTotal.Describe(ch) - metricsReplOplogInsertTotalMilliseconds.Describe(ch) - metricsReplOplogInsertBytesTotal.Describe(ch) - metricsReplPreloadDocsNumTotal.Describe(ch) - metricsReplPreloadDocsTotalMilliseconds.Describe(ch) - metricsReplPreloadIndexesNumTotal.Describe(ch) - metricsReplPreloadIndexesTotalMilliseconds.Describe(ch) - metricsStorageFreelistSearchTotal.Describe(ch) - metricsTTLDeletedDocumentsTotal.Describe(ch) - metricsTTLPassesTotal.Describe(ch) } From 458be2ce07189d7f516eb9edf2a87aafce9c6dea Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Mon, 11 Apr 2016 12:03:43 +0200 Subject: [PATCH 3/6] added index metrics upstream bugfix, also updated test files --- collector/mongod/index_counters.go | 2 +- collector/mongod/server_status_test.go | 73 ++++++++++++++++++++++++++ collector/mongodb_collector_test.go | 22 ++------ collector/mongos/server_status_test.go | 73 ++++++++++++++++++++++++++ 4 files changed, 150 insertions(+), 20 deletions(-) create mode 100644 collector/mongod/server_status_test.go create mode 100644 collector/mongos/server_status_test.go diff --git a/collector/mongod/index_counters.go b/collector/mongod/index_counters.go index f3b1ec33b..05b621ea3 100644 --- a/collector/mongod/index_counters.go +++ b/collector/mongod/index_counters.go @@ -23,7 +23,7 @@ var ( //IndexCounterStats index counter stats type IndexCounterStats struct { - Accesses float64 `bson:"accesses` + Accesses float64 `bson:"accesses"` Hits float64 `bson:"hits"` Misses float64 `bson:"misses"` Resets float64 `bson:"resets"` diff --git a/collector/mongod/server_status_test.go b/collector/mongod/server_status_test.go new file mode 100644 index 000000000..d4094c1a9 --- /dev/null +++ b/collector/mongod/server_status_test.go @@ -0,0 +1,73 @@ +package collector_mongod + +import ( + "testing" + + "gopkg.in/mgo.v2/bson" +) + +func Test_ParserServerStatus(t *testing.T) { + data := LoadFixture("server_status.bson") + + serverStatus := &ServerStatus{} + loadServerStatusFromBson(data, serverStatus) + + if serverStatus.Asserts == nil { + t.Error("Asserts group was not loaded") + } + + if serverStatus.Dur == nil { + t.Error("Dur group was not loaded") + } + + if serverStatus.BackgroundFlushing == nil { + t.Error("BackgroundFlushing group was not loaded") + } + + if serverStatus.Connections == nil { + t.Error("Connections group was not loaded") + } + + if serverStatus.ExtraInfo == nil { + t.Error("ExtraInfo group was not loaded") + } + + if serverStatus.GlobalLock == nil { + t.Error("GlobalLock group was not loaded") + } + + if serverStatus.Network == nil { + t.Error("Network group was not loaded") + } + + if serverStatus.Opcounters == nil { + t.Error("Opcounters group was not loaded") + } + + if serverStatus.OpcountersRepl == nil { + t.Error("OpcountersRepl group was not loaded") + } + + if serverStatus.Mem == nil { + t.Error("Mem group was not loaded") + } + + if serverStatus.Connections == nil { + t.Error("Connections group was not loaded") + } + + if serverStatus.Locks == nil { + t.Error("Locks group was not loaded") + } + + if serverStatus.Metrics.Document.Deleted != 45726 { + t.Error("Metrics group was not loaded correctly") + } +} + +func loadServerStatusFromBson(data []byte, status *ServerStatus) { + err := bson.Unmarshal(data, status) + if err != nil { + panic(err) + } +} diff --git a/collector/mongodb_collector_test.go b/collector/mongodb_collector_test.go index 7eaa7e269..bee9c797c 100644 --- a/collector/mongodb_collector_test.go +++ b/collector/mongodb_collector_test.go @@ -1,19 +1,16 @@ package collector import ( + "testing" + "github.com/dcu/mongodb_exporter/shared" "github.com/prometheus/client_golang/prometheus" - "testing" ) func Test_CollectServerStatus(t *testing.T) { shared.ParseEnabledGroups("assers,durability,backgrond_flushing,connections,extra_info,global_lock,index_counters,network,op_counters,memory,locks,metrics,cursors") collector := NewMongodbCollector(MongodbCollectorOpts{URI: "localhost"}) - serverStatus := collector.collectServerStatus(nil) - - if serverStatus.Asserts == nil { - t.Error("Error loading document.") - } + go collector.Collect(nil) } func Test_DescribeCollector(t *testing.T) { @@ -29,16 +26,3 @@ func Test_CollectCollector(t *testing.T) { ch := make(chan prometheus.Metric) go collector.Collect(ch) } - -func Test_InvalidConnection(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - collector := NewMongodbCollector(MongodbCollectorOpts{URI: "s://localhost:123"}) - serverStatus := collector.collectServerStatus(nil) - - if serverStatus != nil { - t.Fail() - } -} diff --git a/collector/mongos/server_status_test.go b/collector/mongos/server_status_test.go new file mode 100644 index 000000000..a5a52653d --- /dev/null +++ b/collector/mongos/server_status_test.go @@ -0,0 +1,73 @@ +package collector_mongos + +import ( + "testing" + + "gopkg.in/mgo.v2/bson" +) + +func Test_ParserServerStatus(t *testing.T) { + data := LoadFixture("server_status.bson") + + serverStatus := &ServerStatus{} + loadServerStatusFromBson(data, serverStatus) + + if serverStatus.Asserts == nil { + t.Error("Asserts group was not loaded") + } + + if serverStatus.Dur == nil { + t.Error("Dur group was not loaded") + } + + if serverStatus.BackgroundFlushing == nil { + t.Error("BackgroundFlushing group was not loaded") + } + + if serverStatus.Connections == nil { + t.Error("Connections group was not loaded") + } + + if serverStatus.ExtraInfo == nil { + t.Error("ExtraInfo group was not loaded") + } + + if serverStatus.GlobalLock == nil { + t.Error("GlobalLock group was not loaded") + } + + if serverStatus.Network == nil { + t.Error("Network group was not loaded") + } + + if serverStatus.Opcounters == nil { + t.Error("Opcounters group was not loaded") + } + + if serverStatus.OpcountersRepl == nil { + t.Error("OpcountersRepl group was not loaded") + } + + if serverStatus.Mem == nil { + t.Error("Mem group was not loaded") + } + + if serverStatus.Connections == nil { + t.Error("Connections group was not loaded") + } + + if serverStatus.Locks == nil { + t.Error("Locks group was not loaded") + } + + if serverStatus.Metrics.Document.Deleted != 45726 { + t.Error("Metrics group was not loaded correctly") + } +} + +func loadServerStatusFromBson(data []byte, status *ServerStatus) { + err := bson.Unmarshal(data, status) + if err != nil { + panic(err) + } +} From 4670f7c0fa586efa191559a9d211254c2ebf0a89 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 13 Apr 2016 16:32:32 +0200 Subject: [PATCH 4/6] moving 'timed_out' back to 'noTimeout' because that was the correct name --- collector/mongod/metrics.go | 2 +- collector/mongos/metrics.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/collector/mongod/metrics.go b/collector/mongod/metrics.go index 893521d0c..20587c50f 100644 --- a/collector/mongod/metrics.go +++ b/collector/mongod/metrics.go @@ -397,7 +397,7 @@ type CursorStats struct { // Export exports the cursor stats. func (cursorStats *CursorStats) Export(ch chan<- prometheus.Metric) { metricsCursorTimedOutTotal.Set(cursorStats.TimedOut) - metricsCursorOpen.WithLabelValues("timed_out").Set(cursorStats.Open.NoTimeout) + metricsCursorOpen.WithLabelValues("noTimeout").Set(cursorStats.Open.NoTimeout) metricsCursorOpen.WithLabelValues("pinned").Set(cursorStats.Open.Pinned) metricsCursorOpen.WithLabelValues("total").Set(cursorStats.Open.Total) } diff --git a/collector/mongos/metrics.go b/collector/mongos/metrics.go index 70419a0b1..9ba1224d1 100644 --- a/collector/mongos/metrics.go +++ b/collector/mongos/metrics.go @@ -78,7 +78,7 @@ type CursorStats struct { // Export exports the cursor stats. func (cursorStats *CursorStats) Export(ch chan<- prometheus.Metric) { metricsCursorTimedOutTotal.Set(cursorStats.TimedOut) - metricsCursorOpen.WithLabelValues("timed_out").Set(cursorStats.Open.NoTimeout) + metricsCursorOpen.WithLabelValues("noTimeout").Set(cursorStats.Open.NoTimeout) metricsCursorOpen.WithLabelValues("pinned").Set(cursorStats.Open.Pinned) metricsCursorOpen.WithLabelValues("total").Set(cursorStats.Open.Total) } From 37ec32bf5e157010df5c3493fbb79762752074ea Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 13 Apr 2016 16:39:59 +0200 Subject: [PATCH 5/6] adding IsVersionGreater function to shared/utils.go for future functionality --- shared/utils.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/shared/utils.go b/shared/utils.go index 19fb80a3e..6a0ddb054 100644 --- a/shared/utils.go +++ b/shared/utils.go @@ -24,3 +24,17 @@ func ParameterizeString(text string) string { result := parameterizeRegexp.ReplaceAllString(text, "_") return strings.ToLower(result) } + +func IsVersionGreater(version string, major int, minor int, release int) bool { + split := strings.Split(version, ".") + cmp_major, _ := strconv.Atoi(split[0]) + cmp_minor, _ := strconv.Atoi(split[1]) + cmp_release, _ := strconv.Atoi(split[2]) + + if cmp_major >= major && cmp_minor >= minor && cmp_release >= release { + return true + } + + return false +} + From 1afa2cfde39e603c30896bd7ca73754ad874b8bc Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 13 Apr 2016 16:48:40 +0200 Subject: [PATCH 6/6] missing strconv import --- shared/utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/utils.go b/shared/utils.go index 6a0ddb054..c13182d15 100644 --- a/shared/utils.go +++ b/shared/utils.go @@ -3,6 +3,7 @@ package shared import ( "regexp" "strings" + "strconv" ) var (