Mongo dbstats

metricbeat/docs/fields.asciidoc

@@ -2059,6 +2059,63 @@ MongoDB metrics.
 
 
 
+[float]
+== dbstats Fields
+
+dbstats provides an overview of a particular mongo database. This document is most concerned with data volumes of a database.
+
+
+
+[float]
+=== mongodb.dbstats.avg_object_size
+
+type: long
+
+[float]
+=== mongodb.dbstats.collections
+
+type: integer
+
+[float]
+=== mongodb.dbstats.data_size
+
+type: long
+
+[float]
+=== mongodb.dbstats.db
+
+type: keyword
+
+[float]
+=== mongodb.dbstats.file_size
+
+type: long
+
+[float]
+=== mongodb.dbstats.index_size
+
+type: long
+
+[float]
+=== mongodb.dbstats.indexes
+
+type: long
+
+[float]
+=== mongodb.dbstats.num_extents
+
+type: long
+
+[float]
+=== mongodb.dbstats.objects
+
+type: long
+
+[float]
+=== mongodb.dbstats.storage_size
+
+type: long
+
 [float]
 == status Fields
 

metricbeat/docs/modules/mongodb.asciidoc

@@ -88,7 +88,11 @@ metricbeat.modules:
 
 The following metricsets are available:
 
+* <<metricbeat-metricset-mongodb-dbstats,dbstats>>
+
 * <<metricbeat-metricset-mongodb-status,status>>
 
+include::mongodb/dbstats.asciidoc[]
+
 include::mongodb/status.asciidoc[]
 

metricbeat/include/list.go

@@ -23,6 +23,7 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/kafka"
 	_ "github.com/elastic/beats/metricbeat/module/kafka/partition"
 	_ "github.com/elastic/beats/metricbeat/module/mongodb"
+	_ "github.com/elastic/beats/metricbeat/module/mongodb/dbstats"
 	_ "github.com/elastic/beats/metricbeat/module/mongodb/status"
 	_ "github.com/elastic/beats/metricbeat/module/mysql"
 	_ "github.com/elastic/beats/metricbeat/module/mysql/status"

metricbeat/metricbeat.template-es2x.json

@@ -1054,6 +1054,42 @@
         },
         "mongodb": {
           "properties": {
+            "dbstats": {
+              "properties": {
+                "avg_object_size": {
+                  "type": "long"
+                },
+                "collections": {
+                  "type": "long"
+                },
+                "data_size": {
+                  "type": "long"
+                },
+                "db": {
+                  "ignore_above": 1024,
+                  "index": "not_analyzed",
+                  "type": "string"
+                },
+                "file_size": {
+                  "type": "long"
+                },
+                "index_size": {
+                  "type": "long"
+                },
+                "indexes": {
+                  "type": "long"
+                },
+                "num_extents": {
+                  "type": "long"
+                },
+                "objects": {
+                  "type": "long"
+                },
+                "storage_size": {
+                  "type": "long"
+                }
+              }
+            },
             "status": {
               "properties": {
                 "asserts": {

metricbeat/metricbeat.template.json

@@ -1051,6 +1051,41 @@
         },
         "mongodb": {
           "properties": {
+            "dbstats": {
+              "properties": {
+                "avg_object_size": {
+                  "type": "long"
+                },
+                "collections": {
+                  "type": "long"
+                },
+                "data_size": {
+                  "type": "long"
+                },
+                "db": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "file_size": {
+                  "type": "long"
+                },
+                "index_size": {
+                  "type": "long"
+                },
+                "indexes": {
+                  "type": "long"
+                },
+                "num_extents": {
+                  "type": "long"
+                },
+                "objects": {
+                  "type": "long"
+                },
+                "storage_size": {
+                  "type": "long"
+                }
+              }
+            },
             "status": {
               "properties": {
                 "asserts": {

metricbeat/module/mongodb/dbstats/_meta/data.json

@@ -0,0 +1,19 @@
+{
+    "@timestamp":"2016-05-23T08:05:34.853Z",
+    "beat":{
+        "hostname":"beathost",
+        "name":"beathost"
+    },
+    "metricset":{
+        "host":"localhost",
+        "module":"mysql",
+        "name":"status",
+        "rtt":44269
+    },
+    "mongodb":{
+        "dbstats":{
+            "example": "dbstats"
+        }
+    },
+    "type":"metricsets"
+}

metricbeat/module/mongodb/dbstats/_meta/docs.asciidoc

@@ -0,0 +1,3 @@
+=== mongodb dbstats MetricSet
+
+This is the dbstats metricset of the module mongodb.

metricbeat/module/mongodb/dbstats/_meta/fields.yml

@@ -0,0 +1,35 @@
+- name: dbstats
+  type: group
+  description: >
+    dbstats provides an overview of a particular mongo database. This document
+    is most concerned with data volumes of a database.
+  fields:
+    - name: avg_object_size
+      type: long
+
+    - name: collections
+      type: integer
+
+    - name: data_size
+      type: long
+
+    - name: db
+      type: keyword
+
+    - name: file_size
+      type: long
+
+    - name: index_size
+      type: long
+
+    - name: indexes
+      type: long
+
+    - name: num_extents
+      type: long
+
+    - name: objects
+      type: long
+
+    - name: storage_size
+      type: long

metricbeat/module/mongodb/dbstats/data.go

@@ -0,0 +1,21 @@
+package dbstats
+
+import (
+	s "github.com/elastic/beats/metricbeat/schema"
+	c "github.com/elastic/beats/metricbeat/schema/mapstriface"
+)
+
+var schema = s.Schema{
+	"db":              c.Str("db"),
+	"collection":      c.Int("collections"),
+	"objects":         c.Int("objects"),
+	"avg_object_size": c.Int("avgObjectSize"),
+	"data_size":       c.Int("dataSize"),
+	"storage_size":    c.Int("storageSize"),
+	"num_extents":     c.Int("numExtents"),
+	"indexes":         c.Int("indexes"),
+	"index_size":      c.Int("indexSize"),
+	"file_size":       c.Int("fileSize"),
+}
+
+var eventMapping = schema.Apply

metricbeat/module/mongodb/dbstats/dbstats.go

@@ -0,0 +1,90 @@
+package dbstats
+
+import (
+	"errors"
+
+	"github.com/beats/libbeat/logp"
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/metricbeat/mb"
+	"gopkg.in/mgo.v2"
+)
+
+// init registers the MetricSet with the central registry.
+// The New method will be called after the setup of the module and before starting to fetch data
+func init() {
+	if err := mb.Registry.AddMetricSet("mongodb", "dbstats", New); err != nil {
+		panic(err)
+	}
+}
+
+// MetricSet type defines all fields of the MetricSet
+// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with
+// additional entries. These variables can be used to persist data or configuration between
+// multiple fetch calls.
+type MetricSet struct {
+	mb.BaseMetricSet
+	dialInfo *mgo.DialInfo
+}
+
+// New create a new instance of the MetricSet
+// Part of new is also setting up the configuration by processing additional
+// configuration entries if needed.
+func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	dialInfo, err := mgo.ParseURL(base.HostData().URI)
+	if err != nil {
+		return nil, err
+	}
+	dialInfo.Timeout = base.Module().Config().Timeout
+
+	return &MetricSet{
+		BaseMetricSet: base,
+		dialInfo:      dialInfo,
+	}, nil
+}
+
+// Fetch methods implements the data gathering and data conversion to the right format
+// It returns the event which is then forward to the output. In case of an error, a
+// descriptive error must be returned.
+func (m *MetricSet) Fetch() ([]common.MapStr, error) {
+
+	// establish connection to mongo
+	session, err := mgo.DialWithInfo(m.dialInfo)
+	if err != nil {
+		return nil, err
+	}
+	defer session.Close()
+
+	session.SetMode(mgo.Monotonic, true)
+
+	// Get the list of databases database names, which we'll use to call db.stats() on each
+	dbNames, err := session.DatabaseNames()
+	if err != nil {
+		logp.Err("Error retrieving database names from Mongo instance")
+		return []common.MapStr{}, err
+	}
+
+	// events is the list of events collected from each of the databases.
+	events := []common.MapStr{}
+
+	// for each database, call db.stats() and append to events
+	for _, dbName := range dbNames {
+		db := session.DB(dbName)
+
+		result := map[string]interface{}{}
+
+		err := db.Run("dbStats", &result)
+		if err != nil {
+			logp.Err("Failed to retrieve stats for db %s", dbName)
+			continue
+		}
+		events = append(events, eventMapping(result))
+	}
+
+	// if we failed to collect on any databases, return an error
+	if len(events) == 0 {
+		err = errors.New("Failed to fetch stats for all databases in mongo instance")
+		return []common.MapStr{}, err
+	}
+
+	return events, nil
+}

metricbeat/module/mongodb/dbstats/dbstats_integration_test.go

@@ -0,0 +1,47 @@
+package dbstats
+
+import (
+	"testing"
+
+	"github.com/beats/metricbeat/module/mongodb"
+	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFetch(t *testing.T) {
+	f := mbtest.NewEventsFetcher(t, getConfig())
+	events, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	for _, event := range events {
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+		// Check a few event Fields
+		db := event["db"].(string)
+		assert.NotEqual(t, db, "")
+
+		dataSize := event["data_size"].(int64)
+		assert.True(t, dataSize > 0)
+
+		collections := event["collections"].(int64)
+		assert.True(t, collections > 0)
+	}
+}
+
+func TestData(t *testing.T) {
+	f := mbtest.NewEventsFetcher(t, getConfig())
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
+func getConfig() map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "mongodb",
+		"metricsets": []string{"dbstats"},
+		"hosts":      []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()},
+	}
+}