From d896b012f9386cab398c579601f90606c90fb542 Mon Sep 17 00:00:00 2001 From: Ryan Cox Date: Wed, 3 Nov 2010 14:04:09 +0100 Subject: [PATCH 01/15] configure.in: Check for libmongoc. --- configure.in | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/configure.in b/configure.in index 9b6397f863..6a8899ed88 100644 --- a/configure.in +++ b/configure.in @@ -1241,6 +1241,15 @@ AC_CHECK_MEMBERS([kstat_io_t.nwritten, kstat_io_t.writes, kstat_io_t.nwrites, ks # # Checks for libraries begin here # + +with_libmongoc="yes" +AC_CHECK_LIB(mongoc, mongo_run_command, +[ + AC_DEFINE(HAVE_LIBMONGOC, 1, [Define to 1 if you have the 'mongoc' library (-lmongoc).]) +], +[with_libmongoc="no"]) +AM_CONDITIONAL(BUILD_WITH_LIBMONGOC, test "x$with_libmongoc" = "xyes") + with_libresolv="yes" AC_CHECK_LIB(resolv, res_search, [ From 4ce283aaf014d2fa76984ed1fcaccbf5ad8ed08c Mon Sep 17 00:00:00 2001 From: Akkarit Sangpetch Date: Wed, 3 Nov 2010 14:01:11 +0100 Subject: [PATCH 02/15] Add write_mongo plugin support for mongodb output (based on write_redis) --- configure.in | 2 + src/Makefile.am | 11 ++ src/collectd.conf.in | 9 ++ src/write_mongo.c | 244 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 266 insertions(+) create mode 100644 src/write_mongo.c diff --git a/configure.in b/configure.in index 6a8899ed88..186082a5af 100644 --- a/configure.in +++ b/configure.in @@ -4688,6 +4688,7 @@ AC_PLUGIN([vserver], [$plugin_vserver], [Linux VServer statistics]) AC_PLUGIN([wireless], [$plugin_wireless], [Wireless statistics]) AC_PLUGIN([write_http], [$with_libcurl], [HTTP output plugin]) AC_PLUGIN([write_redis], [$with_libcredis], [Redis output plugin]) +AC_PLUGIN([write_mongo], [yes], [Mongo output plugin]) AC_PLUGIN([xmms], [$with_libxmms], [XMMS statistics]) AC_PLUGIN([zfs_arc], [$plugin_zfs_arc], [ZFS ARC statistics]) @@ -5009,6 +5010,7 @@ Configuration: wireless . . . . . . $enable_wireless write_http . . . . . $enable_write_http write_redis . . . . . $enable_write_redis + write_mongo . . . . . $enable_write_mongo xmms . . . . . . . . $enable_xmms zfs_arc . . . . . . . $enable_zfs_arc diff --git a/src/Makefile.am b/src/Makefile.am index 795de5767c..37012512ef 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1253,6 +1253,17 @@ collectd_LDADD += "-dlopen" write_redis.la collectd_DEPENDENCIES += write_redis.la endif +if BUILD_PLUGIN_WRITE_MONGO +pkglib_LTLIBRARIES += write_mongo.la +write_mongo_la_SOURCES = write_mongo.c +write_mongo_la_LDFLAGS = -module -avoid-version +write_mongo_la_CFLAGS = -DMONGO_HAVE_STDINT $(AM_CFLAGS) +write_mongo_la_LIBADD = libmongo/libmongo.la +write_mongo_la_DEPENDENCIES = libmongo/libmongo.la +collectd_LDADD += "-dlopen" write_mongo.la +collectd_DEPENDENCIES += write_mongo.la +endif + if BUILD_PLUGIN_XMMS pkglib_LTLIBRARIES += xmms.la xmms_la_SOURCES = xmms.c diff --git a/src/collectd.conf.in b/src/collectd.conf.in index 33e9660c40..455b547db2 100644 --- a/src/collectd.conf.in +++ b/src/collectd.conf.in @@ -146,6 +146,7 @@ #@BUILD_PLUGIN_WIRELESS_TRUE@LoadPlugin wireless #@BUILD_PLUGIN_WRITE_HTTP_TRUE@LoadPlugin write_http #@BUILD_PLUGIN_WRITE_REDIS_TRUE@LoadPlugin write_redis +#@BUILD_PLUGIN_WRITE_MONGO_TRUE@LoadPlugin write_mongo #@BUILD_PLUGIN_XMMS_TRUE@LoadPlugin xmms #@BUILD_PLUGIN_ZFS_ARC_TRUE@LoadPlugin zfs_arc @@ -972,6 +973,14 @@ # # +# +# +# Host "localhost" +# Port "27017" +# Timeout 1000 +# +# + ############################################################################## # Filter configuration # #----------------------------------------------------------------------------# diff --git a/src/write_mongo.c b/src/write_mongo.c new file mode 100644 index 0000000000..c17c35f105 --- /dev/null +++ b/src/write_mongo.c @@ -0,0 +1,244 @@ +/** + * collectd - src/write_mongo.c + * Copyright (C) 2010 Florian Forster, Akkarit Sangpetch + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: + * Florian Forster + * Akkarit Sangpetch + **/ + +#include "collectd.h" +#include "plugin.h" +#include "common.h" +#include "configfile.h" + +#include +#include "libmongo/bson.h" +#include "libmongo/mongo.h" + +struct wm_node_s +{ + char name[DATA_MAX_NAME_LEN]; + + char *host; + int port; + int timeout; + + int connected; + + mongo_connection conn[1]; + mongo_connection_options opts[1]; + pthread_mutex_t lock; +}; +typedef struct wm_node_s wm_node_t; + +/* + * Functions + */ +static int wm_write (const data_set_t *ds, /* {{{ */ + const value_list_t *vl, + user_data_t *ud) +{ + wm_node_t *node = ud->data; + char collection_name[512]; + int status; + int i; + bson record[1]; + bson_buffer record_buf[1]; + + ssnprintf(collection_name, sizeof (collection_name), "collectd.%s", vl->plugin); + + bson_buffer_init(record_buf); + bson_append_time_t(record_buf,"ts",vl->time); + bson_append_string(record_buf,"h",vl->host); + bson_append_string(record_buf,"i",vl->plugin_instance); + bson_append_string(record_buf,"t",vl->plugin_instance); + bson_append_string(record_buf,"ti",vl->plugin_instance); + + if (ds->ds_num == 1) { + if (ds->ds[0].type == DS_TYPE_COUNTER) + bson_append_long(record_buf, "v", vl->values[0].counter); + else if (ds->ds[0].type == DS_TYPE_GAUGE) + bson_append_double(record_buf, "v", vl->values[0].gauge); + else if (ds->ds[0].type == DS_TYPE_DERIVE) + bson_append_long(record_buf, "v", vl->values[0].derive); + else if (ds->ds[0].type == DS_TYPE_ABSOLUTE) + bson_append_long(record_buf, "v", vl->values[0].absolute); + else + assert (23 == 42); + } else { + bson_append_start_object(record_buf,"v"); + for (i = 0; i < ds->ds_num; i++) + { + if (ds->ds[i].type == DS_TYPE_COUNTER) + bson_append_long(record_buf, ds->ds[i].name, vl->values[i].counter); + else if (ds->ds[i].type == DS_TYPE_GAUGE) + bson_append_double(record_buf, ds->ds[i].name, vl->values[i].gauge); + else if (ds->ds[i].type == DS_TYPE_DERIVE) + bson_append_long(record_buf, ds->ds[i].name, vl->values[i].derive); + else if (ds->ds[i].type == DS_TYPE_ABSOLUTE) + bson_append_long(record_buf, ds->ds[i].name, vl->values[i].absolute); + else + assert (23 == 42); + } + bson_append_finish_object(record_buf); + } + bson_from_buffer(record,record_buf); + + pthread_mutex_lock (&node->lock); + + if (node->connected == 0) + { + strcpy(node->opts->host, node->host); + node->opts->port = node->port; + + status = mongo_connect(node->conn,node->opts); + if (status!=mongo_conn_success) { + ERROR ("write_mongo plugin: Connecting to host \"%s\" (port %i) failed.", + (node->host != NULL) ? node->host : "localhost", + (node->port != 0) ? node->port : 6379); + pthread_mutex_unlock (&node->lock); + return (-1); + } else { + node->connected = 1; + } + } + + /* Assert if the connection has been established */ + assert (node->connected == 1); + + mongo_insert(node->conn,collection_name,record); + + pthread_mutex_unlock (&node->lock); + + return (0); +} /* }}} int wm_write */ + +static void wm_config_free (void *ptr) /* {{{ */ +{ + wm_node_t *node = ptr; + + if (node == NULL) + return; + + if (node->connected != 0) + { + mongo_destroy(node->conn); + node->connected = 0; + } + + sfree (node->host); + sfree (node); +} /* }}} void wm_config_free */ + +static int wm_config_node (oconfig_item_t *ci) /* {{{ */ +{ + wm_node_t *node; + int status; + int i; + + node = malloc (sizeof (*node)); + if (node == NULL) + return (ENOMEM); + memset (node, 0, sizeof (*node)); + node->host = NULL; + node->port = 0; + node->timeout = 1000; + node->connected = 0; + pthread_mutex_init (&node->lock, /* attr = */ NULL); + + status = cf_util_get_string_buffer (ci, node->name, sizeof (node->name)); + + if (status != 0) + { + sfree (node); + return (status); + } + + for (i = 0; i < ci->children_num; i++) + { + oconfig_item_t *child = ci->children + i; + + if (strcasecmp ("Host", child->key) == 0) + status = cf_util_get_string (child, &node->host); + else if (strcasecmp ("Port", child->key) == 0) + { + status = cf_util_get_port_number (child); + if (status > 0) + { + node->port = status; + status = 0; + } + } + else if (strcasecmp ("Timeout", child->key) == 0) + status = cf_util_get_int (child, &node->timeout); + else + WARNING ("write_mongo plugin: Ignoring unknown config option \"%s\".", + child->key); + + if (status != 0) + break; + } /* for (i = 0; i < ci->children_num; i++) */ + + if (status == 0) + { + char cb_name[DATA_MAX_NAME_LEN]; + user_data_t ud; + + ssnprintf (cb_name, sizeof (cb_name), "write_mongo/%s", node->name); + + ud.data = node; + ud.free_func = wm_config_free; + + status = plugin_register_write (cb_name, wm_write, &ud); + INFO ("write_mongo plugin: registered write plugin %s %d",cb_name,status); + } + + if (status != 0) + wm_config_free (node); + + return (status); +} /* }}} int wm_config_node */ + +static int wm_config (oconfig_item_t *ci) /* {{{ */ +{ + int i; + + for (i = 0; i < ci->children_num; i++) + { + oconfig_item_t *child = ci->children + i; + + if (strcasecmp ("Node", child->key) == 0) + wm_config_node (child); + else + WARNING ("write_mongo plugin: Ignoring unknown " + "configuration option \"%s\" at top level.", child->key); + } + + return (0); +} /* }}} int wm_config */ + +void module_register (void) +{ + plugin_register_complex_config ("write_mongo", wm_config); +} + +/* vim: set sw=2 sts=2 tw=78 et fdm=marker : */ From 26e39b822919e7fd701e8c6c80f01ee3e3ffec19 Mon Sep 17 00:00:00 2001 From: Akkarit Sangpetch Date: Sat, 2 Oct 2010 11:49:43 -0400 Subject: [PATCH 03/15] Fix plugin and type instance field name, also free bson object buffer --- src/write_mongo.c | 63 +++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/src/write_mongo.c b/src/write_mongo.c index c17c35f105..1f7e3d0965 100644 --- a/src/write_mongo.c +++ b/src/write_mongo.c @@ -61,47 +61,33 @@ static int wm_write (const data_set_t *ds, /* {{{ */ char collection_name[512]; int status; int i; - bson record[1]; - bson_buffer record_buf[1]; + bson record; + bson_buffer record_buf; ssnprintf(collection_name, sizeof (collection_name), "collectd.%s", vl->plugin); - bson_buffer_init(record_buf); - bson_append_time_t(record_buf,"ts",vl->time); - bson_append_string(record_buf,"h",vl->host); - bson_append_string(record_buf,"i",vl->plugin_instance); - bson_append_string(record_buf,"t",vl->plugin_instance); - bson_append_string(record_buf,"ti",vl->plugin_instance); - - if (ds->ds_num == 1) { - if (ds->ds[0].type == DS_TYPE_COUNTER) - bson_append_long(record_buf, "v", vl->values[0].counter); - else if (ds->ds[0].type == DS_TYPE_GAUGE) - bson_append_double(record_buf, "v", vl->values[0].gauge); - else if (ds->ds[0].type == DS_TYPE_DERIVE) - bson_append_long(record_buf, "v", vl->values[0].derive); - else if (ds->ds[0].type == DS_TYPE_ABSOLUTE) - bson_append_long(record_buf, "v", vl->values[0].absolute); + bson_buffer_init(&record_buf); + bson_append_time_t(&record_buf,"ts",vl->time); + bson_append_string(&record_buf,"h",vl->host); + bson_append_string(&record_buf,"i",vl->plugin_instance); + bson_append_string(&record_buf,"t",vl->type); + bson_append_string(&record_buf,"ti",vl->type_instance); + + for (i = 0; i < ds->ds_num; i++) + { + if (ds->ds[i].type == DS_TYPE_COUNTER) + bson_append_long(&record_buf, ds->ds[i].name, vl->values[i].counter); + else if (ds->ds[i].type == DS_TYPE_GAUGE) + bson_append_double(&record_buf, ds->ds[i].name, vl->values[i].gauge); + else if (ds->ds[i].type == DS_TYPE_DERIVE) + bson_append_long(&record_buf, ds->ds[i].name, vl->values[i].derive); + else if (ds->ds[i].type == DS_TYPE_ABSOLUTE) + bson_append_long(&record_buf, ds->ds[i].name, vl->values[i].absolute); else assert (23 == 42); - } else { - bson_append_start_object(record_buf,"v"); - for (i = 0; i < ds->ds_num; i++) - { - if (ds->ds[i].type == DS_TYPE_COUNTER) - bson_append_long(record_buf, ds->ds[i].name, vl->values[i].counter); - else if (ds->ds[i].type == DS_TYPE_GAUGE) - bson_append_double(record_buf, ds->ds[i].name, vl->values[i].gauge); - else if (ds->ds[i].type == DS_TYPE_DERIVE) - bson_append_long(record_buf, ds->ds[i].name, vl->values[i].derive); - else if (ds->ds[i].type == DS_TYPE_ABSOLUTE) - bson_append_long(record_buf, ds->ds[i].name, vl->values[i].absolute); - else - assert (23 == 42); - } - bson_append_finish_object(record_buf); } - bson_from_buffer(record,record_buf); + + bson_from_buffer(&record,&record_buf); pthread_mutex_lock (&node->lock); @@ -114,7 +100,8 @@ static int wm_write (const data_set_t *ds, /* {{{ */ if (status!=mongo_conn_success) { ERROR ("write_mongo plugin: Connecting to host \"%s\" (port %i) failed.", (node->host != NULL) ? node->host : "localhost", - (node->port != 0) ? node->port : 6379); + (node->port != 0) ? node->port : 27017); + mongo_destroy(node->conn); pthread_mutex_unlock (&node->lock); return (-1); } else { @@ -125,10 +112,12 @@ static int wm_write (const data_set_t *ds, /* {{{ */ /* Assert if the connection has been established */ assert (node->connected == 1); - mongo_insert(node->conn,collection_name,record); + mongo_insert(node->conn,collection_name,&record); pthread_mutex_unlock (&node->lock); + bson_buffer_destroy(&record_buf); + return (0); } /* }}} int wm_write */ From 212ef86c9038207108837448ec4eafd8d4ab0cb9 Mon Sep 17 00:00:00 2001 From: Florian Forster Date: Wed, 3 Nov 2010 15:13:28 +0100 Subject: [PATCH 04/15] write_mongo plugin: Build fixes. Enable the "write_mongo" plugin when the library is available. Move the "MONGO_HAVE_STDINT" define into the .c file. Avoid the poisoned "strcpy". --- configure.in | 2 +- src/Makefile.am | 20 +++++++++----------- src/write_mongo.c | 12 +++++++++--- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/configure.in b/configure.in index 186082a5af..387c2201c4 100644 --- a/configure.in +++ b/configure.in @@ -4688,7 +4688,7 @@ AC_PLUGIN([vserver], [$plugin_vserver], [Linux VServer statistics]) AC_PLUGIN([wireless], [$plugin_wireless], [Wireless statistics]) AC_PLUGIN([write_http], [$with_libcurl], [HTTP output plugin]) AC_PLUGIN([write_redis], [$with_libcredis], [Redis output plugin]) -AC_PLUGIN([write_mongo], [yes], [Mongo output plugin]) +AC_PLUGIN([write_mongo], [$with_libmongoc], [MongoDB output plugin]) AC_PLUGIN([xmms], [$with_libxmms], [XMMS statistics]) AC_PLUGIN([zfs_arc], [$plugin_zfs_arc], [ZFS ARC statistics]) diff --git a/src/Makefile.am b/src/Makefile.am index 37012512ef..757efa2218 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1243,6 +1243,15 @@ endif collectd_DEPENDENCIES += write_http.la endif +if BUILD_PLUGIN_WRITE_MONGO +pkglib_LTLIBRARIES += write_mongo.la +write_mongo_la_SOURCES = write_mongo.c +write_mongo_la_LDFLAGS = -module -avoid-version +write_mongo_la_LIBADD = -lmongoc +collectd_LDADD += "-dlopen" write_mongo.la +collectd_DEPENDENCIES += write_mongo.la +endif + if BUILD_PLUGIN_WRITE_REDIS pkglib_LTLIBRARIES += write_redis.la write_redis_la_SOURCES = write_redis.c @@ -1253,17 +1262,6 @@ collectd_LDADD += "-dlopen" write_redis.la collectd_DEPENDENCIES += write_redis.la endif -if BUILD_PLUGIN_WRITE_MONGO -pkglib_LTLIBRARIES += write_mongo.la -write_mongo_la_SOURCES = write_mongo.c -write_mongo_la_LDFLAGS = -module -avoid-version -write_mongo_la_CFLAGS = -DMONGO_HAVE_STDINT $(AM_CFLAGS) -write_mongo_la_LIBADD = libmongo/libmongo.la -write_mongo_la_DEPENDENCIES = libmongo/libmongo.la -collectd_LDADD += "-dlopen" write_mongo.la -collectd_DEPENDENCIES += write_mongo.la -endif - if BUILD_PLUGIN_XMMS pkglib_LTLIBRARIES += xmms.la xmms_la_SOURCES = xmms.c diff --git a/src/write_mongo.c b/src/write_mongo.c index 1f7e3d0965..b20d0c9bf8 100644 --- a/src/write_mongo.c +++ b/src/write_mongo.c @@ -31,8 +31,13 @@ #include "configfile.h" #include -#include "libmongo/bson.h" -#include "libmongo/mongo.h" + +#if HAVE_STDINT_H +# define MONGO_HAVE_STDINT 1 +#else +# define MONGO_USE_LONG_LONG_INT 1 +#endif +#include struct wm_node_s { @@ -93,7 +98,8 @@ static int wm_write (const data_set_t *ds, /* {{{ */ if (node->connected == 0) { - strcpy(node->opts->host, node->host); + sstrncpy(node->opts->host, node->host, + sizeof (node->opts->host)); node->opts->port = node->port; status = mongo_connect(node->conn,node->opts); From e853b1cad4b79bcab0a2ba000ec78d97fee1b1f3 Mon Sep 17 00:00:00 2001 From: Florian Forster Date: Wed, 3 Nov 2010 15:19:26 +0100 Subject: [PATCH 05/15] write_mongodb plugin: Rename the "write_mongo" plugin. --- configure.in | 4 ++-- src/Makefile.am | 14 +++++++------- src/{write_mongo.c => write_mongodb.c} | 14 +++++++------- 3 files changed, 16 insertions(+), 16 deletions(-) rename src/{write_mongo.c => write_mongodb.c} (92%) diff --git a/configure.in b/configure.in index 387c2201c4..bbdf2732cd 100644 --- a/configure.in +++ b/configure.in @@ -4688,7 +4688,7 @@ AC_PLUGIN([vserver], [$plugin_vserver], [Linux VServer statistics]) AC_PLUGIN([wireless], [$plugin_wireless], [Wireless statistics]) AC_PLUGIN([write_http], [$with_libcurl], [HTTP output plugin]) AC_PLUGIN([write_redis], [$with_libcredis], [Redis output plugin]) -AC_PLUGIN([write_mongo], [$with_libmongoc], [MongoDB output plugin]) +AC_PLUGIN([write_mongodb], [$with_libmongoc], [MongoDB output plugin]) AC_PLUGIN([xmms], [$with_libxmms], [XMMS statistics]) AC_PLUGIN([zfs_arc], [$plugin_zfs_arc], [ZFS ARC statistics]) @@ -5010,7 +5010,7 @@ Configuration: wireless . . . . . . $enable_wireless write_http . . . . . $enable_write_http write_redis . . . . . $enable_write_redis - write_mongo . . . . . $enable_write_mongo + write_mongodb . . . . $enable_write_mongodb xmms . . . . . . . . $enable_xmms zfs_arc . . . . . . . $enable_zfs_arc diff --git a/src/Makefile.am b/src/Makefile.am index 757efa2218..40ab538226 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1243,13 +1243,13 @@ endif collectd_DEPENDENCIES += write_http.la endif -if BUILD_PLUGIN_WRITE_MONGO -pkglib_LTLIBRARIES += write_mongo.la -write_mongo_la_SOURCES = write_mongo.c -write_mongo_la_LDFLAGS = -module -avoid-version -write_mongo_la_LIBADD = -lmongoc -collectd_LDADD += "-dlopen" write_mongo.la -collectd_DEPENDENCIES += write_mongo.la +if BUILD_PLUGIN_WRITE_MONGODB +pkglib_LTLIBRARIES += write_mongodb.la +write_mongodb_la_SOURCES = write_mongodb.c +write_mongodb_la_LDFLAGS = -module -avoid-version +write_mongodb_la_LIBADD = -lmongoc +collectd_LDADD += "-dlopen" write_mongodb.la +collectd_DEPENDENCIES += write_mongodb.la endif if BUILD_PLUGIN_WRITE_REDIS diff --git a/src/write_mongo.c b/src/write_mongodb.c similarity index 92% rename from src/write_mongo.c rename to src/write_mongodb.c index b20d0c9bf8..2b809215e4 100644 --- a/src/write_mongo.c +++ b/src/write_mongodb.c @@ -1,5 +1,5 @@ /** - * collectd - src/write_mongo.c + * collectd - src/write_mongodb.c * Copyright (C) 2010 Florian Forster, Akkarit Sangpetch * * Permission is hereby granted, free of charge, to any person obtaining a @@ -104,7 +104,7 @@ static int wm_write (const data_set_t *ds, /* {{{ */ status = mongo_connect(node->conn,node->opts); if (status!=mongo_conn_success) { - ERROR ("write_mongo plugin: Connecting to host \"%s\" (port %i) failed.", + ERROR ("write_mongodb plugin: Connecting to host \"%s\" (port %i) failed.", (node->host != NULL) ? node->host : "localhost", (node->port != 0) ? node->port : 27017); mongo_destroy(node->conn); @@ -186,7 +186,7 @@ static int wm_config_node (oconfig_item_t *ci) /* {{{ */ else if (strcasecmp ("Timeout", child->key) == 0) status = cf_util_get_int (child, &node->timeout); else - WARNING ("write_mongo plugin: Ignoring unknown config option \"%s\".", + WARNING ("write_mongodb plugin: Ignoring unknown config option \"%s\".", child->key); if (status != 0) @@ -198,13 +198,13 @@ static int wm_config_node (oconfig_item_t *ci) /* {{{ */ char cb_name[DATA_MAX_NAME_LEN]; user_data_t ud; - ssnprintf (cb_name, sizeof (cb_name), "write_mongo/%s", node->name); + ssnprintf (cb_name, sizeof (cb_name), "write_mongodb/%s", node->name); ud.data = node; ud.free_func = wm_config_free; status = plugin_register_write (cb_name, wm_write, &ud); - INFO ("write_mongo plugin: registered write plugin %s %d",cb_name,status); + INFO ("write_mongodb plugin: registered write plugin %s %d",cb_name,status); } if (status != 0) @@ -224,7 +224,7 @@ static int wm_config (oconfig_item_t *ci) /* {{{ */ if (strcasecmp ("Node", child->key) == 0) wm_config_node (child); else - WARNING ("write_mongo plugin: Ignoring unknown " + WARNING ("write_mongodb plugin: Ignoring unknown " "configuration option \"%s\" at top level.", child->key); } @@ -233,7 +233,7 @@ static int wm_config (oconfig_item_t *ci) /* {{{ */ void module_register (void) { - plugin_register_complex_config ("write_mongo", wm_config); + plugin_register_complex_config ("write_mongodb", wm_config); } /* vim: set sw=2 sts=2 tw=78 et fdm=marker : */ From 13322bb3c3253dce20bd0241c3725d27dc1fd732 Mon Sep 17 00:00:00 2001 From: Florian Forster Date: Sat, 6 Nov 2010 11:53:42 +0100 Subject: [PATCH 06/15] configure.in: Improve check for libmongoc (aka. mongo-c-driver) --- configure.in | 71 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 8 deletions(-) diff --git a/configure.in b/configure.in index bbdf2732cd..e8a6b12b2b 100644 --- a/configure.in +++ b/configure.in @@ -1242,14 +1242,6 @@ AC_CHECK_MEMBERS([kstat_io_t.nwritten, kstat_io_t.writes, kstat_io_t.nwrites, ks # Checks for libraries begin here # -with_libmongoc="yes" -AC_CHECK_LIB(mongoc, mongo_run_command, -[ - AC_DEFINE(HAVE_LIBMONGOC, 1, [Define to 1 if you have the 'mongoc' library (-lmongoc).]) -], -[with_libmongoc="no"]) -AM_CONDITIONAL(BUILD_WITH_LIBMONGOC, test "x$with_libmongoc" = "xyes") - with_libresolv="yes" AC_CHECK_LIB(resolv, res_search, [ @@ -2152,6 +2144,69 @@ then fi # }}} +# --with-libmongoc {{{ +AC_ARG_WITH(libmongoc, [AS_HELP_STRING([--with-libmongoc@<:@=PREFIX@:>@], [Path to libmongoc.])], +[ + if test "x$withval" = "xyes" + then + with_libmongoc="yes" + else if test "x$withval" = "xno" + then + with_libmongoc="no" + else + with_libmongoc="yes" + LIBMONGOC_CPPFLAGS="$LIBMONGOC_CPPFLAGS -I$withval/include" + LIBMONGOC_LDFLAGS="$LIBMONGOC_LDFLAGS -L$withval/lib" + fi; fi +], +[with_libmongoc="yes"]) + +SAVE_CPPFLAGS="$CPPFLAGS" +SAVE_LDFLAGS="$LDFLAGS" + +CPPFLAGS="$CPPFLAGS $LIBMONGOC_CPPFLAGS" +LDFLAGS="$LDFLAGS $LIBMONGOC_LDFLAGS" + +if test "x$with_libmongoc" = "xyes" +then + if test "x$LIBMONGOC_CPPFLAGS" != "x" + then + AC_MSG_NOTICE([libmongoc CPPFLAGS: $LIBMONGOC_CPPFLAGS]) + fi + AC_CHECK_HEADERS(mongo.h, + [with_libmongoc="yes"], + [with_libmongoc="no ('mongo.h' not found)"], +[#if HAVE_STDINT_H +# define MONGO_HAVE_STDINT 1 +#else +# define MONGO_USE_LONG_LONG_INT 1 +#endif +]) +fi +if test "x$with_libmongoc" = "xyes" +then + if test "x$LIBMONGOC_LDFLAGS" != "x" + then + AC_MSG_NOTICE([libmongoc LDFLAGS: $LIBMONGOC_LDFLAGS]) + fi + AC_CHECK_LIB(mongoc, mongo_run_command, + [with_libmongoc="yes"], + [with_libmongoc="no (symbol 'mongo_run_command' not found)"]) +fi + +CPPFLAGS="$SAVE_CPPFLAGS" +LDFLAGS="$SAVE_LDFLAGS" + +if test "x$with_libmongoc" = "xyes" +then + BUILD_WITH_LIBMONGOC_CPPFLAGS="$LIBMONGOC_CPPFLAGS" + BUILD_WITH_LIBMONGOC_LDFLAGS="$LIBMONGOC_LDFLAGS" + AC_SUBST(BUILD_WITH_LIBMONGOC_CPPFLAGS) + AC_SUBST(BUILD_WITH_LIBMONGOC_LDFLAGS) +fi +AM_CONDITIONAL(BUILD_WITH_LIBMONGOC, test "x$with_libmongoc" = "xyes") +# }}} + # --with-libmysql {{{ with_mysql_config="mysql_config" with_mysql_cflags="" From f9d98b27fd63f050f2dc2a825fe3aa130aaab841 Mon Sep 17 00:00:00 2001 From: Chris Lundquist Date: Fri, 13 Jan 2012 14:33:29 -0800 Subject: [PATCH 07/15] compiles cleanly. libmongoc doesn't have an install target, will try to commit one there instead of having to hack the directory stucture to please ./configure. --- src/write_mongodb.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/write_mongodb.c b/src/write_mongodb.c index 2b809215e4..cabfd64348 100644 --- a/src/write_mongodb.c +++ b/src/write_mongodb.c @@ -39,6 +39,14 @@ #endif #include +struct mongo_options +{ + char *host; + int port; + int timeout; +}; +typedef struct mongo_options mongo_options; + struct wm_node_s { char name[DATA_MAX_NAME_LEN]; @@ -49,8 +57,8 @@ struct wm_node_s int connected; - mongo_connection conn[1]; - mongo_connection_options opts[1]; + mongo conn[1]; + mongo_options opts[1]; pthread_mutex_t lock; }; typedef struct wm_node_s wm_node_t; @@ -67,32 +75,32 @@ static int wm_write (const data_set_t *ds, /* {{{ */ int status; int i; bson record; - bson_buffer record_buf; + /*bson_data record_buf; */ ssnprintf(collection_name, sizeof (collection_name), "collectd.%s", vl->plugin); - bson_buffer_init(&record_buf); - bson_append_time_t(&record_buf,"ts",vl->time); - bson_append_string(&record_buf,"h",vl->host); - bson_append_string(&record_buf,"i",vl->plugin_instance); - bson_append_string(&record_buf,"t",vl->type); - bson_append_string(&record_buf,"ti",vl->type_instance); + bson_init(&record); + bson_append_time_t(&record,"ts",vl->time); + bson_append_string(&record,"h",vl->host); + bson_append_string(&record,"i",vl->plugin_instance); + bson_append_string(&record,"t",vl->type); + bson_append_string(&record,"ti",vl->type_instance); for (i = 0; i < ds->ds_num; i++) { if (ds->ds[i].type == DS_TYPE_COUNTER) - bson_append_long(&record_buf, ds->ds[i].name, vl->values[i].counter); + bson_append_long(&record, ds->ds[i].name, vl->values[i].counter); else if (ds->ds[i].type == DS_TYPE_GAUGE) - bson_append_double(&record_buf, ds->ds[i].name, vl->values[i].gauge); + bson_append_double(&record, ds->ds[i].name, vl->values[i].gauge); else if (ds->ds[i].type == DS_TYPE_DERIVE) - bson_append_long(&record_buf, ds->ds[i].name, vl->values[i].derive); + bson_append_long(&record, ds->ds[i].name, vl->values[i].derive); else if (ds->ds[i].type == DS_TYPE_ABSOLUTE) - bson_append_long(&record_buf, ds->ds[i].name, vl->values[i].absolute); + bson_append_long(&record, ds->ds[i].name, vl->values[i].absolute); else assert (23 == 42); } - bson_from_buffer(&record,&record_buf); + /* bson_from_buffer(&record,&record_buf); */ pthread_mutex_lock (&node->lock); @@ -102,8 +110,8 @@ static int wm_write (const data_set_t *ds, /* {{{ */ sizeof (node->opts->host)); node->opts->port = node->port; - status = mongo_connect(node->conn,node->opts); - if (status!=mongo_conn_success) { + status = mongo_connect(node->conn,node->opts->host, node->opts->port); + if (status!=MONGO_OK) { ERROR ("write_mongodb plugin: Connecting to host \"%s\" (port %i) failed.", (node->host != NULL) ? node->host : "localhost", (node->port != 0) ? node->port : 27017); @@ -122,7 +130,7 @@ static int wm_write (const data_set_t *ds, /* {{{ */ pthread_mutex_unlock (&node->lock); - bson_buffer_destroy(&record_buf); + /* bson_buffer_destroy(&record_buf); */ return (0); } /* }}} int wm_write */ From 575200360174d98b14f7377a7aa74ae702960bcb Mon Sep 17 00:00:00 2001 From: Chris Lundquist Date: Mon, 16 Jan 2012 16:13:13 -0800 Subject: [PATCH 08/15] added more debugging. cleaned up commented lines --- src/write_mongodb.c | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/write_mongodb.c b/src/write_mongodb.c index cabfd64348..900fb8786b 100644 --- a/src/write_mongodb.c +++ b/src/write_mongodb.c @@ -39,6 +39,7 @@ #endif #include +/* struct mongo_options { char *host; @@ -46,6 +47,7 @@ struct mongo_options int timeout; }; typedef struct mongo_options mongo_options; +*/ struct wm_node_s { @@ -58,7 +60,7 @@ struct wm_node_s int connected; mongo conn[1]; - mongo_options opts[1]; +/* mongo_options opts[1]; */ pthread_mutex_t lock; }; typedef struct wm_node_s wm_node_t; @@ -100,21 +102,23 @@ static int wm_write (const data_set_t *ds, /* {{{ */ assert (23 == 42); } - /* bson_from_buffer(&record,&record_buf); */ pthread_mutex_lock (&node->lock); if (node->connected == 0) { +/* sstrncpy(node->opts->host, node->host, sizeof (node->opts->host)); node->opts->port = node->port; +*/ - status = mongo_connect(node->conn,node->opts->host, node->opts->port); - if (status!=MONGO_OK) { + /* status = mongo_connect(node->conn,node->opts->host, node->opts->port);*/ + status = mongo_connect(node->conn, node->host, node->port); + if (status != MONGO_OK) { ERROR ("write_mongodb plugin: Connecting to host \"%s\" (port %i) failed.", (node->host != NULL) ? node->host : "localhost", - (node->port != 0) ? node->port : 27017); + (node->port != 0) ? node->port : MONGO_DEFAULT_PORT); mongo_destroy(node->conn); pthread_mutex_unlock (&node->lock); return (-1); @@ -126,11 +130,22 @@ static int wm_write (const data_set_t *ds, /* {{{ */ /* Assert if the connection has been established */ assert (node->connected == 1); - mongo_insert(node->conn,collection_name,&record); + DEBUG ( "write_mongodb plugin: writing record"); + /* bson_print(&record); */ - pthread_mutex_unlock (&node->lock); + status = mongo_insert(node->conn,collection_name,&record); + + if(status != MONGO_OK) + { + ERROR ( "write_mongodb plugin: error inserting record: "); + if(node->conn->err == MONGO_BSON_INVALID) + { + ERROR (record.errstr); + } + } - /* bson_buffer_destroy(&record_buf); */ + + pthread_mutex_unlock (&node->lock); return (0); } /* }}} int wm_write */ From 731f60bad6b6b051731f88051fa32876f2be6a45 Mon Sep 17 00:00:00 2001 From: Chris Lundquist Date: Mon, 16 Jan 2012 21:10:46 -0800 Subject: [PATCH 09/15] Ok it is posting to the local server --- src/write_mongodb.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/write_mongodb.c b/src/write_mongodb.c index 900fb8786b..5ee220e165 100644 --- a/src/write_mongodb.c +++ b/src/write_mongodb.c @@ -101,6 +101,7 @@ static int wm_write (const data_set_t *ds, /* {{{ */ else assert (23 == 42); } + bson_finish(&record); pthread_mutex_lock (&node->lock); @@ -137,11 +138,15 @@ static int wm_write (const data_set_t *ds, /* {{{ */ if(status != MONGO_OK) { - ERROR ( "write_mongodb plugin: error inserting record: "); + ERROR ( "write_mongodb plugin: error inserting record: %d", node->conn->err); if(node->conn->err == MONGO_BSON_INVALID) + { + ERROR (node->conn->errstr); + } else if ( record.err) { ERROR (record.errstr); } + } From e315325cacd8463b32fc61f91abf05c66de7fc8c Mon Sep 17 00:00:00 2001 From: Chris Lundquist Date: Mon, 16 Jan 2012 09:44:37 -0800 Subject: [PATCH 10/15] remove commented out code --- src/write_mongodb.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/write_mongodb.c b/src/write_mongodb.c index 5ee220e165..cb4161c871 100644 --- a/src/write_mongodb.c +++ b/src/write_mongodb.c @@ -77,7 +77,6 @@ static int wm_write (const data_set_t *ds, /* {{{ */ int status; int i; bson record; - /*bson_data record_buf; */ ssnprintf(collection_name, sizeof (collection_name), "collectd.%s", vl->plugin); @@ -103,7 +102,6 @@ static int wm_write (const data_set_t *ds, /* {{{ */ } bson_finish(&record); - pthread_mutex_lock (&node->lock); if (node->connected == 0) @@ -139,14 +137,14 @@ static int wm_write (const data_set_t *ds, /* {{{ */ if(status != MONGO_OK) { ERROR ( "write_mongodb plugin: error inserting record: %d", node->conn->err); - if(node->conn->err == MONGO_BSON_INVALID) + if(node->conn->err == MONGO_BSON_INVALID) { ERROR (node->conn->errstr); } else if ( record.err) { ERROR (record.errstr); } - + } From b9cd6ea32dd72d250727db1b76e4bc978d005574 Mon Sep 17 00:00:00 2001 From: Chris Lundquist Date: Fri, 20 Jan 2012 14:29:31 -0800 Subject: [PATCH 11/15] We have to convert CDTime to seconds since epoch --- src/write_mongodb.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/write_mongodb.c b/src/write_mongodb.c index cb4161c871..f13cdc332b 100644 --- a/src/write_mongodb.c +++ b/src/write_mongodb.c @@ -39,16 +39,6 @@ #endif #include -/* -struct mongo_options -{ - char *host; - int port; - int timeout; -}; -typedef struct mongo_options mongo_options; -*/ - struct wm_node_s { char name[DATA_MAX_NAME_LEN]; @@ -60,7 +50,6 @@ struct wm_node_s int connected; mongo conn[1]; -/* mongo_options opts[1]; */ pthread_mutex_t lock; }; typedef struct wm_node_s wm_node_t; @@ -81,7 +70,7 @@ static int wm_write (const data_set_t *ds, /* {{{ */ ssnprintf(collection_name, sizeof (collection_name), "collectd.%s", vl->plugin); bson_init(&record); - bson_append_time_t(&record,"ts",vl->time); + bson_append_time_t(&record,"ts",CDTIME_T_TO_TIME_T(vl->time)); bson_append_string(&record,"h",vl->host); bson_append_string(&record,"i",vl->plugin_instance); bson_append_string(&record,"t",vl->type); From 830b900e1f2473904f15eb29475a1f857a017ba1 Mon Sep 17 00:00:00 2001 From: Chris Lundquist Date: Tue, 24 Jan 2012 11:57:06 -0800 Subject: [PATCH 12/15] remove more commented out code --- src/write_mongodb.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/write_mongodb.c b/src/write_mongodb.c index f13cdc332b..23cca685ac 100644 --- a/src/write_mongodb.c +++ b/src/write_mongodb.c @@ -95,13 +95,6 @@ static int wm_write (const data_set_t *ds, /* {{{ */ if (node->connected == 0) { -/* - sstrncpy(node->opts->host, node->host, - sizeof (node->opts->host)); - node->opts->port = node->port; -*/ - - /* status = mongo_connect(node->conn,node->opts->host, node->opts->port);*/ status = mongo_connect(node->conn, node->host, node->port); if (status != MONGO_OK) { ERROR ("write_mongodb plugin: Connecting to host \"%s\" (port %i) failed.", From ca039cc92a09714cc2a4a1d05673bff34d395edc Mon Sep 17 00:00:00 2001 From: Florian Forster Date: Wed, 25 Jan 2012 11:24:28 +0100 Subject: [PATCH 13/15] write_mongodb plugin: Update copyright header. Change-Id: Ie939a7f3df1db9fc2aa3493028e3b3d74cdaeb61 --- AUTHORS | 6 ++++++ src/write_mongodb.c | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 0e0a5fedba..14e96be887 100644 --- a/AUTHORS +++ b/AUTHORS @@ -15,6 +15,9 @@ Sebastian "tokkee" Harl Contributors (sorted alphabetically) ==================================== +Akkarit Sangpetch + - write_mongodb plugin. + Alessandro Iurlano - Initial filecount plugin. @@ -44,6 +47,9 @@ Bruno Prémont especially a nasty bug in the network plugin. - Wireshark dissector. +Chris Lundquist + - Improvements to the write_mongodb plugin. + Christophe Kalt - The version 3 `log' mode. - Many Solaris related hints and fixes. diff --git a/src/write_mongodb.c b/src/write_mongodb.c index 23cca685ac..510caf90bb 100644 --- a/src/write_mongodb.c +++ b/src/write_mongodb.c @@ -1,6 +1,8 @@ /** * collectd - src/write_mongodb.c - * Copyright (C) 2010 Florian Forster, Akkarit Sangpetch + * Copyright (C) 2010 Florian Forster + * Copyright (C) 2010 Akkarit Sangpetch + * Copyright (C) 2012 Chris Lundquist * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -22,7 +24,8 @@ * * Authors: * Florian Forster - * Akkarit Sangpetch + * Akkarit Sangpetch + * Chris Lundquist **/ #include "collectd.h" From 44733bbe7d0585c9e459132053bd372ce0ba1630 Mon Sep 17 00:00:00 2001 From: Florian Forster Date: Wed, 25 Jan 2012 11:47:37 +0100 Subject: [PATCH 14/15] write_mongodb plugin: Fix the use of {CPP,LD}FLAGS. Change-Id: Ieb8467d3c1d0fac819864ff873594432170289d3 --- src/Makefile.am | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Makefile.am b/src/Makefile.am index 40ab538226..702ee0fe22 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1246,7 +1246,8 @@ endif if BUILD_PLUGIN_WRITE_MONGODB pkglib_LTLIBRARIES += write_mongodb.la write_mongodb_la_SOURCES = write_mongodb.c -write_mongodb_la_LDFLAGS = -module -avoid-version +write_mongodb_la_CPPFLAGS = $(AM_CPPFLAGS) $(BUILD_WITH_LIBMONGOC_CPPFLAGS) +write_mongodb_la_LDFLAGS = -module -avoid-version $(BUILD_WITH_LIBMONGOC_LDFLAGS) write_mongodb_la_LIBADD = -lmongoc collectd_LDADD += "-dlopen" write_mongodb.la collectd_DEPENDENCIES += write_mongodb.la From fc5c43d7db4f05aae907eec5b0c5121bcf650709 Mon Sep 17 00:00:00 2001 From: Florian Forster Date: Wed, 25 Jan 2012 11:51:13 +0100 Subject: [PATCH 15/15] write_mongodb plugin: Fix a format-string error. Change-Id: Ia18cfa8ea7ef8536957c0807112620d2e0e2a976 --- src/write_mongodb.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/write_mongodb.c b/src/write_mongodb.c index 510caf90bb..704ac07103 100644 --- a/src/write_mongodb.c +++ b/src/write_mongodb.c @@ -122,17 +122,12 @@ static int wm_write (const data_set_t *ds, /* {{{ */ if(status != MONGO_OK) { ERROR ( "write_mongodb plugin: error inserting record: %d", node->conn->err); - if(node->conn->err == MONGO_BSON_INVALID) - { - ERROR (node->conn->errstr); - } else if ( record.err) - { - ERROR (record.errstr); - } - + if (node->conn->err == MONGO_BSON_INVALID) + ERROR ("write_mongodb plugin: %s", node->conn->errstr); + else if (record.err) + ERROR ("write_mongodb plugin: %s", record.errstr); } - pthread_mutex_unlock (&node->lock); return (0);