Permalink
Browse files

Finished README, added License, and various fix ups in the plugin.

  • Loading branch information...
1 parent 7d2f585 commit f91b843c4dafce6c1e816537392e47fb3b6be381 @mheffner mheffner committed Dec 3, 2011
Showing with 505 additions and 32 deletions.
  1. +202 −0 LICENSE
  2. +216 −0 README.md
  3. +87 −32 lib/collectd-librato.py
View
202 LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright 2011 Librato, Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
View
216 README.md
@@ -0,0 +1,216 @@
+# Introduction
+
+collectd-librato is a [collectd](http://www.collectd.org/) plugin that
+publishes collectd values to [Librato
+Metrics][https://metrics.librato.com] using the Librato Metrics
+[API][http://dev.librato.com]. Librato Metrics is a hosted, time-series
+data service.
+
+Collectd-librato was largely influenced by
+[collectd-carbon][https://github.com/indygreg/collectd-carbon].
+
+# Requirements
+
+* Collectd 4.9 or later (for the Python plugin) (A patch may be required
+ to fix the Python plugin - see below)
+* Python 2.4 or later
+* An active Librato Metrics account (sign up
+ [here][https://metrics.librato.com/sign_up]).
+
+# Configuration
+
+The plugin requires some configuration. This is done by passing
+parameters via the <Module> config section in your Collectd config.
+
+The following parameters are required:
+
+* Email - The email address associated with your Librato Metrics
+ account.
+* APIToken - The API token for you Librato Metrics account. This value
+ can be found your [account
+ page][https://metrics.librato.com/account].
+
+The following parameters are optional:
+
+* TypesDB - file(s) defining your Collectd types. This should be the
+ sames as your TypesDB global config parameters. This will default to
+ the file `/usr/share/collectd/types.db`. **NOTE**: This plugin will
+ not work if it can't find the types.db file.
+* LowercaseMetricNames - If preset, all metric names will be converted
+ to lower-case (default no lower-casing).
+* MetricPrefix - If present, all metric names will contain this string
+ prefix. Do not include a trailing period or separation character
+ (see MetricSeparator). Set to the empty string to disable any
+ prefix. Defaults to "collectd".
+* MetricSeparator - String to separate the components of a metric name
+ when combining the plugin name, type, and instance name. Defaults to
+ a period (".").
+* IncludeSingleValueNames - Normally, any metric type listed in
+ `types.db` that only has a single value will not have the name of
+ the value suffixed onto the metric name. For most single value
+ metrics the name is simply a placeholder like "value" or "count", so
+ adding it to the metric name does not add any particular value. If
+ `IncludeSingleValueNames` is set however, these value names will be
+ suffixed onto the metric name regardless.
+* FlushIntervalSecs - This value determines how frequently metrics are
+ flushed to Librato Metrics. For each collectd write request the
+ plugin will check if it has been FlushIntervalSecs seconds since the
+ last flush and if so will POST all metrics to Librato. Internally
+ there is a hard limit on the maximum number of metrics that the
+ plugin will buffer before a flush is forced which may supersede the
+ FlushIntervalSecs. The default FlushIntervalSecs is 30 seconds.
+
+## Supported Metrics
+
+Collectd-Librato currently supports the following collectd metric
+types:
+
+* GAUGE - Reported as a Librato Metric
+ [gauge][http://dev.librato.com/v1/gauges].
+* COUNTER - Reported as a Librato Metric
+ [counter][http://dev.librato.com/v1/counters].
+
+Other metric types are currently ignored. This list will be expanded
+in the future.
+
+## Example
+
+The following is an example Collectd configuration for this plugin:
+
+ <LoadPlugin "python">
+ Globals true
+ </LoadPlugin>
+
+ <Plugin "python">
+ # collectd-librato.py is at /opt/collectd-librato-0.0.1/lib/collectd-librato.py
+ ModulePath "/opt/collectd-librato-0.0.1/lib"
+
+ Import "collectd-librato"
+
+ <Module "collectd-librato">
+ APIToken "1985481910fe29ab201302011054857292"
+ Email "joe@example.com"
+ </Module>
+ </Plugin>
+
+# Operational Notes
+
+This plugin uses a best-effort attempt to deliver metrics to Librato
+Metrics. If a flush fails to POST metrics to Librato Metrics the flush
+will not currently be retried, but instead dropped. In most cases this
+should not happen, but if it does the plugin will continue to flush
+metrics after the failure. So in the worst case there may appear a
+short gap in your metric graphs.
+
+The plugin needs to parse Collectd type files. If there was an error
+parsing a specific type (look for log messages at Collectd startup
+time), the plugin will fail to write values for this type. It will
+simply skip over them and move on to the next value. It will write a log
+message every time this happens so you can correct the problem.
+
+The plugin needs to perform redundant parsing of the type files because
+the Collectd Python API does not provide an interface to the types
+information (unlike the Perl and Java plugin APIs). Hopefully this will
+be addressed in a future version of Collectd.
+
+# Data Mangling
+
+Collectd data is collected/written in discrete tuples having the
+following:
+
+ (host, plugin, plugin_instance, type, type_instance, time, interval, metadata, values)
+
+_values_ is itself a list of { counter, gauge, derive, absolute }
+(numeric) values. To further complicate things, each distinct _type_ has
+its own definition corresponding to what's in the _values_ field.
+
+Librato Metrics, by contrast, deals with tuples of:
+
+ (source, metric_name, value, measurement_time)
+
+So we effectively have to mangle the collectd tuple down to the fields
+above.
+
+The `source` is simply set to the *host* field of the collectd
+tuple. The plugin mangles the remaining fields of the collectd tuple
+to the following Librato Metrics `metric_name`:
+
+ [metric_prefix.]plugin[.plugin_instance].type[.type_instance].data_source
+
+Where *data_source* is the name of the data source (i.e. ds_name) in
+the type being written. In the case that the plugin data source only
+has a single value, the *data_source* is not included in the name
+(unless `IncludeSingleValueNames` is set).
+
+For example, the Collectd distribution has a built-in _df_ type:
+
+ df used:GAUGE:0:1125899906842623, free:GAUGE:0:1125899906842623
+
+The *data_source* values for this type would be *used* and *free*
+yielding the metric names (along the lines of)
+*collectd.plugin.df.used* and *collectd.plugin.df.free*.
+
+# Troubleshooting
+
+## Collectd Python Write Callback Bug
+
+Collectd versions through 4.10.2 and 4.9.4 have a bug in the Python
+plugin where Python would receive bad values for certain data
+sets. The bug would typically manifest as data values appearing to be
+0. The *collectd-carbon* author identified the bug and sent a fix to
+the Collectd development team.
+
+Collectd versions 4.9.5, 4.10.3, and 5.0.0 are the first official
+versions with a fix for this bug. If you are not running one of these
+versions or have not applied the fix (which can be seen at
+<https://github.com/indygreg/collectd/commit/31bc4bc67f9ae12fb593e18e0d3649e5d4fa13f2>),
+you will likely dispatch wrong values to Librato Metrics.
+
+## Collectd 4.10.3 on EL5 ImportError
+
+Using the plugin with collectd on EPEL5 on RHEL or CentOS 5.x may
+produce the following error:
+
+ Jul 20 14:54:38 mon0 collectd[2487]: plugin_load_file: The global flag is not supported, libtool 2 is required for this.
+ Jul 20 14:54:38 mon0 collectd[2487]: python plugin: Error importing module "collectd_librato".
+ Jul 20 14:54:38 mon0 collectd[2487]: Unhandled python exception in importing module: ImportError: /usr/lib64/python2.4/lib-dynload/_socketmodule.so: undefined symbol: PyExc_ValueError
+ Jul 20 14:54:38 mon0 collectd[2487]: python plugin: Found a configuration for the "collectd_librato" plugin, but the plugin isn't loaded or didn't register a configuration callback.
+ Jul 20 14:54:38 mon0 collectd[2488]: plugin_dispatch_values: No write callback has been registered. Please load at least one output plugin, if you want the collected data to be stored.
+ Jul 20 14:54:38 mon0 collectd[2488]: Filter subsystem: Built-in target `write': Dispatching value to all write plugins failed with status 2 (ENOENT). Most likely this means you didn't load any write plugins.
+
+This may also occur on other operating systems and collectd
+versions. It is caused by a libtool/libltdl quirk described in
+[this mailing list
+thread](http://mailman.verplant.org/pipermail/collectd/2008-March/001616.html).
+As per the workarounds detailed there, you may either:
+
+ 1. Modify the init script `/etc/init.d/collectd` to preload the
+ libpython shared library:
+
+ @@ -25,7 +25,7 @@
+ echo -n $"Starting $prog: "
+ if [ -r "$CONFIG" ]
+ then
+ - daemon /usr/sbin/collectd -C "$CONFIG"
+ + LD_PRELOAD=/usr/lib64/libpython2.4.so daemon /usr/sbin/collectd -C "$CONFIG"
+ RETVAL=$?
+ echo
+ [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog
+
+ 1. Modify the RPM and rebuild.
+
+ @@ -182,7 +182,7 @@
+
+
+ %build
+ -%configure \
+ +%configure CFLAGS=-"DLT_LAZY_OR_NOW='RTLD_LAZY|RTLD_GLOBAL'" \
+ --disable-static \
+ --disable-ascent \
+ --disable-apple_sensors \
+
+# Contributing
+
+If you would like to contribute a fix or feature to this plugin please
+feel free to fork this repo, make your change and submit us a pull
+request!
View
119 lib/collectd-librato.py
@@ -1,3 +1,18 @@
+# Copyright 2011 Librato, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
import collectd
import errno
import json
@@ -7,38 +22,30 @@
import sys
import base64
from string import maketrans
+from copy import copy
version = "0.0.1"
#config = { 'url' : 'https://metrics-api.librato.com/v1/metrics.json' }
-plugin_name = 'collectd-librato.py'
-config = { 'url' : 'http://localhost:9292/v1/metrics.json',
+config = { 'url' : 'https://metrics-api-stg.librato.com/v1/metrics.json',
'types_db' : '/usr/share/collectd/types.db',
'metric_prefix' : 'collectd',
'metric_separator' : '.',
- 'flush_interval_secs' : 10,
+ 'flush_interval_secs' : 30,
'flush_max_measurements' : 600,
- 'flush_timeout_secs' : 15
+ 'flush_timeout_secs' : 15,
+ 'lower_case' : False,
+ 'single_value_names' : False
}
+plugin_name = 'Collectd-Librato.py'
types = {}
-def build_user_agent():
- try:
- uname = os.uname()
- system = "; ".join([uname[0], uname[4]])
- except:
- system = os.name()
-
- pver = sys.version_info
- user_agent = 'Collectd-Librato.py/%s (%s) Python-Urllib2/%d.%d' % \
- (version, system, pver.major, pver.minor)
- return user_agent
+def str_to_num(s):
+ """
+ Convert type limits from strings to floats for arithmetic.
+ """
-def build_http_auth():
- base64string = base64.encodestring('%s:%s' % \
- (config['email'],
- config['api_token']))[:-1]
- return base64string
+ return float(s)
def get_time():
"""
@@ -97,18 +104,53 @@ def librato_parse_types_file(path):
f.close()
+def build_user_agent():
+ try:
+ uname = os.uname()
+ system = "; ".join([uname[0], uname[4]])
+ except:
+ system = os.name()
+
+ pver = sys.version_info
+ user_agent = '%s/%s (%s) Python-Urllib2/%d.%d' % \
+ (plugin_name, version, system, pver.major, pver.minor)
+ return user_agent
+
+def build_http_auth():
+ base64string = base64.encodestring('%s:%s' % \
+ (config['email'],
+ config['api_token']))
+ return base64string.translate(None, '\n')
+
def librato_config(c):
global config
for child in c.children:
+ val = child.values[0]
+
if child.key == 'APIToken':
- config['api_token'] = child.values[0]
+ config['api_token'] = val
elif child.key == 'Email':
- config['email'] = child.values[0]
+ config['email'] = val
elif child.key == 'MetricPrefix':
- config['metric_prefix'] = child.values[0]
+ config['metric_prefix'] = val
elif child.key == 'TypesDB':
- collectd.warning("typesdb = %s" % child.values[0])
+ config['types_db'] = val
+ elif child.key == 'MetricPrefix':
+ config['metric_prefix'] = val
+ elif child.key == 'MetricSeparator':
+ config['metric_separator'] = val
+ elif child.key == 'LowercaseMetricNames':
+ config['lower_case'] = True
+ elif child.key == 'IncludeSingleValueNames':
+ config['single_value_names'] = True
+ elif child.key == 'FlushIntervalSecs':
+ try:
+ config['flush_interval_secs'] = int(str_to_num(val))
+ except:
+ msg = '%s: Invalid value for FlushIntervalSecs: %s' % \
+ (plugin_name, val)
+ raise Exception(msg)
if not config.has_key('api_token'):
raise Exception('APIToken not defined')
@@ -157,13 +199,12 @@ def librato_queue_measurements(gauges, counters, data):
last_flush = curr_time - data['last_flush_time']
length = len(data['gauges']) + len(data['counters'])
- if last_flush < config['flush_interval_secs'] and \
- length < config['flush_max_measurements']:
+ if (last_flush < config['flush_interval_secs'] and \
+ length < config['flush_max_measurements']) or \
+ length == 0:
data['lock'].release()
return
- collectd.warning("flushing, last_flush: %d" % (last_flush))
-
flush_gauges = data['gauges']
flush_counters = data['counters']
data['gauges'] = []
@@ -205,20 +246,29 @@ def librato_write(v, data=None):
gauges = []
counters = []
- i = 0
- for value in v.values:
+ for i in range(len(v.values)):
+ value = v.values[i]
ds_name = v_type[i][0]
ds_type = v_type[i][1]
+ # We only support Gauges and Counters at this time
if ds_type != 'GAUGE' and ds_type != 'COUNTER':
continue
# Can value be None?
if value is None:
continue
+ name_tuple = copy(name)
+ if len(v.values) > 1 or config['single_value_names']:
+ name_tuple.append(ds_name)
+
+ metric_name = config['metric_separator'].join(name_tuple)
+ if config['lower_case']:
+ metric_name = metric_name.lower()
+
measurement = {
- 'name' : config['metric_separator'].join(name + [ds_name]),
+ 'name' : metric_name,
'source' : v.host,
'measure_time' : int(v.time),
'value' : value
@@ -234,7 +284,12 @@ def librato_write(v, data=None):
def librato_init():
import threading
- librato_parse_types_file(config['types_db'])
+ try:
+ librato_parse_types_file(config['types_db'])
+ except:
+ msg = '%s: ERROR: Unable to open TypesDB file: %s.' % \
+ (plugin_name, config['types_db'])
+ raise Exception(msg)
d = {
'lock' : threading.Lock(),

0 comments on commit f91b843

Please sign in to comment.