Merge pull request #38 from python-diamond/new_threading_model

New threading model

CHANGELOG

@@ -1,5 +1,9 @@
 This file is a log of any potentially user affecting changes to Diamond.
 
+2014-10-19 new_threading_model
+Collectors are no longer enabled via code. We only enable collectors via config
+files.
+
 2013-03-06 master 390f999274ed3bc9a31264377b48da432824333f
 Changes the layout of the postgresql collector metrics to be layed out correctly
 

README.md

@@ -28,6 +28,3 @@ Contact
 =====
  * IRC [#python-diamond](irc://chat.freenode.net:6667/%23python-diamond) on [freenode](http://www.freenode.net). [Webchat](http://webchat.freenode.net/?channels=python-diamond)
  * Mailing List [diamond@librelist.com](mailto:diamond@librelist.com) - Email the list and you will automatically subscribe. [Archive](http://librelist.com/browser/diamond/)
-
-[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/BrightcoveOS/diamond/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
-

bin/diamond

@@ -10,7 +10,6 @@ if os.name != 'nt':
 
 try:
     from setproctitle import setproctitle
-    setproctitle  # workaround for pyflakes issue #13
 except ImportError:
     setproctitle = None
 
@@ -24,10 +23,10 @@ for path in [
 
 from diamond.server import Server
 from diamond.util import get_diamond_version
+from diamond.utils.log import setup_logging
 
 import optparse
 import signal
-import logging.config
 
 
 def main():
@@ -114,44 +113,17 @@ def main():
             sys.exit(0)
 
         # Initialize Config
+        options.configfile = os.path.abspath(options.configfile)
         if os.path.exists(options.configfile):
-            config = configobj.ConfigObj(os.path.abspath(options.configfile))
-            config['configfile'] = options.configfile
+            config = configobj.ConfigObj(options.configfile)
         else:
             print >> sys.stderr, "ERROR: Config file: %s does not exist." % (
                 options.configfile)
             parser.print_help(sys.stderr)
             sys.exit(1)
 
         # Initialize Logging
-        log = logging.getLogger('diamond')
-
-        if options.log_stdout:
-            log.setLevel(logging.DEBUG)
-            # Configure Logging Format
-            formatter = logging.Formatter(
-                '[%(asctime)s] [%(threadName)s] %(message)s')
-            # handler
-            streamHandler = logging.StreamHandler(sys.stdout)
-            streamHandler.setFormatter(formatter)
-            streamHandler.setLevel(logging.DEBUG)
-            log.addHandler(streamHandler)
-        else:
-            try:
-                if sys.version_info >= (2, 6):
-                    logging.config.fileConfig(options.configfile,
-                                              disable_existing_loggers=False)
-                else:
-                    # python <= 2.5 does not have disable_existing_loggers
-                    # default was to always disable them, in our case we want to
-                    # keep any logger created by handlers
-                    logging.config.fileConfig(options.configfile)
-                    for logger in logging.root.manager.loggerDict.values():
-                        logger.disabled = 0
-            except Exception, e:
-                sys.stderr.write("Error occurs when initialize logging: ")
-                sys.stderr.write(str(e))
-                sys.stderr.write(os.linesep)
+        log = setup_logging(options.configfile, options.log_stdout)
 
     # Pass the exit up stream rather then handle it as an general exception
     except SystemExit, e:
@@ -281,6 +253,8 @@ def main():
                 os.close(0)
                 os.close(1)
                 os.close(2)
+                sys.stdout = open(os.devnull, 'w')
+                sys.stderr = open(os.devnull, 'w')
 
         # PID MANAGEMENT
         if not options.skip_pidfile:
@@ -299,29 +273,22 @@ def main():
                 log.debug("Wrote child PID file: %s" % (options.pidfile))
 
         # Initialize Server
-        server = Server(config)
+        server = Server(configfile=options.configfile)
 
         def sigint_handler(signum, frame):
+            log.info("Signal Received: %d" % (signum))
+            # Delete Pidfile
+            if not options.skip_pidfile and os.path.exists(options.pidfile):
+                os.remove(options.pidfile)
                 # Log
-                log.debug("Signal Received: %d" % (signum))
-                # Stop Server
-                server.stop()
-                # Delete Pidfile
-                if not options.skip_pidfile and os.path.exists(options.pidfile):
-                    os.remove(options.pidfile)
-                    # Log
-                    log.debug("Removed PID file: %s" % (options.pidfile))
+                log.debug("Removed PID file: %s" % (options.pidfile))
+            sys.exit(0)
 
         # Set the signal handlers
         signal.signal(signal.SIGINT, sigint_handler)
         signal.signal(signal.SIGTERM, sigint_handler)
 
-        if options.collector:
-            # Run Server with one collector
-            server.run_one(options.collector)
-        else:
-            # Run Server
-            server.run()
+        server.run()
 
     # Pass the exit up stream rather then handle it as an general exception
     except SystemExit, e:
@@ -335,5 +302,5 @@ def main():
 
 if __name__ == "__main__":
     if setproctitle:
-        setproctitle('diamond')
+        setproctitle(os.path.basename(__file__))
     main()

bin/diamond-setup

@@ -10,7 +10,6 @@ from configobj import ConfigObj
 
 try:
     from setproctitle import setproctitle
-    setproctitle  # workaround for pyflakes issue #13
 except ImportError:
     setproctitle = None
 
@@ -177,7 +176,6 @@ if __name__ == "__main__":
     # Initialize Config
     if os.path.exists(options.configfile):
         config = ConfigObj(os.path.abspath(options.configfile))
-        config['configfile'] = options.configfile
     else:
         print >> sys.stderr, "ERROR: Config file: %s does not exist." % (
             options.configfile)
@@ -239,12 +237,9 @@ if __name__ == "__main__":
             # Manage Keys
             config_keys['enabled'] = True
             config_keys['path'] = False
-            config_keys['method'] = False
             config_keys['path_prefix'] = False
             config_keys['instance_prefix'] = False
-            config_keys['splay'] = False
             config_keys['interval'] = False
-            config_keys['method'] = False
 
             print "\n\t\tNow configuring " + collector
             print collectors[collector].__doc__

build_doc.py

@@ -116,7 +116,6 @@ def getHandlers(path):
     # Initialize Config
     if os.path.exists(options.configfile):
         config = configobj.ConfigObj(os.path.abspath(options.configfile))
-        config['configfile'] = options.configfile
     else:
         print >> sys.stderr, "ERROR: Config file: %s does not exist." % (
             options.configfile)

conf/diamond.conf.example

@@ -29,11 +29,9 @@ collectors_config_path = /etc/diamond/collectors/
 # Directory to load handler configs from
 handlers_config_path = /etc/diamond/handlers/
 
+# Directory to load handler modules from
 handlers_path = /usr/share/diamond/handlers/
 
-# Interval to reload collectors
-collectors_reload_interval = 3600
-
 ################################################################################
 ### Options for handlers
 [handlers]
@@ -175,6 +173,28 @@ batch = 100
 # Default Poll Interval (seconds)
 # interval = 300
 
+################################################################################
+# Default enabled collectors
+################################################################################
+
+[[CPUCollector]]
+enabled = True
+
+[[DiskSpaceCollector]]
+enabled = True
+
+[[DiskUsageCollector]]
+enabled = True
+
+[[LoadAverageCollector]]
+enabled = True
+
+[[MemoryCollector]]
+enabled = True
+
+[[VMStatCollector]]
+enabled = True
+
 ################################################################################
 ### Options for logging
 # for more information on file format syntax:

conf/diamond.conf.example.windows

@@ -27,10 +27,10 @@ collectors_path = C:\\Program Files\\diamond\\collectors
 collectors_config_path = C:\\Program Files\\diamond\\conf\\collectors
 
 # Directory to load handler configs from
-handlers_config_path = C:\\Program Files\\diamond\\handlers
+handlers_config_path = C:\\Program Files\\diamond\\conf\\handlers
 
-# Interval to reload collectors
-collectors_reload_interval = 3600
+# Directory to load handler modules from
+handlers_path = C:\\Program Files\\diamond\\handlers
 
 ################################################################################
 ### Options for handlers
@@ -165,6 +165,28 @@ batch = 100
 # Default Poll Interval (seconds)
 # interval = 300
 
+################################################################################
+# Default enabled collectors
+################################################################################
+
+[[CPUCollector]]
+enabled = True
+
+[[DiskSpaceCollector]]
+enabled = True
+
+[[DiskUsageCollector]]
+enabled = True
+
+[[LoadAverageCollector]]
+enabled = True
+
+[[MemoryCollector]]
+enabled = True
+
+[[VMStatCollector]]
+enabled = True
+
 ################################################################################
 ### Options for logging
 # for more information on file format syntax:
@@ -200,4 +222,4 @@ args = ('C:\\Program Files\\diamond\\diamond.log', 'midnight', 1, 7)
 [formatter_default]
 
 format = [%(asctime)s] [%(threadName)s] %(message)s
-datefmt =
+datefmt =

setup.py

@@ -19,7 +19,6 @@ def running_under_virtualenv():
 
 if os.environ.get('USE_SETUPTOOLS'):
     from setuptools import setup
-    setup  # workaround for pyflakes issue #13
     setup_kwargs = dict(zip_safe=0)
 else:
     from distutils.core import setup
@@ -141,7 +140,7 @@ def pkgPath(root, path, rpath="/"):
     license='MIT License',
     description='Smart data producer for graphite graphing package',
     package_dir={'': 'src'},
-    packages=['diamond', 'diamond.handler'],
+    packages=['diamond', 'diamond.handler', 'diamond.utils'],
     scripts=['bin/diamond', 'bin/diamond-setup'],
     data_files=data_files,
     install_requires=install_requires,

src/collectors/beanstalkd/beanstalkd.py

@@ -16,7 +16,6 @@
 
 try:
     import beanstalkc
-    beanstalkc  # workaround for pyflakes issue #13
 except ImportError:
     beanstalkc = None
 

src/collectors/beanstalkd/test/testbeanstalkd.py

@@ -18,7 +18,6 @@
 def run_only_if_beanstalkc_is_available(func):
     try:
         import beanstalkc
-        beanstalkc  # workaround for pyflakes issue #13
     except ImportError:
         beanstalkc = None
     pred = lambda: beanstalkc is not None

src/collectors/bind/bind.py

@@ -16,7 +16,6 @@
 
 if sys.version_info >= (2, 5):
     import xml.etree.cElementTree as ElementTree
-    ElementTree  # workaround for pyflakes issue #13
 else:
     import cElementTree as ElementTree
 

src/collectors/celerymon/celerymon.py

@@ -23,7 +23,6 @@
 
 try:
     import json
-    json  # workaround for pyflakes issue #13
 except ImportError:
     import simplejson as json
 

src/collectors/celerymon/test/testcelerymon.py

@@ -5,11 +5,7 @@
 from test import CollectorTestCase
 from test import get_collector_config
 from test import unittest
-from test import run_only
-from mock import Mock
-from mock import patch
 
-from diamond.collector import Collector
 from celerymon import CelerymonCollector
 
 

src/collectors/ceph/ceph.py

@@ -14,7 +14,6 @@
 
 try:
     import json
-    json  # workaround for pyflakes issue #13
 except ImportError:
     import simplejson as json
 

src/collectors/ceph/test/testceph.py

@@ -3,7 +3,6 @@
 
 try:
     import json
-    json  # workaround for pyflakes issue #13
 except ImportError:
     import simplejson as json
 

src/collectors/conntrack/test/testconntrack.py

@@ -11,12 +11,6 @@
 from diamond.collector import Collector
 from conntrack import ConnTrackCollector
 
-try:
-    from cStringIO import StringIO
-    StringIO  # workaround for pyflakes issue #13
-except ImportError:
-    from StringIO import StringIO
-
 ################################################################################
 
 

src/collectors/cpu/cpu.py

@@ -16,7 +16,6 @@
 
 try:
     import psutil
-    psutil  # workaround for pyflakes issue #13
 except ImportError:
     psutil = None
 
@@ -54,7 +53,6 @@ def get_default_config(self):
         """
         config = super(CPUCollector, self).get_default_config()
         config.update({
-            'enabled':  'True',
             'path':     'cpu',
             'percore':  'True',
             'xenfix':   None,

src/collectors/cpu/test/testcpu.py

@@ -10,7 +10,6 @@
 
 try:
     from cStringIO import StringIO
-    StringIO  # workaround for pyflakes issue #13
 except ImportError:
     from StringIO import StringIO
 

src/collectors/cpuacct_cgroup/test/testcpuacct_cgroup.py

@@ -5,12 +5,10 @@
 from test import CollectorTestCase
 from test import get_collector_config
 from test import unittest
-from mock import Mock
 from mock import patch
 
 try:
     from cStringIO import StringIO
-    StringIO  # workaround for pyflakes issue #13
 except ImportError:
     from StringIO import StringIO