Support running a collector multiple times concurrently

BrightcoveOS · Dec 21, 2014 · c251180 · c251180
1 parent d429c52
commit c251180
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 12 deletions.
diff --git a/src/diamond/collector.py b/src/diamond/collector.py
@@ -171,9 +171,10 @@ def __init__(self, config=None, handlers=[], name=None, configfile=None):
             self.name = self.__class__.__name__
         else:
             self.name = name
+
         self.handlers = handlers
         self.last_values = {}
-        
+
         self.configfile = None
         self.load_config(configfile, config)
 
@@ -190,22 +191,22 @@ def load_config(self, configfile=None, override_config=None):
 
         if configfile is not None:
             self.configfile = os.path.abspath(configfile)
-            
+
         if self.configfile is not None:
             config = load_config(self.configfile)
-    
+
             if 'collectors' in config:
                 if 'default' in config['collectors']:
                     self.config.merge(config['collectors']['default'])
-        
+
                 if self.name in config['collectors']:
                     self.config.merge(config['collectors'][self.name])
-        
+
         if override_config is not None:
             if 'collectors' in override_config:
                 if 'default' in override_config['collectors']:
                     self.config.merge(override_config['collectors']['default'])
-        
+
                 if self.name in override_config['collectors']:
                     self.config.merge(override_config['collectors'][self.name])
 

diff --git a/src/diamond/server.py b/src/diamond/server.py
@@ -157,21 +157,27 @@ def run(self):
                             process.terminate()
 
                 for process_name in running_collectors - running_processes:
-                    if 'Collector' not in process_name:
+                    # To handle running multiple collectors concurrently, we
+                    # split on white space and use the first word as the
+                    # collector name to spin
+                    collector_name = process_name.split()[0]
+
+                    if 'Collector' not in collector_name:
                         continue
+
                     # Find the class
                     for cls in collectors.values():
                         cls_name = cls.__name__.split('.')[-1]
-                        if cls_name == process_name:
+                        if cls_name == collector_name:
                             break
-                    if cls_name != process_name:
+                    if cls_name != collector_name:
                         self.log.error('Can not find collector %s',
-                                       process_name)
+                                       collector_name)
                         continue
 
                     collector = initialize_collector(
                         cls,
-                        name=cls.__name__,
+                        name=process_name,
                         configfile=self.configfile,
                         handlers=[self.handler_queue])
 
@@ -184,7 +190,7 @@ def run(self):
                     time.sleep(1)
 
                     process = multiprocessing.Process(
-                        name=collector.__class__.__name__,
+                        name=process_name,
                         target=collector_process,
                         args=(collector, self.metric_queue, self.log)
                         )