[apptags] Added apptag support to dd-agent

The agent now sends one "dd_check:appname" tag to dogweb for each
running check (including the JMX checks) if the user enables the
create_dd_check_tags option. This enables slice/dice by app in the
backend.

The running JMX apps list is retrieved from the {tmp}/jmx_status.yaml
file. Indeed, these tags have to be sent from the collectors side, along
with host-tags and this file happens to be the only communication
channel between JMXFetch.jar and the agent.

checks/collector.py

@@ -1,18 +1,19 @@
 # Core modules
 import os
 import re
-import logging
-import subprocess
 import sys
 import time
-import datetime
 import socket
+import logging
+import datetime
+import subprocess
 
 import modules
 
+from config import get_version, get_system_stats, get_config
 from util import get_os, get_uuid, md5, Timer, get_hostname, EC2, GCE
-from config import get_version, get_system_stats
 
+import jmxfetch
 import checks.system.unix as u
 import checks.system.win32 as w32
 from checks import create_service_check, AgentCheck
@@ -28,6 +29,7 @@
 
 FLUSH_LOGGING_PERIOD = 10
 FLUSH_LOGGING_INITIAL = 5
+APPTAG = 'dd_check:{0}'
 
 class Collector(object):
     """
@@ -368,7 +370,7 @@ def run(self, checksd=None, start_event=True):
             payload['metrics'].extend(self._agent_metrics.check(payload, self.agentConfig,
                 collect_duration, self.emit_duration))
 
-
+        # Let's send our payload 
         emitter_statuses = self._emit(payload)
         self.emit_duration = timer.step()
 
@@ -474,16 +476,23 @@ def _build_payload(self, start_event=True):
             if self.agentConfig['collect_ec2_tags']:
                 host_tags.extend(EC2.get_tags(self.agentConfig))
 
+            # If required by the user, let's create the dd_check:xxx host tags
+            if self.agentConfig['create_dd_check_tags']:
+                app_tags_list = [APPTAG.format(c.name) for c in self.initialized_checks_d]
+                app_tags_list.extend([APPTAG.format(cname) for cname in jmxfetch._get_jmx_appnames()])
+
+                host_tags.extend(app_tags_list)
+
             if host_tags:
                 payload['host-tags']['system'] = host_tags
 
             GCE_tags = GCE.get_tags(self.agentConfig)
             if GCE_tags is not None:
                 payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags
 
-            # Log the metadata on the first run
-            if self._is_first_run():
-                log.info("Hostnames: %s, tags: %s" % (repr(self.metadata_cache), payload['host-tags']))
+                # Log the metadata on the first run
+                if self._is_first_run():
+                    log.info("Hostnames: %s, tags: %s" % (repr(self.metadata_cache), payload['host-tags']))
 
         # Periodically send extra hosts metadata (vsphere)
         # Metadata of hosts that are not the host where the agent runs, not all the checks use
@@ -536,5 +545,3 @@ def _should_send_additional_data(self, data_name):
             return True
 
         return False
-
-

config.py

@@ -426,8 +426,12 @@ def get_config(parse_args=True, cfg_path=None, options=None):
                 agentConfig[key] = config.get('Main', key)
             else:
                 agentConfig[key] = value
+
+        # Create app:xxx tags based on monitored apps
+        agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags')\
+                and _is_affirmative(config.get('Main', 'create_dd_check_tags'))
 
-        #Forwarding to external statsd server
+        # Forwarding to external statsd server
         if config.has_option('Main', 'statsd_forward_host'):
             agentConfig['statsd_forward_host'] = config.get('Main', 'statsd_forward_host')
             if config.has_option('Main', 'statsd_forward_port'):

jmxfetch.py

@@ -344,6 +344,21 @@ def _get_path_to_jmxfetch(self):
                                 "jmxfetch", JMX_FETCH_JAR_NAME))
 
 
+def _get_jmx_appnames():
+    """ 
+    Retrieves the running JMX checks based on the {tmp}/jmx_status.yaml file 
+    updated by JMXFetch (and the only communication channel between JMXFetch
+    and the collector since JMXFetch).
+    """
+    check_names = []
+    jmx_status_path = os.path.join(get_jmx_status_path(), "jmx_status.yaml")
+    if os.path.exists(jmx_status_path):
+        jmx_checks = yaml.load(file(jmx_status_path)).get('checks', {})
+        for name, instances in jmx_checks.get('initialized_checks', {}).iteritems():
+            check_names.append(name)
+    return check_names
+
+
 def init(config_path=None):
     agentConfig = get_config(parse_args=False, cfg_path=config_path)
     osname = get_os()

tests/test_common.py

@@ -156,6 +156,7 @@ def test_collector(self):
             'check_timings': True,
             'collect_ec2_tags': True,
             'collect_instance_metadata': False,
+            'create_dd_check_tags': False,
             'version': 'test',
             'tags': '',
         }