Adding support for kinesis application metrics

- Adding another template support for pulling metrics for
    kinesis applications. Kinesis applications cannot be
    "discovered" so they can't be parameterized and they
    have to be known in advance. In other words, the namespace
    field has to be filled out. The template however can discover
    all the shards and streams that are available

Change-Id: I84793c40c3cbe3fd0c6800df58d2bad4bec58b76

plumbum.py

@@ -38,6 +38,7 @@
 import boto.rds
 import boto.elasticache
 import boto.ec2.autoscale
+import boto.kinesis
 import boto.sqs
 import jinja2
 
@@ -156,13 +157,27 @@ def list_sqs(region, filter_by_kwargs):
     return lookup(queues, filter_by=filter_by_kwargs)
 
 
+def list_kinesis_applications(region, filter_by_kwargs):
+    """List all the kinesis applications along with the shards for each stream"""
+    conn = boto.kinesis.connect_to_region(region)
+    streams = conn.list_streams()['StreamNames']
+    kinesis_streams = {}
+    for stream_name in streams:
+        shard_ids = []
+        shards = conn.describe_stream(stream_name)['StreamDescription']['Shards']
+        for shard in shards:
+            shard_ids.append(shard['ShardId'])
+        kinesis_streams[stream_name] = shard_ids
+    return kinesis_streams
+
 list_resources = {
     'ec2': list_ec2,
     'elb': list_elb,
     'rds': list_rds,
     'elasticache': list_elasticache,
     'asg': list_autoscaling_group,
     'sqs': list_sqs,
+    'kinesisapp': list_kinesis_applications,
 }
 
 

sample_templates/kinesisapp.yml.js

@@ -0,0 +1,31 @@
+{%- set metrics = {'DataBytesProcessed': {'stat': 'Average', 'unit': 'Bytes'},
+                   'KinesisDataFetcher.getRecords.Success': {'stat': 'Average', 'unit': 'Count'},
+                   'KinesisDataFetcher.getRecords.Time': {'stat': 'Average', 'unit': 'Milliseconds'},
+                   'MillisBehindLatest': {'stat': 'Average', 'unit': 'Milliseconds'},
+                   'RecordsProcessed': {'stat': 'Average', 'unit': 'Count'},
+                   'Success': {'stat': 'Average', 'unit': 'Count'},
+                   'Time': {'stat': 'Average', 'unit': 'Milliseconds'},
+                   'UpdateLease.Success': {'stat': 'Average', 'unit': 'Count'},
+                   'UpdateLease.Time': {'stat': 'Average', 'unit': 'Milliseconds'}
+                   } -%}
+
+# If connecting to a different region other than default, set region
+Auth:
+  region: "{{ region }}"
+Metrics:
+{%- for stream_name, shards in resources.iteritems() %}
+  {%- for shard in shards %}
+    {%- for metric in metrics %}
+- Namespace: "kinesis-application-name"
+  MetricName: "{{ metric }}"
+  Statistics: "{{ metrics[metric]['stat'] }}"
+  Unit: "{{ metrics[metric]['unit'] }}"
+  Dimensions:
+    ShardId: "{{ shard }}"
+    Operation: "ProcessTask"
+  Options:
+    Formatter: 'cloudwatch.{{ stream_name}}.%(Namespace)s.%(MetricName)s.{{ shard }}.%(statistic)s.%(Unit)s'
+    Period: 5
+     {% endfor %}
+   {% endfor %}
+{% endfor %}