Permalink
Browse files

Adding support for kinesis application metrics

- Adding another template support for pulling metrics for
    kinesis applications. Kinesis applications cannot be
    "discovered" so they can't be parameterized and they
    have to be known in advance. In other words, the namespace
    field has to be filled out. The template however can discover
    all the shards and streams that are available

Change-Id: I84793c40c3cbe3fd0c6800df58d2bad4bec58b76
  • Loading branch information...
1 parent 914488a commit bd9fa0eb56169e1325d4d5361cfe6ffc9a77d016 @gneokleo gneokleo committed Aug 20, 2015
Showing with 46 additions and 0 deletions.
  1. +15 −0 plumbum.py
  2. +31 −0 sample_templates/kinesisapp.yml.js
View
@@ -38,6 +38,7 @@
import boto.rds
import boto.elasticache
import boto.ec2.autoscale
+import boto.kinesis
import boto.sqs
import jinja2
@@ -156,13 +157,27 @@ def list_sqs(region, filter_by_kwargs):
return lookup(queues, filter_by=filter_by_kwargs)
+def list_kinesis_applications(region, filter_by_kwargs):
+ """List all the kinesis applications along with the shards for each stream"""
+ conn = boto.kinesis.connect_to_region(region)
+ streams = conn.list_streams()['StreamNames']
+ kinesis_streams = {}
+ for stream_name in streams:
+ shard_ids = []
+ shards = conn.describe_stream(stream_name)['StreamDescription']['Shards']
+ for shard in shards:
+ shard_ids.append(shard['ShardId'])
+ kinesis_streams[stream_name] = shard_ids
+ return kinesis_streams
+
list_resources = {
'ec2': list_ec2,
'elb': list_elb,
'rds': list_rds,
'elasticache': list_elasticache,
'asg': list_autoscaling_group,
'sqs': list_sqs,
+ 'kinesisapp': list_kinesis_applications,
}
@@ -0,0 +1,31 @@
+{%- set metrics = {'DataBytesProcessed': {'stat': 'Average', 'unit': 'Bytes'},
+ 'KinesisDataFetcher.getRecords.Success': {'stat': 'Average', 'unit': 'Count'},
+ 'KinesisDataFetcher.getRecords.Time': {'stat': 'Average', 'unit': 'Milliseconds'},
+ 'MillisBehindLatest': {'stat': 'Average', 'unit': 'Milliseconds'},
+ 'RecordsProcessed': {'stat': 'Average', 'unit': 'Count'},
+ 'Success': {'stat': 'Average', 'unit': 'Count'},
+ 'Time': {'stat': 'Average', 'unit': 'Milliseconds'},
+ 'UpdateLease.Success': {'stat': 'Average', 'unit': 'Count'},
+ 'UpdateLease.Time': {'stat': 'Average', 'unit': 'Milliseconds'}
+ } -%}
+
+# If connecting to a different region other than default, set region
+Auth:
+ region: "{{ region }}"
+Metrics:
+{%- for stream_name, shards in resources.iteritems() %}
+ {%- for shard in shards %}
+ {%- for metric in metrics %}
+- Namespace: "kinesis-application-name"
+ MetricName: "{{ metric }}"
+ Statistics: "{{ metrics[metric]['stat'] }}"
+ Unit: "{{ metrics[metric]['unit'] }}"
+ Dimensions:
+ ShardId: "{{ shard }}"
+ Operation: "ProcessTask"
+ Options:
+ Formatter: 'cloudwatch.{{ stream_name}}.%(Namespace)s.%(MetricName)s.{{ shard }}.%(statistic)s.%(Unit)s'
+ Period: 5
+ {% endfor %}
+ {% endfor %}
+{% endfor %}

0 comments on commit bd9fa0e

Please sign in to comment.