From 3c50ab99f74a0a88b1f181c9303a3751906c34b7 Mon Sep 17 00:00:00 2001 From: Arkadiusz Adamski Date: Fri, 25 Aug 2017 14:02:05 +0200 Subject: [PATCH] Push graphs to statsd (#3148) Add integration dashboard's graph with statsd --- docs/user/dashboards.md | 3 ++ src/ralph/dashboards/admin.py | 14 +++++++- src/ralph/dashboards/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../commands/push_graphs_to_statsd.py | 33 +++++++++++++++++++ .../migrations/0003_graph_push_to_statsd.py | 19 +++++++++++ src/ralph/dashboards/models.py | 4 +++ src/ralph/settings/base.py | 2 ++ src/ralph/settings/prod.py | 8 +++++ 9 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 src/ralph/dashboards/management/__init__.py create mode 100644 src/ralph/dashboards/management/commands/__init__.py create mode 100644 src/ralph/dashboards/management/commands/push_graphs_to_statsd.py create mode 100644 src/ralph/dashboards/migrations/0003_graph_push_to_statsd.py diff --git a/docs/user/dashboards.md b/docs/user/dashboards.md index d29d9a9274..26b90fb2d4 100644 --- a/docs/user/dashboards.md +++ b/docs/user/dashboards.md @@ -2,6 +2,9 @@ The dashboard provide basic mechanism for displaying data via bar or pie charts. +# Integration with statsd +Each graph can push data to statsd. You must add ``STATSD_GRAPHS_PREFIX`` to yours settings and set ``ALLOW_PUSH_GRAPHS_DATA_TO_STATSD`` and ``COLLECT_METRICS`` to ``True``. Next, check ``Push to statsd`` on concrete graph and use Ralph's management command ``push_graphs_to_statsd`` to push your data to statsd. + ## Getting started All example data in this tutorial was generated by Ralph's command - ``ralph make_demo_data``. diff --git a/src/ralph/dashboards/admin.py b/src/ralph/dashboards/admin.py index ab7bfec1ae..e4e93d17ae 100644 --- a/src/ralph/dashboards/admin.py +++ b/src/ralph/dashboards/admin.py @@ -2,6 +2,7 @@ from itertools import groupby from django import forms +from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.urlresolvers import reverse from django.utils.translation import ugettext as _ @@ -78,7 +79,8 @@ class GraphAdmin(RalphAdmin): 'aggregate_type', 'chart_type', 'params', - 'active' + 'active', + 'push_to_statsd', ) }), ('Preview', { @@ -86,6 +88,16 @@ class GraphAdmin(RalphAdmin): }), ) + def get_readonly_fields(self, *args, **kwargs): + readonly_fields = super().get_readonly_fields(*args, **kwargs) + allow_push_graphs_data_to_statsd = ( + not settings.ALLOW_PUSH_GRAPHS_DATA_TO_STATSD and + not settings.COLLECT_METRICS + ) + if allow_push_graphs_data_to_statsd: + readonly_fields.append('push_to_statsd') + return readonly_fields + def get_preview(self, obj): return obj.render(name='preview') diff --git a/src/ralph/dashboards/management/__init__.py b/src/ralph/dashboards/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/ralph/dashboards/management/commands/__init__.py b/src/ralph/dashboards/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/ralph/dashboards/management/commands/push_graphs_to_statsd.py b/src/ralph/dashboards/management/commands/push_graphs_to_statsd.py new file mode 100644 index 0000000000..350a1b2a6a --- /dev/null +++ b/src/ralph/dashboards/management/commands/push_graphs_to_statsd.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +import logging +import textwrap + +from django.conf import settings +from django.core.management.base import BaseCommand +from django.utils.text import slugify + +from ralph.dashboards.models import Graph +from ralph.lib.metrics import statsd + +logger = logging.getLogger(__name__) +PREFIX = settings.STATSD_GRAPHS_PREFIX +STATSD_PATH = '{}.{{}}.{{}}'.format(PREFIX) + + +def normalize(s): + s = slugify(s) + return s.replace('-', '_') + + +class Command(BaseCommand): + """Push to statsd data generated by graphs.""" + help = textwrap.dedent(__doc__).strip() + + def handle(self, *args, **kwargs): + graphs = Graph.objects.filter(push_to_statsd=True) + for graph in graphs: + graph_data = graph.get_data() + graph_name = normalize(graph.name) + for label, value in zip(graph_data['labels'], graph_data['series']): + path = STATSD_PATH.format(graph_name, normalize(label)) + statsd.gauge(path, value) diff --git a/src/ralph/dashboards/migrations/0003_graph_push_to_statsd.py b/src/ralph/dashboards/migrations/0003_graph_push_to_statsd.py new file mode 100644 index 0000000000..20b22378c7 --- /dev/null +++ b/src/ralph/dashboards/migrations/0003_graph_push_to_statsd.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dashboards', '0002_auto_20170509_1404'), + ] + + operations = [ + migrations.AddField( + model_name='graph', + name='push_to_statsd', + field=models.BooleanField(default=False, help_text="Push graph's data to statsd."), + ), + ] diff --git a/src/ralph/dashboards/models.py b/src/ralph/dashboards/models.py index 62863f9d8c..a2694f0cc9 100644 --- a/src/ralph/dashboards/models.py +++ b/src/ralph/dashboards/models.py @@ -126,6 +126,10 @@ class Graph(AdminAbsoluteUrlMixin, NamedMixin, TimeStampMixin, models.Model): chart_type = models.PositiveIntegerField(choices=ChartType()) params = JSONField(blank=True) active = models.BooleanField(default=True) + push_to_statsd = models.BooleanField( + default=False, + help_text='Push graph\'s data to statsd.' + ) def pop_annotate_filters(self, filters): annotate_filters = {} diff --git a/src/ralph/settings/base.py b/src/ralph/settings/base.py index 011765e285..a753a41894 100644 --- a/src/ralph/settings/base.py +++ b/src/ralph/settings/base.py @@ -533,3 +533,5 @@ def str_to_bool(s: str) -> bool: # METRICS COLLECT_METRICS = False +ALLOW_PUSH_GRAPHS_DATA_TO_STATSD = False +STATSD_GRAPHS_PREFIX = 'ralph.graphs' diff --git a/src/ralph/settings/prod.py b/src/ralph/settings/prod.py index ea76f4d67e..8200acf9a7 100644 --- a/src/ralph/settings/prod.py +++ b/src/ralph/settings/prod.py @@ -68,3 +68,11 @@ MIDDLEWARE_CLASSES = ( 'ralph.lib.metrics.middlewares.RequestMetricsMiddleware', ) + MIDDLEWARE_CLASSES + + ALLOW_PUSH_GRAPHS_DATA_TO_STATSD = bool_from_env( + 'ALLOW_PUSH_GRAPHS_DATA_TO_STATSD' + ) + if ALLOW_PUSH_GRAPHS_DATA_TO_STATSD: + STATSD_GRAPHS_PREFIX = os.environ.get( + 'STATSD_GRAPHS_PREFIX', 'ralph.graphs' + )