Merge pull request #1205 from graphite-project/graphite-templates

Add template() function to parameterize seriesList arguments

docs/render_api.rst

@@ -203,6 +203,43 @@ Examples:
   &from=monday
   (show data since the previous monday)
 
+template
+--------
+
+The ``target`` metrics can use a special ``template`` function which
+allows the metric paths to contain variables. Values for these variables
+can be provided via the ``template`` query parameter.
+
+Examples
+^^^^^^^^
+
+Example:
+
+.. code-block:: none
+
+  &target=template(hosts.$hostname.cpu)&template[hostname]=worker1
+
+Default values for the template variables can also be provided:
+
+.. code-block:: none
+
+  &target=template(hosts.$hostname.cpu, hostname="worker1")
+
+Positional arguments can be used instead of named ones:
+
+.. code-block:: none
+
+  &target=template(hosts.$1.cpu, "worker1")
+  &target=template(hosts.$1.cpu, "worker1")&template[1]=worker*
+
+In addition to path substitution, variables can be used for numeric and string literals:
+
+.. code-block:: none
+
+  &target=template(constantLine($number))&template[number]=123
+  &target=template(sinFunction($name))&template[name]=nameOfMySineWaveMetric
+
+
 Data Display Formats
 ====================
 

webapp/graphite/render/evaluator.py

@@ -1,3 +1,4 @@
+import re
 from graphite.render.grammar import grammar
 from graphite.render.datalib import fetchData, TimeSeries
 
@@ -13,17 +14,45 @@ def evaluateTarget(requestContext, target):
     return result
 
 
-def evaluateTokens(requestContext, tokens):
-  if tokens.expression:
-    return evaluateTokens(requestContext, tokens.expression)
+def evaluateTokens(requestContext, tokens, replacements=None):
+  if tokens.template:
+    arglist = dict()
+    if tokens.template.kwargs:
+      arglist.update(dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0])) for kwarg in tokens.template.kwargs]))
+    if tokens.template.args:
+      arglist.update(dict([(str(i+1), evaluateTokens(requestContext, arg)) for i, arg in enumerate(tokens.template.args)]))
+    if 'template' in requestContext:
+      arglist.update(requestContext['template'])
+    return evaluateTokens(requestContext, tokens.template, arglist)
+
+  elif tokens.expression:
+    return evaluateTokens(requestContext, tokens.expression, replacements)
 
   elif tokens.pathExpression:
-    return fetchData(requestContext, tokens.pathExpression)
+    expression = tokens.pathExpression
+    if replacements:
+      for name in replacements:
+        if expression == '$'+name:
+          val = replacements[name]
+          if not isinstance(val, str) and not isinstance(val, basestring):
+            return val
+          elif re.match('^-?[\d.]+$', val):
+            return float(val)
+          else:
+            return val
+        else:
+          expression = expression.replace('$'+name, str(replacements[name]))
+    return fetchData(requestContext, expression)
 
   elif tokens.call:
+    if tokens.call.funcname == 'template':
+      # if template propagates down here, it means the grammar didn't match the invocation
+      # as tokens.template. this generally happens if you try to pass non-numeric/string args
+      raise ValueError("invaild template() syntax, only string/numeric arguments are allowed")
+
     func = SeriesFunctions[tokens.call.funcname]
-    args = [evaluateTokens(requestContext, arg) for arg in tokens.call.args]
-    kwargs = dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0]))
+    args = [evaluateTokens(requestContext, arg, replacements) for arg in tokens.call.args]
+    kwargs = dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0], replacements))
                    for kwarg in tokens.call.kwargs])
     try:
       return func(requestContext, *args, **kwargs)
@@ -44,6 +73,9 @@ def evaluateTokens(requestContext, tokens):
   elif tokens.boolean:
     return tokens.boolean[0] == 'true'
 
+  else:
+    raise ValueError("unknown token in target evaulator")
+
 
 # Avoid import circularities
 from graphite.render.functions import (SeriesFunctions,

webapp/graphite/render/grammar.py

@@ -100,11 +100,25 @@
 )
 pathExpression = delimitedList(pathElement, delim='.', combine=True)('pathExpression')
 
+litarg = Group(
+  number | aString
+)('args*')
+litkwarg = Group(argname + equal + litarg)('kwargs*')
+litargs = delimitedList(~litkwarg + litarg)  # lookahead to prevent failing on equals
+litkwargs = delimitedList(litkwarg)
+
+template = Group(
+  Literal('template') + leftParen +
+  (call | pathExpression) +
+  Optional(comma + (litargs | litkwargs)) +
+  rightParen
+)('template')
+
 if __version__.startswith('1.'):
-    expression << Group(call | pathExpression)('expression')
+    expression << Group(template | call | pathExpression)('expression')
     grammar << expression
 else:
-    expression <<= Group(call | pathExpression)('expression')
+    expression <<= Group(template | call | pathExpression)('expression')
     grammar <<= expression
 
 

webapp/graphite/render/views.py

@@ -54,6 +54,7 @@ def renderView(request):
     'startTime' : requestOptions['startTime'],
     'endTime' : requestOptions['endTime'],
     'localOnly' : requestOptions['localOnly'],
+    'template' : requestOptions['template'],
     'data' : []
   }
   data = requestContext['data']
@@ -252,6 +253,12 @@ def parseOptions(request):
   for target in mytargets:
     requestOptions['targets'].append(target)
 
+  template = dict()
+  for key, val in queryParams.items():
+    if key.startswith("template["):
+      template[key[9:-1]] = val
+  requestOptions['template'] = template
+
   if 'pickle' in queryParams:
     requestOptions['format'] = 'pickle'
   if 'rawData' in queryParams:

webapp/tests/test_render.py

@@ -3,6 +3,7 @@
 import os
 import time
 import logging
+import shutil
 
 from graphite.render.hashing import hashRequest, hashData
 import whisper
@@ -22,13 +23,38 @@
 
 class RenderTest(TestCase):
     db = os.path.join(settings.WHISPER_DIR, 'test.wsp')
+    hostcpu = os.path.join(settings.WHISPER_DIR, 'hosts/hostname/cpu.wsp')
 
     def wipe_whisper(self):
         try:
             os.remove(self.db)
         except OSError:
             pass
 
+    def create_whisper_hosts(self):
+        worker1 = self.hostcpu.replace('hostname', 'worker1')
+        worker2 = self.hostcpu.replace('hostname', 'worker2')
+        try:
+            os.makedirs(worker1.replace('cpu.wsp', ''))
+            os.makedirs(worker2.replace('cpu.wsp', ''))
+        except OSError:
+            pass
+
+        whisper.create(worker1, [(1, 60)])
+        whisper.create(worker2, [(1, 60)])
+
+        ts = int(time.time())
+        whisper.update(worker1, 1, ts)
+        whisper.update(worker2, 2, ts)
+
+    def wipe_whisper_hosts(self):
+        try:
+            os.remove(self.hostcpu.replace('hostname', 'worker1'))
+            os.remove(self.hostcpu.replace('hostname', 'worker2'))
+            shutil.rmtree(self.hostcpu.replace('hostname/cpu.wsp', ''))
+        except OSError:
+            pass
+
     def test_render_view(self):
         url = reverse('graphite.render.views.renderView')
 
@@ -132,3 +158,101 @@ def test_correct_timezone(self):
         # all the from/until/tz combinations lead to the same window
         expected = [[12, 1393398060], [12, 1393401660]]
         self.assertEqual(data, expected)
+
+    def test_template_numeric_variables(self):
+        url = reverse('graphite.render.views.renderView')
+        response = self.client.get(url, {
+                 'target': 'template(constantLine($1),12)',
+                 'format': 'json',
+                 'from': '07:01_20140226',
+                 'until': '08:01_20140226',
+        })
+        data = json.loads(response.content)[0]['datapoints']
+        # all the from/until/tz combinations lead to the same window
+        expected = [[12, 1393398060], [12, 1393401660]]
+        self.assertEqual(data, expected)
+
+        url = reverse('graphite.render.views.renderView')
+        response = self.client.get(url, {
+                 'target': 'template(constantLine($num),num=12)',
+                 'format': 'json',
+                 'from': '07:01_20140226',
+                 'until': '08:01_20140226',
+        })
+        data = json.loads(response.content)[0]['datapoints']
+        # all the from/until/tz combinations lead to the same window
+        expected = [[12, 1393398060], [12, 1393401660]]
+        self.assertEqual(data, expected)
+
+        url = reverse('graphite.render.views.renderView')
+        response = self.client.get(url, {
+                 'target': 'template(constantLine($num))',
+                 'format': 'json',
+                 'from': '07:01_20140226',
+                 'until': '08:01_20140226',
+                 'template[num]': '12',
+        })
+        data = json.loads(response.content)[0]['datapoints']
+        # all the from/until/tz combinations lead to the same window
+        expected = [[12, 1393398060], [12, 1393401660]]
+        self.assertEqual(data, expected)
+
+    def test_template_string_variables(self):
+        url = reverse('graphite.render.views.renderView')
+        response = self.client.get(url, {
+                 'target': 'template(time($1),"nameOfSeries")',
+                 'format': 'json',
+                 'from': '07:01_20140226',
+                 'until': '08:01_20140226',
+        })
+        data = json.loads(response.content)[0]
+        self.assertEqual(data['target'], 'nameOfSeries')
+
+        url = reverse('graphite.render.views.renderView')
+        response = self.client.get(url, {
+                 'target': 'template(time($name),name="nameOfSeries")',
+                 'format': 'json',
+                 'from': '07:01_20140226',
+                 'until': '08:01_20140226',
+        })
+        data = json.loads(response.content)[0]
+        self.assertEqual(data['target'], 'nameOfSeries')
+
+        url = reverse('graphite.render.views.renderView')
+        response = self.client.get(url, {
+                 'target': 'template(time($name))',
+                 'format': 'json',
+                 'from': '07:01_20140226',
+                 'until': '08:01_20140226',
+                 'template[name]': 'nameOfSeries',
+        })
+        data = json.loads(response.content)[0]
+        self.assertEqual(data['target'], 'nameOfSeries')
+
+    def test_template_pathExpression_variables(self):
+        self.create_whisper_hosts()
+        self.addCleanup(self.wipe_whisper_hosts)
+
+        url = reverse('graphite.render.views.renderView')
+        response = self.client.get(url, {
+                 'target': 'template(sumSeries(hosts.$1.cpu),"worker1")',
+                 'format': 'json',
+        })
+        data = json.loads(response.content)[0]
+        self.assertEqual(data['target'], 'sumSeries(hosts.worker1.cpu)')
+
+        response = self.client.get(url, {
+                 'target': 'template(sumSeries(hosts.$1.cpu),"worker1")',
+                 'format': 'json',
+                 'template[1]': 'worker*'
+        })
+        data = json.loads(response.content)[0]
+        self.assertEqual(data['target'], 'sumSeries(hosts.worker*.cpu)')
+
+        response = self.client.get(url, {
+                 'target': 'template(sumSeries(hosts.$hostname.cpu))',
+                 'format': 'json',
+                 'template[hostname]': 'worker*'
+        })
+        data = json.loads(response.content)[0]
+        self.assertEqual(data['target'], 'sumSeries(hosts.worker*.cpu)')