port smooth() and resample() functions from graphite-project#92

webapp/content/js/composer_widgets.js

@@ -1100,7 +1100,8 @@ function createFunctionsMenu() {
         {text: 'timeShift', handler: applyFuncToEachWithInput('timeShift', 'Shift this metric ___ back in time (examples: 10min, 7d, 2w)', {quote: true})},
         {text: 'timeSlice', handler: applyFuncToEachWithInput('timeSlice', 'Start showing metric at (example: 14:57 20150115)', {quote: true})},
         {text: 'Summarize', handler: applyFuncToEachWithInput('summarize', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})},
-        {text: 'Hit Count', handler: applyFuncToEachWithInput('hitcount', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})}
+        {text: 'Hit Count', handler: applyFuncToEachWithInput('hitcount', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})},
+        {text: 'Smooth', handler: applyFuncToEachWithInput('smooth', 'Please enter the number of pixels to average for each point')}
       ]
     }, {
       text: 'Calculate',
@@ -1114,7 +1115,8 @@ function createFunctionsMenu() {
         {text: 'Linear Regression', handler: applyFuncToEachWithInput('linearRegression', 'Start source of regression at (example: 14:57 20150115)', {quote: true})},
         {text: 'As Percent', handler: applyFuncToEachWithInput('asPercent', 'Please enter the value that corresponds to 100% or leave blank to use the total', {allowBlank: true})},
         {text: 'Difference (of 2 series)', handler: applyFuncToAll('diffSeries')},
-        {text: 'Ratio (of 2 series)', handler: applyFuncToAll('divideSeries')}
+        {text: 'Ratio (of 2 series)', handler: applyFuncToAll('divideSeries')},
+        {text: 'Resample', handler: applyFuncToEachWithInput('resample', 'Please enter the desired points-per-pixel for the new resolution')}
       ]
     }, {
       text: 'Filter',

webapp/graphite/render/functions.py

@@ -1096,6 +1096,108 @@ def consolidateBy(requestContext, seriesList, consolidationFunc):
     series.pathExpression = series.name
   return seriesList
 
+def resample(requestContext, seriesList, pointsPerPx = 1):
+  """
+  Resamples the given series according to the requested graph width and
+  $pointsPerPx aggregating by average. Total number of points after this
+  function == graph width * pointsPerPx.
+
+  This has two significant uses:
+
+   * Drastically speeds up render time when graphing high resolution data
+     or many metrics.
+   * Allows movingAverage() to have a consistent smoothness across timescales.
+     * Example:  movingAverage(resample(metric,2),20) would end up with
+       a 10px moving average no matter what the scale of your graph.
+   * Allows a consistent number-of-samples to be returned from JSON requests
+     * the number of samples returned == graph width * points per pixel
+
+  Example:
+
+  .. code-block:: none
+
+    &target=resample(metric, 2)
+    &target=movingAverage(resample(metric, 2), 20)
+  """
+  newSampleCount = requestContext['width']
+
+  for seriesIndex, series in enumerate(seriesList):
+    newValues = []
+    seriesLength = (series.end - series.start)
+    newStep = (float(seriesLength) / float(newSampleCount)) / float(pointsPerPx)
+
+    # Leave this series alone if we're asked to do upsampling
+    if newStep < series.step:
+      continue
+
+    sampleWidth = 0
+    sampleCount = 0
+    sampleSum = 0
+
+    for value in series:
+      if (value is not None):
+        sampleCount += 1
+        sampleSum += value
+      sampleWidth += series.step
+
+      # If the current sample covers the width of a new step, add it to the
+      # result
+      if (sampleWidth >= newStep):
+        if sampleCount > 0:
+          newValues.append(sampleSum / sampleCount)
+        else:
+          newValues.append(None)
+        sampleWidth -= newStep
+        sampleSum = 0
+        sampleCount = 0
+
+    # Process and add the left-over sample if it's not empty
+    if sampleCount > 0:
+      newValues.append(sampleSum / sampleCount)
+
+    newName = "resample(%s, %s)" % (series.name, pointsPerPx)
+    newSeries = TimeSeries(newName, series.start, series.end, newStep, newValues)
+    newSeries.pathExpression = newName
+    seriesList[seriesIndex] = newSeries
+
+  return seriesList
+
+
+def smooth(requestContext, seriesList, windowPixelSize = 5):
+  """
+  Resample and smooth a set of metrics. Provides line smoothing that is
+  independent of time scale (windowPixelSize ~ movingAverage over pixels).
+
+  A shorter and safer way of calling:
+     movingAverage(resample(seriesList, 2), smoothFactor * 2)
+
+  The windowPixelSize is effectively the number of pixels over which to perform
+  the movingAverage (default: 5).
+
+  Note: This is safer in that if a series has fewer data points than pixels,
+  the metric won't be upsampled.  Instead the movingAverage window size will be
+  adjusted to cover the same number of pixels.
+  """
+  pointsPerPixel = 2
+  resampled = resample(requestContext, seriesList, pointsPerPixel)
+
+  sampleSize = int(windowPixelSize * pointsPerPixel)
+  expectedSamples = requestContext['width'] * pointsPerPixel
+
+  for index, series in enumerate(resampled):
+    # if we have fewer samples than expected, adjust the movingAverage sample
+    # size so it covers the same number of pixels
+    if (len(series) < expectedSamples * 0.95):
+      movingAverageSize = int((float(len(series)) / (expectedSamples)) * sampleSize)
+    else:
+      movingAverageSize = sampleSize
+
+    newSeries = movingAverage(requestContext, [series], movingAverageSize)[0]
+    newSeries.name = "smooth(%s, %i)" % (series.name, windowPixelSize)
+    resampled[index] = newSeries
+
+  return resampled
+
 def derivative(requestContext, seriesList):
   """
   This is the opposite of the integral function.  This is useful for taking a
@@ -3750,6 +3852,8 @@ def pieMinimum(requestContext, series):
   'summarize': summarize,
   'smartSummarize': smartSummarize,
   'hitcount': hitcount,
+  'resample'  : resample,
+  'smooth' : smooth,
   'absolute': absolute,
   'interpolate': interpolate,
 

webapp/graphite/render/views.py

@@ -56,6 +56,8 @@ def renderView(request):
     'localOnly' : requestOptions['localOnly'],
     'template' : requestOptions['template'],
     'tzinfo' : requestOptions['tzinfo'],
+    'width' : graphOptions['width'],
+    'height' : graphOptions['height'],
     'data' : []
   }
   data = requestContext['data']

webapp/tests/test_functions.py

@@ -866,6 +866,38 @@ def test_consolidateBy(self):
             results = functions.consolidateBy({}, seriesList, func)
             self._verify_series_consolidationFunc(results, func)
 
+    def test_smooth(self):
+        seriesList = [
+            TimeSeries('collectd.test-db1.load.value',0,1,1,[range(20)]),
+        ]
+
+        def mock_evaluateTokens(reqCtx, tokens, replacements=None):
+            seriesList = [
+                TimeSeries('collectd.test-db1.load.value',0,1,1,[range(20)]),
+            ]
+            for series in seriesList:
+                series.pathExpression = series.name
+            return seriesList
+
+        expectedResults = [
+            TimeSeries('smooth(collectd.test-db1.load.value, 5)',0,1,1,[None,None,None,None,2,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20])
+        ]
+
+        with patch('graphite.render.functions.evaluateTokens', mock_evaluateTokens):
+            result = functions.smooth(
+                {
+                    'template': {},
+                    'args': ({},{}),
+                    'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.timezone(settings.TIME_ZONE)),
+                    'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.timezone(settings.TIME_ZONE)),
+                    'localOnly': False,
+                    'width': 400,
+                    'data': []
+                },
+                seriesList, 5
+            )
+        self.assertEqual(result, expectedResults)
+
     def test_weightedAverage(self):
         seriesList = [
             TimeSeries('collectd.test-db1.load.value',0,1,1,[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]),