diff --git a/webapp/content/js/composer_widgets.js b/webapp/content/js/composer_widgets.js index a30bb30b3..d8fc9b3e2 100644 --- a/webapp/content/js/composer_widgets.js +++ b/webapp/content/js/composer_widgets.js @@ -1100,7 +1100,8 @@ function createFunctionsMenu() { {text: 'timeShift', handler: applyFuncToEachWithInput('timeShift', 'Shift this metric ___ back in time (examples: 10min, 7d, 2w)', {quote: true})}, {text: 'timeSlice', handler: applyFuncToEachWithInput('timeSlice', 'Start showing metric at (example: 14:57 20150115)', {quote: true})}, {text: 'Summarize', handler: applyFuncToEachWithInput('summarize', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})}, - {text: 'Hit Count', handler: applyFuncToEachWithInput('hitcount', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})} + {text: 'Hit Count', handler: applyFuncToEachWithInput('hitcount', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})}, + {text: 'Smooth', handler: applyFuncToEachWithInput('smooth', 'Please enter the number of pixels to average for each point')} ] }, { text: 'Calculate', @@ -1114,7 +1115,8 @@ function createFunctionsMenu() { {text: 'Linear Regression', handler: applyFuncToEachWithInput('linearRegression', 'Start source of regression at (example: 14:57 20150115)', {quote: true})}, {text: 'As Percent', handler: applyFuncToEachWithInput('asPercent', 'Please enter the value that corresponds to 100% or leave blank to use the total', {allowBlank: true})}, {text: 'Difference (of 2 series)', handler: applyFuncToAll('diffSeries')}, - {text: 'Ratio (of 2 series)', handler: applyFuncToAll('divideSeries')} + {text: 'Ratio (of 2 series)', handler: applyFuncToAll('divideSeries')}, + {text: 'Resample', handler: applyFuncToEachWithInput('resample', 'Please enter the desired points-per-pixel for the new resolution')} ] }, { text: 'Filter', diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index f6bbc60cb..6d12de986 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -1096,6 +1096,108 @@ def consolidateBy(requestContext, seriesList, consolidationFunc): series.pathExpression = series.name return seriesList +def resample(requestContext, seriesList, pointsPerPx = 1): + """ + Resamples the given series according to the requested graph width and + $pointsPerPx aggregating by average. Total number of points after this + function == graph width * pointsPerPx. + + This has two significant uses: + + * Drastically speeds up render time when graphing high resolution data + or many metrics. + * Allows movingAverage() to have a consistent smoothness across timescales. + * Example: movingAverage(resample(metric,2),20) would end up with + a 10px moving average no matter what the scale of your graph. + * Allows a consistent number-of-samples to be returned from JSON requests + * the number of samples returned == graph width * points per pixel + + Example: + + .. code-block:: none + + &target=resample(metric, 2) + &target=movingAverage(resample(metric, 2), 20) + """ + newSampleCount = requestContext['width'] + + for seriesIndex, series in enumerate(seriesList): + newValues = [] + seriesLength = (series.end - series.start) + newStep = (float(seriesLength) / float(newSampleCount)) / float(pointsPerPx) + + # Leave this series alone if we're asked to do upsampling + if newStep < series.step: + continue + + sampleWidth = 0 + sampleCount = 0 + sampleSum = 0 + + for value in series: + if (value is not None): + sampleCount += 1 + sampleSum += value + sampleWidth += series.step + + # If the current sample covers the width of a new step, add it to the + # result + if (sampleWidth >= newStep): + if sampleCount > 0: + newValues.append(sampleSum / sampleCount) + else: + newValues.append(None) + sampleWidth -= newStep + sampleSum = 0 + sampleCount = 0 + + # Process and add the left-over sample if it's not empty + if sampleCount > 0: + newValues.append(sampleSum / sampleCount) + + newName = "resample(%s, %s)" % (series.name, pointsPerPx) + newSeries = TimeSeries(newName, series.start, series.end, newStep, newValues) + newSeries.pathExpression = newName + seriesList[seriesIndex] = newSeries + + return seriesList + + +def smooth(requestContext, seriesList, windowPixelSize = 5): + """ + Resample and smooth a set of metrics. Provides line smoothing that is + independent of time scale (windowPixelSize ~ movingAverage over pixels). + + A shorter and safer way of calling: + movingAverage(resample(seriesList, 2), smoothFactor * 2) + + The windowPixelSize is effectively the number of pixels over which to perform + the movingAverage (default: 5). + + Note: This is safer in that if a series has fewer data points than pixels, + the metric won't be upsampled. Instead the movingAverage window size will be + adjusted to cover the same number of pixels. + """ + pointsPerPixel = 2 + resampled = resample(requestContext, seriesList, pointsPerPixel) + + sampleSize = int(windowPixelSize * pointsPerPixel) + expectedSamples = requestContext['width'] * pointsPerPixel + + for index, series in enumerate(resampled): + # if we have fewer samples than expected, adjust the movingAverage sample + # size so it covers the same number of pixels + if (len(series) < expectedSamples * 0.95): + movingAverageSize = int((float(len(series)) / (expectedSamples)) * sampleSize) + else: + movingAverageSize = sampleSize + + newSeries = movingAverage(requestContext, [series], movingAverageSize)[0] + newSeries.name = "smooth(%s, %i)" % (series.name, windowPixelSize) + resampled[index] = newSeries + + return resampled + def derivative(requestContext, seriesList): """ This is the opposite of the integral function. This is useful for taking a @@ -3750,6 +3852,8 @@ def pieMinimum(requestContext, series): 'summarize': summarize, 'smartSummarize': smartSummarize, 'hitcount': hitcount, + 'resample' : resample, + 'smooth' : smooth, 'absolute': absolute, 'interpolate': interpolate, diff --git a/webapp/graphite/render/views.py b/webapp/graphite/render/views.py index 3ef9b256e..a81d71862 100644 --- a/webapp/graphite/render/views.py +++ b/webapp/graphite/render/views.py @@ -56,6 +56,8 @@ def renderView(request): 'localOnly' : requestOptions['localOnly'], 'template' : requestOptions['template'], 'tzinfo' : requestOptions['tzinfo'], + 'width' : graphOptions['width'], + 'height' : graphOptions['height'], 'data' : [] } data = requestContext['data'] diff --git a/webapp/tests/test_functions.py b/webapp/tests/test_functions.py index 7d59d2027..c81ac415a 100644 --- a/webapp/tests/test_functions.py +++ b/webapp/tests/test_functions.py @@ -866,6 +866,38 @@ def test_consolidateBy(self): results = functions.consolidateBy({}, seriesList, func) self._verify_series_consolidationFunc(results, func) + def test_smooth(self): + seriesList = [ + TimeSeries('collectd.test-db1.load.value',0,1,1,[range(20)]), + ] + + def mock_evaluateTokens(reqCtx, tokens, replacements=None): + seriesList = [ + TimeSeries('collectd.test-db1.load.value',0,1,1,[range(20)]), + ] + for series in seriesList: + series.pathExpression = series.name + return seriesList + + expectedResults = [ + TimeSeries('smooth(collectd.test-db1.load.value, 5)',0,1,1,[None,None,None,None,2,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]) + ] + + with patch('graphite.render.functions.evaluateTokens', mock_evaluateTokens): + result = functions.smooth( + { + 'template': {}, + 'args': ({},{}), + 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.timezone(settings.TIME_ZONE)), + 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.timezone(settings.TIME_ZONE)), + 'localOnly': False, + 'width': 400, + 'data': [] + }, + seriesList, 5 + ) + self.assertEqual(result, expectedResults) + def test_weightedAverage(self): seriesList = [ TimeSeries('collectd.test-db1.load.value',0,1,1,[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]),