From a971bcd26d0ebdfbe5f7f527fca2ea0d16803185 Mon Sep 17 00:00:00 2001 From: Shannon Carey Date: Fri, 30 Sep 2016 17:00:06 -0500 Subject: [PATCH] Render infinite values to valid JSON values. Fixes #813 - The 'json' and 'jsonp' formats output float('inf') as 1e9999, float ('-inf') as -1e9999, and float('nan') as null - The 'dygraph' format (a JSON-based format) outputs the same as Infinity, -Infinity, and null (see http://dygraphs .com/tests/gviz-infinity.html) - Minor updates to documents to make sure commands can be run without errors or problems --- CONTRIBUTING.md | 1 + requirements.txt | 6 ++- webapp/graphite/render/float_encoder.py | 59 +++++++++++++++++++++++++ webapp/graphite/render/views.py | 16 +++++-- webapp/tests/test_render.py | 58 +++++++++++++++++------- 5 files changed, 118 insertions(+), 22 deletions(-) create mode 100644 webapp/graphite/render/float_encoder.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e0a1ea5d1..6294627d6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,6 +32,7 @@ We are always looking to improve our test coverage. New tests will be appreciat If you see a mistake, have a feature, or a performance gain, we'd love to see it. It's _strongly_ encouraged for contributions that aren't already covered by tests to come with them. We're not trying to foist work on to you, but it makes it much easier for us to accept your contributions. +To set up your development environment, see the instructions in requirements.txt ### Documentation diff --git a/requirements.txt b/requirements.txt index 87386190f..d4ede9c79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,13 @@ # This is a PIP requirements file. # To setup a dev environment: # +# Use Python 2.7 +# # If you use virtualenvwrapper, you can use the misc/virtualenvwrapper hook scripts to # automate most of the following commands # # easy_install virtualenv -# virtualenv --distribute --no-site-packages --prompt "(graphite venv) " .venv +# virtualenv --distribute --no-site-packages --prompt "(graphite venv) " --python=/usr/bin/python2.7 .venv # source .venv/bin/activate # # brew install cairo && brew link cairo # on OSX @@ -23,7 +25,7 @@ # # mkdir -p .venv/storage/log/webapp /opt/graphite/storage/ # sudo chown -R $USER: /opt/graphite -# .venv/bin/django-admin.py syncdb --settings=graphite.settings --pythonpath=webapp +# .venv/bin/django-admin.py migrate --run-syncdb --settings=graphite.settings --pythonpath=webapp # bin/run-graphite-devel-server.py ./ # # or # # cd webapp/graphite && $GRAPHITE_ROOT/.venv/bin/gunicorn_django -b 127.0.0.1:8080 diff --git a/webapp/graphite/render/float_encoder.py b/webapp/graphite/render/float_encoder.py new file mode 100644 index 000000000..ac346a9d8 --- /dev/null +++ b/webapp/graphite/render/float_encoder.py @@ -0,0 +1,59 @@ +import json + + +class FloatEncoder(json.JSONEncoder): + def __init__(self, nan_str="null", **kwargs): + super(FloatEncoder, self).__init__(**kwargs) + self.nan_str = nan_str + + def iterencode(self, o, _one_shot=False): + """Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + """ + if self.check_circular: + markers = {} + else: + markers = None + if self.ensure_ascii: + _encoder = json.encoder.encode_basestring_ascii + else: + _encoder = json.encoder.encode_basestring + if self.encoding != 'utf-8': + def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): + if isinstance(o, str): + o = o.decode(_encoding) + return _orig_encoder(o) + + def floatstr(o, allow_nan=self.allow_nan, _repr=json.encoder.FLOAT_REPR, + _inf=json.encoder.INFINITY, _neginf=-json.encoder.INFINITY, + nan_str=self.nan_str): + # Check for specials. Note that this type of test is processor + # and/or platform-specific, so do tests which don't depend on the + # internals. + + if o != o: + text = nan_str + elif o == _inf: + text = '1e9999' + elif o == _neginf: + text = '-1e9999' + else: + return _repr(o) + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o)) + + return text + + _iterencode = json.encoder._make_iterencode( + markers, self.default, _encoder, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot) + return _iterencode(o, 0) diff --git a/webapp/graphite/render/views.py b/webapp/graphite/render/views.py index ff24daf4a..5312891a5 100644 --- a/webapp/graphite/render/views.py +++ b/webapp/graphite/render/views.py @@ -22,6 +22,7 @@ from urlparse import urlsplit, urlunsplit from cgi import parse_qs from cStringIO import StringIO + try: import cPickle as pickle except ImportError: @@ -36,6 +37,7 @@ from graphite.render.functions import PieFunctions from graphite.render.hashing import hashRequest, hashData from graphite.render.glyph import GraphTypes +from graphite.render.float_encoder import FloatEncoder from django.http import HttpResponseServerError, HttpResponseRedirect from django.template import Context, loader @@ -172,10 +174,10 @@ def renderView(request): if 'jsonp' in requestOptions: response = HttpResponse( - content="%s(%s)" % (requestOptions['jsonp'], json.dumps(series_data)), + content="%s(%s)" % (requestOptions['jsonp'], json.dumps(series_data, cls=FloatEncoder)), content_type='text/javascript') else: - response = HttpResponse(content=json.dumps(series_data), + response = HttpResponse(content=json.dumps(series_data, cls=FloatEncoder), content_type='application/json') if useCache: @@ -193,7 +195,15 @@ def renderView(request): for series in data: labels.append(series.name) for i, point in enumerate(series): - datapoints[i].append(point if point is not None else 'null') + if point is None: + point = 'null' + elif point == float('inf'): + point = 'Infinity' + elif point == float('-inf'): + point = '-Infinity' + elif math.isnan(point): + point = 'null' + datapoints[i].append(point) line_template = '[%%s000%s]' % ''.join([', %s'] * len(data)) lines = [line_template % tuple(points) for points in datapoints] result = '{"labels" : %s, "data" : [%s]}' % (json.dumps(labels), ', '.join(lines)) diff --git a/webapp/tests/test_render.py b/webapp/tests/test_render.py index eb0735a08..4c91ee2c9 100644 --- a/webapp/tests/test_render.py +++ b/webapp/tests/test_render.py @@ -2,6 +2,7 @@ import json import os import time +import math import logging import shutil @@ -90,43 +91,66 @@ def test_render_view(self): whisper.create(self.db, [(1, 60)]) ts = int(time.time()) - whisper.update(self.db, 0.1234567890123456789012, ts - 2) - whisper.update(self.db, 0.4, ts - 1) - whisper.update(self.db, 0.6, ts) + whisper.update(self.db, 0.1234567890123456789012, ts - 5) + whisper.update(self.db, 0.4, ts - 4) + whisper.update(self.db, 0.6, ts - 3) + whisper.update(self.db, float('inf'), ts - 2) + whisper.update(self.db, float('-inf'), ts - 1) + whisper.update(self.db, float('nan'), ts) response = self.client.get(url, {'target': 'test', 'format': 'raw'}) raw_data = ("None,None,None,None,None,None,None,None,None,None,None," "None,None,None,None,None,None,None,None,None,None,None," "None,None,None,None,None,None,None,None,None,None,None," "None,None,None,None,None,None,None,None,None,None,None," - "None,None,None,None,None,None,None,None,None,None,None," - "None,None,0.12345678901234568,0.4,0.6") + "None,None,None,None,None,None,None,None,None,None," + "0.12345678901234568,0.4,0.6,inf,-inf,nan") raw_response = "test,%d,%d,1|%s\n" % (ts-59, ts+1, raw_data) self.assertEqual(response.content, raw_response) response = self.client.get(url, {'target': 'test', 'format': 'json'}) + self.assertIn('[1e9999, ' + str(ts - 2) + ']', response.content) + self.assertIn('[-1e9999, ' + str(ts - 1) + ']', response.content) data = json.loads(response.content) - end = data[0]['datapoints'][-4:] + end = data[0]['datapoints'][-7:] self.assertEqual( - end, [[None, ts - 3], [0.12345678901234568, ts - 2], [0.4, ts - 1], [0.6, ts]]) + end, [[None, ts - 6], + [0.12345678901234568, ts - 5], + [0.4, ts - 4], + [0.6, ts - 3], + [float('inf'), ts - 2], + [float('-inf'), ts - 1], + [None, ts]]) response = self.client.get(url, {'target': 'test', 'format': 'dygraph'}) + self.assertIn('[' + str((ts - 2) * 1000) + ', Infinity]', response.content) + self.assertIn('[' + str((ts - 1) * 1000) + ', -Infinity]', response.content) data = json.loads(response.content) - end = data['data'][-4:] + end = data['data'][-7:] self.assertEqual(end, - [[(ts - 3) * 1000, None], - [(ts - 2) * 1000, 0.123456789012], - [(ts - 1) * 1000, 0.4], - [ts * 1000, 0.6]]) + [[(ts - 6) * 1000, None], + [(ts - 5) * 1000, 0.123456789012], + [(ts - 4) * 1000, 0.4], + [(ts - 3) * 1000, 0.6], + [(ts - 2) * 1000, float('inf')], + [(ts - 1) * 1000, float('-inf')], + [ts * 1000, None]]) response = self.client.get(url, {'target': 'test', 'format': 'rickshaw'}) data = json.loads(response.content) - end = data[0]['datapoints'][-4:] + end = data[0]['datapoints'][-7:-1] self.assertEqual(end, - [{'x': ts - 3, 'y': None}, - {'x': ts - 2, 'y': 0.12345678901234568}, - {'x': ts - 1, 'y': 0.4}, - {'x': ts, 'y': 0.6}]) + [{'x': ts - 6, 'y': None}, + {'x': ts - 5, 'y': 0.12345678901234568}, + {'x': ts - 4, 'y': 0.4}, + {'x': ts - 3, 'y': 0.6}, + {'x': ts - 2, 'y': float('inf')}, + {'x': ts - 1, 'y': float('-inf')}]) + + last = data[0]['datapoints'][-1] + self.assertEqual(last['x'], ts) + self.assertTrue(math.isnan(last['y'])) + def test_hash_request(self): # Requests with the same parameters should hash to the same values,