Permalink
Browse files

Render infinite values to valid JSON values. Fixes #813

- The 'json' and 'jsonp' formats output float('inf') as 1e9999, float
('-inf') as -1e9999, and float('nan') as null
- The 'dygraph' format (a JSON-based format) outputs the same as
Infinity, -Infinity, and null (see http://dygraphs
.com/tests/gviz-infinity.html)
- Minor updates to documents to make sure commands can be run without errors or problems
  • Loading branch information...
1 parent 6811d20 commit a971bcd26d0ebdfbe5f7f527fca2ea0d16803185 @rehevkor5 rehevkor5 committed Sep 30, 2016
Showing with 118 additions and 22 deletions.
  1. +1 −0 CONTRIBUTING.md
  2. +4 −2 requirements.txt
  3. +59 −0 webapp/graphite/render/float_encoder.py
  4. +13 −3 webapp/graphite/render/views.py
  5. +41 −17 webapp/tests/test_render.py
View
@@ -32,6 +32,7 @@ We are always looking to improve our test coverage. New tests will be appreciat
If you see a mistake, have a feature, or a performance gain, we'd love to see it. It's _strongly_ encouraged for contributions that aren't already covered by tests to come with them. We're not trying to foist work on to you, but it makes it much easier for us to accept your contributions.
+To set up your development environment, see the instructions in requirements.txt
### Documentation
View
@@ -1,11 +1,13 @@
# This is a PIP requirements file.
# To setup a dev environment:
#
+# Use Python 2.7
+#
# If you use virtualenvwrapper, you can use the misc/virtualenvwrapper hook scripts to
# automate most of the following commands
#
# easy_install virtualenv
-# virtualenv --distribute --no-site-packages --prompt "(graphite venv) " .venv
+# virtualenv --distribute --no-site-packages --prompt "(graphite venv) " --python=/usr/bin/python2.7 .venv
# source .venv/bin/activate
#
# brew install cairo && brew link cairo # on OSX
@@ -23,7 +25,7 @@
#
# mkdir -p .venv/storage/log/webapp /opt/graphite/storage/
# sudo chown -R $USER: /opt/graphite
-# .venv/bin/django-admin.py syncdb --settings=graphite.settings --pythonpath=webapp
+# .venv/bin/django-admin.py migrate --run-syncdb --settings=graphite.settings --pythonpath=webapp
# bin/run-graphite-devel-server.py ./
# # or
# # cd webapp/graphite && $GRAPHITE_ROOT/.venv/bin/gunicorn_django -b 127.0.0.1:8080
@@ -0,0 +1,59 @@
+import json
+
+
+class FloatEncoder(json.JSONEncoder):
+ def __init__(self, nan_str="null", **kwargs):
+ super(FloatEncoder, self).__init__(**kwargs)
+ self.nan_str = nan_str
+
+ def iterencode(self, o, _one_shot=False):
+ """Encode the given object and yield each string
+ representation as available.
+
+ For example::
+
+ for chunk in JSONEncoder().iterencode(bigobject):
+ mysocket.write(chunk)
+ """
+ if self.check_circular:
+ markers = {}
+ else:
+ markers = None
+ if self.ensure_ascii:
+ _encoder = json.encoder.encode_basestring_ascii
+ else:
+ _encoder = json.encoder.encode_basestring
+ if self.encoding != 'utf-8':
+ def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
+ if isinstance(o, str):
+ o = o.decode(_encoding)
+ return _orig_encoder(o)
+
+ def floatstr(o, allow_nan=self.allow_nan, _repr=json.encoder.FLOAT_REPR,
+ _inf=json.encoder.INFINITY, _neginf=-json.encoder.INFINITY,
+ nan_str=self.nan_str):
+ # Check for specials. Note that this type of test is processor
+ # and/or platform-specific, so do tests which don't depend on the
+ # internals.
+
+ if o != o:
+ text = nan_str
+ elif o == _inf:
+ text = '1e9999'
+ elif o == _neginf:
+ text = '-1e9999'
+ else:
+ return _repr(o)
+
+ if not allow_nan:
+ raise ValueError(
+ "Out of range float values are not JSON compliant: " +
+ repr(o))
+
+ return text
+
+ _iterencode = json.encoder._make_iterencode(
+ markers, self.default, _encoder, self.indent, floatstr,
+ self.key_separator, self.item_separator, self.sort_keys,
+ self.skipkeys, _one_shot)
+ return _iterencode(o, 0)
@@ -22,6 +22,7 @@
from urlparse import urlsplit, urlunsplit
from cgi import parse_qs
from cStringIO import StringIO
+
try:
import cPickle as pickle
except ImportError:
@@ -36,6 +37,7 @@
from graphite.render.functions import PieFunctions
from graphite.render.hashing import hashRequest, hashData
from graphite.render.glyph import GraphTypes
+from graphite.render.float_encoder import FloatEncoder
from django.http import HttpResponseServerError, HttpResponseRedirect
from django.template import Context, loader
@@ -172,10 +174,10 @@ def renderView(request):
if 'jsonp' in requestOptions:
response = HttpResponse(
- content="%s(%s)" % (requestOptions['jsonp'], json.dumps(series_data)),
+ content="%s(%s)" % (requestOptions['jsonp'], json.dumps(series_data, cls=FloatEncoder)),
content_type='text/javascript')
else:
- response = HttpResponse(content=json.dumps(series_data),
+ response = HttpResponse(content=json.dumps(series_data, cls=FloatEncoder),
content_type='application/json')
if useCache:
@@ -193,7 +195,15 @@ def renderView(request):
for series in data:
labels.append(series.name)
for i, point in enumerate(series):
- datapoints[i].append(point if point is not None else 'null')
+ if point is None:
+ point = 'null'
+ elif point == float('inf'):
+ point = 'Infinity'
+ elif point == float('-inf'):
+ point = '-Infinity'
+ elif math.isnan(point):
+ point = 'null'
+ datapoints[i].append(point)
line_template = '[%%s000%s]' % ''.join([', %s'] * len(data))
lines = [line_template % tuple(points) for points in datapoints]
result = '{"labels" : %s, "data" : [%s]}' % (json.dumps(labels), ', '.join(lines))
@@ -2,6 +2,7 @@
import json
import os
import time
+import math
import logging
import shutil
@@ -90,43 +91,66 @@ def test_render_view(self):
whisper.create(self.db, [(1, 60)])
ts = int(time.time())
- whisper.update(self.db, 0.1234567890123456789012, ts - 2)
- whisper.update(self.db, 0.4, ts - 1)
- whisper.update(self.db, 0.6, ts)
+ whisper.update(self.db, 0.1234567890123456789012, ts - 5)
+ whisper.update(self.db, 0.4, ts - 4)
+ whisper.update(self.db, 0.6, ts - 3)
+ whisper.update(self.db, float('inf'), ts - 2)
+ whisper.update(self.db, float('-inf'), ts - 1)
+ whisper.update(self.db, float('nan'), ts)
response = self.client.get(url, {'target': 'test', 'format': 'raw'})
raw_data = ("None,None,None,None,None,None,None,None,None,None,None,"
"None,None,None,None,None,None,None,None,None,None,None,"
"None,None,None,None,None,None,None,None,None,None,None,"
"None,None,None,None,None,None,None,None,None,None,None,"
- "None,None,None,None,None,None,None,None,None,None,None,"
- "None,None,0.12345678901234568,0.4,0.6")
+ "None,None,None,None,None,None,None,None,None,None,"
+ "0.12345678901234568,0.4,0.6,inf,-inf,nan")
raw_response = "test,%d,%d,1|%s\n" % (ts-59, ts+1, raw_data)
self.assertEqual(response.content, raw_response)
response = self.client.get(url, {'target': 'test', 'format': 'json'})
+ self.assertIn('[1e9999, ' + str(ts - 2) + ']', response.content)
+ self.assertIn('[-1e9999, ' + str(ts - 1) + ']', response.content)
data = json.loads(response.content)
- end = data[0]['datapoints'][-4:]
+ end = data[0]['datapoints'][-7:]
self.assertEqual(
- end, [[None, ts - 3], [0.12345678901234568, ts - 2], [0.4, ts - 1], [0.6, ts]])
+ end, [[None, ts - 6],
+ [0.12345678901234568, ts - 5],
+ [0.4, ts - 4],
+ [0.6, ts - 3],
+ [float('inf'), ts - 2],
+ [float('-inf'), ts - 1],
+ [None, ts]])
response = self.client.get(url, {'target': 'test', 'format': 'dygraph'})
+ self.assertIn('[' + str((ts - 2) * 1000) + ', Infinity]', response.content)
+ self.assertIn('[' + str((ts - 1) * 1000) + ', -Infinity]', response.content)
data = json.loads(response.content)
- end = data['data'][-4:]
+ end = data['data'][-7:]
self.assertEqual(end,
- [[(ts - 3) * 1000, None],
- [(ts - 2) * 1000, 0.123456789012],
- [(ts - 1) * 1000, 0.4],
- [ts * 1000, 0.6]])
+ [[(ts - 6) * 1000, None],
+ [(ts - 5) * 1000, 0.123456789012],
+ [(ts - 4) * 1000, 0.4],
+ [(ts - 3) * 1000, 0.6],
+ [(ts - 2) * 1000, float('inf')],
+ [(ts - 1) * 1000, float('-inf')],
+ [ts * 1000, None]])
response = self.client.get(url, {'target': 'test', 'format': 'rickshaw'})
data = json.loads(response.content)
- end = data[0]['datapoints'][-4:]
+ end = data[0]['datapoints'][-7:-1]
self.assertEqual(end,
- [{'x': ts - 3, 'y': None},
- {'x': ts - 2, 'y': 0.12345678901234568},
- {'x': ts - 1, 'y': 0.4},
- {'x': ts, 'y': 0.6}])
+ [{'x': ts - 6, 'y': None},
+ {'x': ts - 5, 'y': 0.12345678901234568},
+ {'x': ts - 4, 'y': 0.4},
+ {'x': ts - 3, 'y': 0.6},
+ {'x': ts - 2, 'y': float('inf')},
+ {'x': ts - 1, 'y': float('-inf')}])
+
+ last = data[0]['datapoints'][-1]
+ self.assertEqual(last['x'], ts)
+ self.assertTrue(math.isnan(last['y']))
+
def test_hash_request(self):
# Requests with the same parameters should hash to the same values,

0 comments on commit a971bcd

Please sign in to comment.