From 44aa9699f667f300d57e5f7291686f5fb2d216ea Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 9 Jul 2013 15:55:50 -0400 Subject: [PATCH 01/11] [#1078] avoid json loads/dumps with LazyJSONObject --- ckan/controllers/api.py | 8 +++++++- ckan/lib/lazyjson.py | 36 ++++++++++++++++++++++++++++++++++++ ckan/logic/action/get.py | 7 ++++++- 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 ckan/lib/lazyjson.py diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index 22ea7d4f5c3..be131ede046 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -34,6 +34,7 @@ 'text': 'text/plain;charset=utf-8', 'html': 'text/html;charset=utf-8', 'json': 'application/json;charset=utf-8', + 'json_string': 'application/json;charset=utf-8', } @@ -161,7 +162,7 @@ def action(self, logic_function, ver=None): _('Action name not known: %s') % logic_function) context = {'model': model, 'session': model.Session, 'user': c.user, - 'api_version': ver} + 'api_version': ver, 'return_type': 'LazyJSONObject'} model.Session()._context = context return_dict = {'help': function.__doc__} try: @@ -185,6 +186,11 @@ def action(self, logic_function, ver=None): try: result = function(context, request_data) return_dict['success'] = True + if hasattr(result, 'to_json_string'): + return_dict['result'] = 463455395108 # magic placeholder + return self._finish_ok(h.json.dumps( + return_dict).replace('463455395108', + result.to_json_string()), 'json_string') return_dict['result'] = result except DataError, e: log.error('Format incorrect: %s - %s' % (e.error, request_data)) diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py new file mode 100644 index 00000000000..321cb2fb806 --- /dev/null +++ b/ckan/lib/lazyjson.py @@ -0,0 +1,36 @@ +import json + +class LazyJSONObject(object): + '''An object that behaves like a dict returned from json.loads''' + def __init__(self, json_string): + self._json_string = json_string + self._json_dict = None + + def _loads(self): + if not self._json_dict: + self._json_dict = json.loads(self._json_string) + self._json_string = None + return self._json_dict + + def __nonzero__(self): + return True + + def to_json_string(self, *args, **kwargs): + if self._json_string: + return self._json_string + return json.dumps(self._json_dict, *args, **kwargs) + + +def _loads_method(name): + def method(self, *args, **kwargs): + return getattr(self._loads(), name)(*args, **kwargs) + return method + +for fn in ['__cmp__', '__contains__', '__delitem__', '__eq__', '__ge__', + '__getitem__', '__gt__', '__iter__', '__le__', '__len__', '__lt__', + '__ne__', '__setitem__', 'clear', 'copy', 'fromkeys', 'get', 'has_key', + 'items', 'iteritems', 'iterkeys', 'itervalues', 'keys', 'pop', + 'popitem', 'setdefault', 'update', 'values']: + setattr(LazyJSONObject, fn, _loads_method(fn)) + + diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 7c9cdfddace..123c431759d 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -21,6 +21,7 @@ import ckan.lib.plugins as lib_plugins import ckan.lib.activity_streams as activity_streams import ckan.new_authz as new_authz +import ckan.lib.lazyjson as lazyjson from ckan.common import _ @@ -781,7 +782,11 @@ def package_show(context, data_dict): else: use_validated_cache = 'schema' not in context if use_validated_cache and 'validated_data_dict' in search_result: - package_dict = json.loads(search_result['validated_data_dict']) + package_json = search_result['validated_data_dict'] + if context.get('return_type') == 'LazyJSONObject': + package_dict = lazyjson.LazyJSONObject(package_json) + else: + package_dict = json.loads(package_json) package_dict_validated = True else: package_dict = json.loads(search_result['data_dict']) From 3ae69be07908ab3fb6305e2b284d2ec0d61d45c9 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 9 Jul 2013 17:19:36 -0400 Subject: [PATCH 02/11] [#1078] LazyJSONEncoder as fallback for api call responses --- ckan/controllers/api.py | 13 ++++++------- ckan/lib/lazyjson.py | 6 ++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index be131ede046..5b23fcc1bec 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -16,6 +16,7 @@ import ckan.lib.navl.dictization_functions import ckan.lib.jsonp as jsonp import ckan.lib.munge as munge +import ckan.lib.lazyjson as lazyjson from ckan.common import _, c, request, response @@ -34,7 +35,6 @@ 'text': 'text/plain;charset=utf-8', 'html': 'text/html;charset=utf-8', 'json': 'application/json;charset=utf-8', - 'json_string': 'application/json;charset=utf-8', } @@ -83,7 +83,11 @@ def _finish(self, status_int, response_data=None, if response_data is not None: response.headers['Content-Type'] = CONTENT_TYPES[content_type] if content_type == 'json': - response_msg = h.json.dumps(response_data) + try: + response_msg = h.json.dumps(response_data) + except TypeError: + response_msg = lazyjson.LazyJSONEncoder().encode( + response_data) else: response_msg = response_data # Support "JSONP" callback. @@ -186,11 +190,6 @@ def action(self, logic_function, ver=None): try: result = function(context, request_data) return_dict['success'] = True - if hasattr(result, 'to_json_string'): - return_dict['result'] = 463455395108 # magic placeholder - return self._finish_ok(h.json.dumps( - return_dict).replace('463455395108', - result.to_json_string()), 'json_string') return_dict['result'] = result except DataError, e: log.error('Format incorrect: %s - %s' % (e.error, request_data)) diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py index 321cb2fb806..b5ec164864c 100644 --- a/ckan/lib/lazyjson.py +++ b/ckan/lib/lazyjson.py @@ -34,3 +34,9 @@ def method(self, *args, **kwargs): setattr(LazyJSONObject, fn, _loads_method(fn)) +class LazyJSONEncoder(json.JSONEncoder): + '''JSON encoder that handles LazyJSONObject elements''' + def _iterencode_default(self, o, markers=None): + if hasattr(o, 'to_json_string'): + return iter([o.to_json_string()]) + return json.JSONEncoder._iterencode_default(self, o, markers) From ce5d7e85280bdd350abbc39c19a9f37066472603 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 9 Jul 2013 17:27:44 -0400 Subject: [PATCH 03/11] [#1078] force package_show return type within resource_update --- ckan/logic/action/update.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index c841a9bb25c..2e363eb6748 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -210,7 +210,8 @@ def resource_update(context, data_dict): del context["resource"] package_id = resource.resource_group.package.id - pkg_dict = _get_action('package_show')(context, {'id': package_id}) + pkg_dict = _get_action('package_show')(dict(context, return_type='dict'), + {'id': package_id}) for n, p in enumerate(pkg_dict['resources']): if p['id'] == id: From 9330b47e0e314ee461aac1529a77a3fb6acb30a0 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 9 Jul 2013 18:40:55 -0400 Subject: [PATCH 04/11] [#1078] pep8 --- ckan/lib/lazyjson.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py index b5ec164864c..d2d7d83deb8 100644 --- a/ckan/lib/lazyjson.py +++ b/ckan/lib/lazyjson.py @@ -1,5 +1,6 @@ import json + class LazyJSONObject(object): '''An object that behaves like a dict returned from json.loads''' def __init__(self, json_string): @@ -27,10 +28,10 @@ def method(self, *args, **kwargs): return method for fn in ['__cmp__', '__contains__', '__delitem__', '__eq__', '__ge__', - '__getitem__', '__gt__', '__iter__', '__le__', '__len__', '__lt__', - '__ne__', '__setitem__', 'clear', 'copy', 'fromkeys', 'get', 'has_key', - 'items', 'iteritems', 'iterkeys', 'itervalues', 'keys', 'pop', - 'popitem', 'setdefault', 'update', 'values']: + '__getitem__', '__gt__', '__iter__', '__le__', '__len__', '__lt__', + '__ne__', '__setitem__', 'clear', 'copy', 'fromkeys', 'get', + 'has_key', 'items', 'iteritems', 'iterkeys', 'itervalues', 'keys', + 'pop', 'popitem', 'setdefault', 'update', 'values']: setattr(LazyJSONObject, fn, _loads_method(fn)) From 12420cefcc8ad3e40e429201189857cf99abd69e Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 9 Jul 2013 18:44:45 -0400 Subject: [PATCH 05/11] [#1078] force package_show return type within resource_create --- ckan/logic/action/create.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index fcb55ceae64..d7d470cba20 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -245,7 +245,8 @@ def resource_create(context, data_dict): package_id = _get_or_bust(data_dict, 'package_id') data_dict.pop('package_id') - pkg_dict = _get_action('package_show')(context, {'id': package_id}) + pkg_dict = _get_action('package_show')(dict(context, return_type='dict'), + {'id': package_id}) _check_access('resource_create', context, data_dict) From 5f0ca73ed2e2e65ec071884f8c17498c666dd448 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 9 Jul 2013 19:52:34 -0400 Subject: [PATCH 06/11] [#1078] 2.7 comatibility fix: use simplejson for lazyjson --- ckan/lib/lazyjson.py | 61 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py index d2d7d83deb8..14ca4daf2e8 100644 --- a/ckan/lib/lazyjson.py +++ b/ckan/lib/lazyjson.py @@ -1,4 +1,5 @@ -import json +import simplejson as json +import simplejson.encoder as json_encoder class LazyJSONObject(object): @@ -35,9 +36,61 @@ def method(self, *args, **kwargs): setattr(LazyJSONObject, fn, _loads_method(fn)) +class JSONString(str): + '''a type for already-encoded JSON''' + pass + + +def _encode_jsonstring(s): + if isinstance(s, JSONString): + return s + return json_encoder.encode_basestring(s) + + class LazyJSONEncoder(json.JSONEncoder): '''JSON encoder that handles LazyJSONObject elements''' - def _iterencode_default(self, o, markers=None): + def iterencode(self, o, _one_shot=False): + ''' + most of JSONEncoder.iterencode() copied so that _encode_jsonstring + may be used instead of encode_basestring + ''' + if self.check_circular: + markers = {} + else: + markers = None + def floatstr(o, allow_nan=self.allow_nan, + _repr=json_encoder.FLOAT_REPR, + _inf=json_encoder.PosInf, + _neginf=-json_encoder.PosInf): + # Check for specials. Note that this type of test is processor + # and/or platform-specific, so do tests which don't depend on the + # internals. + + if o != o: + text = 'NaN' + elif o == _inf: + text = 'Infinity' + elif o == _neginf: + text = '-Infinity' + else: + return _repr(o) + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o)) + + return text + _iterencode = json_encoder._make_iterencode( + markers, self.default, _encode_jsonstring, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot, self.use_decimal, + self.namedtuple_as_object, self.tuple_as_array, + self.bigint_as_string, self.item_sort_key, + self.encoding, Decimal=json_encoder.Decimal) + return _iterencode(o, 0) + + def default(self, o): if hasattr(o, 'to_json_string'): - return iter([o.to_json_string()]) - return json.JSONEncoder._iterencode_default(self, o, markers) + return JSONString(o.to_json_string()) + return json.JSONEncoder.default(self, o) From 5a795f2bd9424d6bd0d30389b0477f7567e83019 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 9 Jul 2013 21:17:07 -0400 Subject: [PATCH 07/11] [#1078] pep8 --- ckan/lib/lazyjson.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py index 14ca4daf2e8..50bcf0a4f6c 100644 --- a/ckan/lib/lazyjson.py +++ b/ckan/lib/lazyjson.py @@ -58,10 +58,11 @@ def iterencode(self, o, _one_shot=False): markers = {} else: markers = None + def floatstr(o, allow_nan=self.allow_nan, - _repr=json_encoder.FLOAT_REPR, - _inf=json_encoder.PosInf, - _neginf=-json_encoder.PosInf): + _repr=json_encoder.FLOAT_REPR, + _inf=json_encoder.PosInf, + _neginf=-json_encoder.PosInf): # Check for specials. Note that this type of test is processor # and/or platform-specific, so do tests which don't depend on the # internals. From 3462450d5deb018d5c67d47b7872cfee075f19ee Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 11 Dec 2014 17:07:56 -0500 Subject: [PATCH 08/11] [#1078] smaller simplejson hack: use for_json and pretend to be an int --- ckan/controllers/api.py | 9 ++---- ckan/lib/lazyjson.py | 72 ++++++++--------------------------------- 2 files changed, 16 insertions(+), 65 deletions(-) diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index 934d531c87f..f81e5d718f9 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -16,7 +16,6 @@ import ckan.lib.navl.dictization_functions import ckan.lib.jsonp as jsonp import ckan.lib.munge as munge -import ckan.lib.lazyjson as lazyjson from ckan.common import _, c, request, response @@ -84,11 +83,9 @@ def _finish(self, status_int, response_data=None, if response_data is not None: response.headers['Content-Type'] = CONTENT_TYPES[content_type] if content_type == 'json': - try: - response_msg = h.json.dumps(response_data) - except TypeError: - response_msg = lazyjson.LazyJSONEncoder().encode( - response_data) + response_msg = h.json.dumps( + response_data, + for_json=True) # handle objects with for_json methods else: response_msg = response_data # Support "JSONP" callback. diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py index 50bcf0a4f6c..2d5d6b5a82a 100644 --- a/ckan/lib/lazyjson.py +++ b/ckan/lib/lazyjson.py @@ -17,10 +17,10 @@ def _loads(self): def __nonzero__(self): return True - def to_json_string(self, *args, **kwargs): + def for_json(self): if self._json_string: - return self._json_string - return json.dumps(self._json_dict, *args, **kwargs) + return JSONString(self._json_string) + return self._json_dict def _loads_method(name): @@ -36,62 +36,16 @@ def method(self, *args, **kwargs): setattr(LazyJSONObject, fn, _loads_method(fn)) -class JSONString(str): - '''a type for already-encoded JSON''' - pass +class JSONString(int): + ''' + A type for already-encoded JSON + Fake-out simplejson by subclassing int so that simplejson calls + our __str__ method to produce JSON. + ''' + def __init__(self, s): + self.s = s + super(JSONString, self).__init__(-1) -def _encode_jsonstring(s): - if isinstance(s, JSONString): + def __str__(self): return s - return json_encoder.encode_basestring(s) - - -class LazyJSONEncoder(json.JSONEncoder): - '''JSON encoder that handles LazyJSONObject elements''' - def iterencode(self, o, _one_shot=False): - ''' - most of JSONEncoder.iterencode() copied so that _encode_jsonstring - may be used instead of encode_basestring - ''' - if self.check_circular: - markers = {} - else: - markers = None - - def floatstr(o, allow_nan=self.allow_nan, - _repr=json_encoder.FLOAT_REPR, - _inf=json_encoder.PosInf, - _neginf=-json_encoder.PosInf): - # Check for specials. Note that this type of test is processor - # and/or platform-specific, so do tests which don't depend on the - # internals. - - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - return _repr(o) - - if not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) - - return text - _iterencode = json_encoder._make_iterencode( - markers, self.default, _encode_jsonstring, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, _one_shot, self.use_decimal, - self.namedtuple_as_object, self.tuple_as_array, - self.bigint_as_string, self.item_sort_key, - self.encoding, Decimal=json_encoder.Decimal) - return _iterencode(o, 0) - - def default(self, o): - if hasattr(o, 'to_json_string'): - return JSONString(o.to_json_string()) - return json.JSONEncoder.default(self, o) From 0afe2dbfbb3155bd5cf7ec4ddf9439a0eb355b08 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 15 Dec 2014 09:07:54 -0500 Subject: [PATCH 09/11] [#1078] inherit from dict to pass some isinstance checks --- ckan/lib/lazyjson.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py index 2d5d6b5a82a..56a1e5c5377 100644 --- a/ckan/lib/lazyjson.py +++ b/ckan/lib/lazyjson.py @@ -2,7 +2,7 @@ import simplejson.encoder as json_encoder -class LazyJSONObject(object): +class LazyJSONObject(dict): '''An object that behaves like a dict returned from json.loads''' def __init__(self, json_string): self._json_string = json_string From 97620dd84cf87161a9cc85edf70e68c427e04471 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 15 Dec 2014 09:08:09 -0500 Subject: [PATCH 10/11] [#1078] pep8 --- ckan/logic/action/create.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 95dac7dc30b..07a1c33ad6b 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -274,7 +274,8 @@ def resource_create(context, data_dict): package_id = _get_or_bust(data_dict, 'package_id') _get_or_bust(data_dict, 'url') - pkg_dict = _get_action('package_show')(dict(context, return_type='dict'), + pkg_dict = _get_action('package_show')( + dict(context, return_type='dict'), {'id': package_id}) _check_access('resource_create', context, data_dict) From 5bb1349c1b95a3d3ccee7ca27adfc9d8f3ec2a76 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 15 Dec 2014 09:38:08 -0500 Subject: [PATCH 11/11] [#1078] in my own defense --- ckan/lib/lazyjson.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py index 56a1e5c5377..6305cb7d894 100644 --- a/ckan/lib/lazyjson.py +++ b/ckan/lib/lazyjson.py @@ -42,6 +42,10 @@ class JSONString(int): Fake-out simplejson by subclassing int so that simplejson calls our __str__ method to produce JSON. + + This trick is unpleasant, but significantly less fragile than + subclassing JSONEncoder and modifying its internal workings, or + monkeypatching the simplejson library. ''' def __init__(self, s): self.s = s