AGENT-65 and AGENT-66 - Broken unit test (json encoding)

dev-requirements.txt

@@ -12,6 +12,7 @@ pysnmp
 requests==2.20.0
 redis==2.10.6
 Twisted==17.9.0
+ujson==1.35
 unittest2==1.1.0
 virtualenv==15.1.0
 webapp2==2.5.2

scalyr_agent/json_lib/parser.py

@@ -176,7 +176,7 @@ class JsonParser(object):
     """Parses raw byte input into JsonObjects and supports Scalyr's extensions.
 
     JsonParser is the main abstraction for parsing raw byte input
-    into JsonObject and other related objects.  It also supports Scaylr's
+    into JsonObject and other related objects.  It also supports Scalyr's
     extensions to the Json format, including comments and binary data.
     Specifically, the following are allowed:
 
@@ -708,7 +708,7 @@ def __peek_next_non_whitespace(self):
 def parse(input_bytes, check_duplicate_keys=False):
     """Parses the input as JSON and returns its contents.
 
-    It supports Scaylr's extensions to the Json format, including comments and
+    It supports Scalyr's extensions to the Json format, including comments and
     binary data. Specifically, the following are allowed:
 
      - // and /* comments

scalyr_agent/json_lib/tests/encode_decode_test.py

@@ -0,0 +1,112 @@
+# Copyright 2014 Scalyr Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ------------------------------------------------------------------------
+#
+# author:  Edward Chee <echee@scalyr.com>
+
+__author__ = 'echee@scalyr.com'
+
+
+import unittest
+
+from scalyr_agent import util
+from scalyr_agent.json_lib import JsonObject
+from scalyr_agent.json_lib import JsonArray
+from scalyr_agent.test_base import ScalyrTestCase
+
+JSON = 1
+UJSON = 2
+FALLBACK = 3
+
+
+class EncodeDecodeTest(ScalyrTestCase):
+    """This test ensures that JsonObject and JsonArray can be correctly encoded/decoded with different JSON libraries"""
+
+    def _setlib(self, library):
+        if library == JSON:
+            util._set_json_lib('json')
+        elif library == UJSON:
+            util._set_json_lib('ujson')
+        else:
+            util._set_json_lib('json_lib')
+
+    def test_invalid_lib(self):
+        with self.assertRaises(ValueError):
+            util._set_json_lib('BAD JSON LIBRARY NAME')
+
+    def test_dict(self):
+        self.__test_encode_decode('{"a":1,"b":2}', {u'a': 1, u'b': 2})
+
+    def test_dict2(self):
+        self.__test_encode_decode('{"a":1,"b":{"c":2}}', {u'a': 1, u'b': {u'c': 2}})
+
+    def test_str(self):
+        self.__test_encode_decode(r'"a"', u'a')
+
+    def test_int(self):
+        self.__test_encode_decode(r'1', 1)
+
+    def test_bool(self):
+        self.__test_encode_decode(r'false', False)
+        self.__test_encode_decode(r'true', True)
+
+    def test_float(self):
+        self.__test_encode_decode(r'1.0003', 1.0003)
+
+    def test_list(self):
+        self.__test_encode_decode(r'[1,2,3]', [1, 2, 3])
+
+    def test_list2(self):
+        self.__test_encode_decode(r'[1,2,"a"]', [1, 2, u'a'])
+
+    def test_jsonarray(self):
+        self.__test_encode_decode(r'[1,2,3]', JsonArray(1, 2, 3))
+
+    def test_jsonobject(self):
+        self.__test_encode_decode(r'{"a":1,"b":2}', JsonObject({u'a': 1, u'b': 2}))
+
+    def test_jsonobject_nested_dict(self):
+        self.__test_encode_decode(r'{"a":{"b":{"c":3}}}', JsonObject({u'a': JsonObject({u'b': JsonObject({u'c': 3})})}))
+
+    def test_jsonobject_nested_jsonarray(self):
+        self.__test_encode_decode(r'{"a":[1,2,3]}', JsonObject({u'a': JsonArray(1, 2, 3)}))
+
+    def test_jsonobject_nested_jsonarray2(self):
+        self.__test_encode_decode(r'{"a":[1,2,3,[1,2,3]]}', JsonObject({u'a': JsonArray(1, 2, 3, JsonArray(1, 2, 3))}))
+
+    def __test_encode_decode(self, text, obj):
+        def __runtest(library):
+            self._setlib(library)
+
+            if library == FALLBACK or not isinstance(obj, (JsonArray, JsonObject)):
+                text2 = util.json_encode(obj)
+                self.assertEquals(text, text2)
+                obj2 = util.json_decode(text2)
+                text3 = util.json_encode(obj2)
+                self.assertEquals(text, text3)
+            else:
+                with self.assertRaises(TypeError):
+                    util.json_encode(obj)
+
+        __runtest(JSON)
+        __runtest(UJSON)
+        __runtest(FALLBACK)
+
+
+def main():
+    unittest.main()
+
+
+if __name__ == '__main__':
+    main()

scalyr_agent/json_lib/tests/serializer_test.py

@@ -18,7 +18,7 @@
 __author__ = 'czerwin@scalyr.com'
 
 import unittest
-from scalyr_agent.json_lib import serialize, serialize_as_length_prefixed_string
+from scalyr_agent.json_lib import serialize
 
 from scalyr_agent.test_base import ScalyrTestCase
 
@@ -106,8 +106,8 @@ def test_length_prefixed_strings_with_unicode(self):
         self.assertEquals('`s\x00\x00\x00\x10Howdy \xe8\x92\xb8 folks!', serialize(u'Howdy \u84b8 folks!',
                                                                                    use_length_prefix_string=True))
 
-    def write(self, value):
-        return serialize(value, use_fast_encoding=True)
+    def write(self, value, sort_keys=False):
+        return serialize(value, use_fast_encoding=True, sort_keys=sort_keys)
 
     def __run_string_test_case(self, input_string, expected_result):
         self.assertEquals(serialize(input_string, use_fast_encoding=True), expected_result)

scalyr_agent/log_processing.py

@@ -1297,6 +1297,9 @@ def __init__(self, file_path, config, log_config, close_when_staleness_exceeds=N
             file_system = FileSystem()
         if log_attributes is None:
             log_attributes = {}
+        else:
+            if not isinstance(log_attributes, dict):
+                raise Exception('log_attributes must be of type dict.')
 
         self.__path = file_path
         # To mimic the behavior of the old agent which would use the ``thread_id`` feature of the Scalyr API to

scalyr_agent/tests/log_processing_test.py

@@ -1442,10 +1442,11 @@ def test_signals_deletion_due_to_staleness(self):
         self.assertTrue(completion_callback(LogFileProcessor.SUCCESS))
 
     def test_log_attributes(self):
-        vals = {'path': self.__path, 'attributes': JsonObject({'host': 'scalyr-1'})}
+        attribs = {'host': 'scalyr-1'}
+        vals = {'path': self.__path, 'attributes': JsonObject(attribs)}
         log_config = DEFAULT_CONFIG.parse_log_config(vals)
         log_processor = LogFileProcessor(self.__path, DEFAULT_CONFIG, log_config, file_system=self.__file_system,
-                                         log_attributes=vals['attributes'])
+                                         log_attributes=attribs)
         log_processor.perform_processing(TestLogFileProcessor.TestAddEventsRequest(), current_time=self.__fake_time)
 
         self.append_file(self.__path, 'First line\nSecond line\n')

scalyr_agent/util.py

@@ -55,47 +55,85 @@
 except ImportError:
     uuid = None
 
-def _fallback_json_encode( obj ):
+
+def _fallback_json_encode( obj, sort_keys=False ):
+    # json_lib.serialize() always ignores sort_keys param (always sorts regardless)
     return json_lib.serialize( obj )
 
+
 def _fallback_json_decode( text ):
     return json_lib.parse( text )
 
-_json_lib = 'json_lib'
 
-_json_encode = _fallback_json_encode
-_json_decode = _fallback_json_decode
+def get_json_implementation(lib_name):
+
+    if lib_name == 'json_lib':
+        return lib_name, _fallback_json_encode, _fallback_json_decode
+
+    elif lib_name == 'ujson':
+        import ujson
+
+        def ujson_dumps_custom(*args, **kwargs):
+            # ujson does not raise exception if you pass it a JsonArray/JsonObject while producing wrong encoding.
+            # Detect and complain loudly.
+            if isinstance(args[0], (json_lib.JsonObject, json_lib.JsonArray)):
+                raise TypeError('ujson does not correctly encode objects of type: %s' % type(args[0]))
+            return ujson.dumps(*args, **kwargs)
+
+        return lib_name, ujson_dumps_custom, ujson.loads
+
+    else:
+        if lib_name != 'json':
+            raise ValueError('Unsupported json library %s' % lib_name)
 
-try:
-    import ujson
-    _json_lib = 'ujson'
-    _json_encode = ujson.dumps
-    _json_decode = ujson.loads
-except ImportError:
-    try:
         import json
-        _json_lib = 'json'
 
-        def dumps_no_space(*args, **kwargs):
+        def json_dumps_custom(*args, **kwargs):
             """Eliminate spaces by default. Python 2.4 does not support partials."""
             if 'separators' not in kwargs:
                 kwargs['separators'] = (',', ':')
             return json.dumps(*args, **kwargs)
 
-        _json_encode = dumps_no_space
-        _json_decode = json.loads
+        return lib_name, json_dumps_custom, json.loads
+
+
+_json_lib = None
+_json_encode = None
+_json_decode = None
+
+
+def _set_json_lib(lib_name):
+    # This function is not meant to be invoked at runtime.  It exists primarily for testing.
+    global _json_lib, _json_encode, _json_decode
+    _json_lib, _json_encode, _json_decode = get_json_implementation(lib_name)
+
+
+_set_json_lib('json_lib')
+try:
+    _set_json_lib('ujson')
+except ImportError:
+    try:
+        _set_json_lib('json')
     except:
         pass
 
+
 def get_json_lib():
     return _json_lib
 
 def json_encode( obj ):
-    """ Encodes an object as json """
+    """Encodes an object into a JSON string.  The underlying implementation either handles JsonObject/JsonArray
+    or else complains loudly (raises Exception) if they do not correctly support encoding.
+    """
     return _json_encode( obj, sort_keys=True )
 
 def json_decode( text ):
-    """ Decodes text containing json and returns a dict containing the contents """
+    """Decodes text containing json and returns either a dict or a scalyr_agent.json_lib.objects.JsonObject
+    depending on which underlying decoder is used.
+
+    If json or ujson are used, a dict is returned.
+    If the Scalyr custom json_lib decoder is used, a JsonObject is returned
+    """
     return _json_decode( text )
 
 def value_to_bool( value ):