Merge branch '1.2.x'

CHANGES.rst

@@ -1,6 +1,24 @@
 Changelog
 =========
 
+1.2.5 - 2020-07-03
+------------------
+
+- Will now import Iterable from the Python 3.3+ location as the previous
+  location is marked for removal in Python 3.9.  The import will still
+  have a fallback to the previous location in order to maintain support
+  for Python 2.7.  [
+  `#31 <https://github.com/calmjs/calmjs.parse/issues/31>`_
+  ]
+- Provide a test case helper to ensure that the generic ``Program`` repr
+  signature is provided to assist with test case portability.  [
+  `#33 <https://github.com/calmjs/calmjs.parse/issues/33>`_
+  ]
+- In the ``calmjs.parse.vlq`` module, implemented the ``decode_vlq``
+  helper for completeness/symmetry to the ``encode_vlq`` helper.  [
+  `#33 <https://github.com/calmjs/calmjs.parse/issues/33>`_
+  ]
+
 1.2.4 - 2020-03-17
 ------------------
 

src/calmjs/parse/io.py

@@ -4,7 +4,10 @@
 """
 
 from itertools import chain
-from collections import Iterable
+try:
+    from collections.abc import Iterable
+except ImportError:  # pragma: no cover
+    from collections import Iterable
 from calmjs.parse.asttypes import Node
 from calmjs.parse import sourcemap
 from calmjs.parse.exceptions import ECMASyntaxError

src/calmjs/parse/tests/parser.py

@@ -249,6 +249,16 @@ def clean(f):
     return ''.join(clean(t) for t in textwrap.dedent(s).splitlines())
 
 
+def format_repr_program_type(label, repr_output, program_type):
+    result = singleline(repr_output)
+    if not result.startswith('<Program'):
+        raise ValueError(
+            "repr test result for '%s' did not start with generic '<Program', "
+            "got: %s" % (label, repr_output)
+        )
+    return result.replace('<Program', '<' + program_type)
+
+
 def build_node_repr_test_cases(clsname, parse, program_type):
 
     def parse_to_repr(value):
@@ -257,7 +267,7 @@ def parse_to_repr(value):
     return build_equality_testcase(clsname, parse_to_repr, ((
         label,
         textwrap.dedent(argument).strip(),
-        singleline(result).replace('<Program', '<' + program_type),
+        format_repr_program_type(label, result, program_type),
     ) for label, argument, result in [(
         'block',
         """
@@ -2815,7 +2825,7 @@ def parse_with_comments_to_repr(value):
     return build_equality_testcase(clsname, parse_with_comments_to_repr, ((
         label,
         textwrap.dedent(argument).strip(),
-        singleline(result).replace('<Program', '<' + program_type),
+        format_repr_program_type(label, result, program_type),
     ) for label, argument, result in [(
         'block_without_comments',
         """

src/calmjs/parse/tests/test_testing.py

@@ -7,6 +7,8 @@
 from calmjs.parse.testing.util import build_exception_testcase
 from calmjs.parse.testing.util import setup_logger
 
+from calmjs.parse.tests.parser import format_repr_program_type
+
 
 def run(self):
     """
@@ -90,3 +92,27 @@ def runTest(self):
         testcase.doCleanups()
         self.assertEqual(original_level, logger.level)
         self.assertEqual(original_handlers, len(logger.handlers))
+
+
+class ParserTestSetupTestCase(unittest.TestCase):
+
+    def test_match(self):
+        result = format_repr_program_type('foo', '<Program>', 'ES4Program')
+        self.assertEqual(result, '<ES4Program>')
+
+    def test_fail(self):
+        with self.assertRaises(ValueError) as e:
+            format_repr_program_type('foo', '<ES4Program>', 'ES4Program')
+
+        self.assertEqual(
+            e.exception.args[0], "repr test result for 'foo' did not start "
+            "with generic '<Program', got: <ES4Program>"
+        )
+
+        with self.assertRaises(ValueError) as e:
+            format_repr_program_type('foo', '<ES5Program>', 'ES4Program')
+
+        self.assertEqual(
+            e.exception.args[0], "repr test result for 'foo' did not start "
+            "with generic '<Program', got: <ES5Program>"
+        )

src/calmjs/parse/tests/test_vlq.py

@@ -29,6 +29,19 @@ def test_vlq_encode_multi(self):
         self.assertEqual(vlq.encode_vlq(789), 'qxB')
         self.assertEqual(vlq.encode_vlq(-789), 'rxB')
 
+    def test_vlq_decode_various(self):
+        self.assertEqual(vlq.decode_vlq('A'), 0)
+        # should never be encoded to this standalone value, but nothing
+        # stops this from being decoded
+        self.assertEqual(vlq.decode_vlq('B'), 0)
+        self.assertEqual(vlq.decode_vlq('C'), 1)
+        self.assertEqual(vlq.decode_vlq('D'), -1)
+        self.assertEqual(vlq.decode_vlq('E'), 2)
+        self.assertEqual(vlq.decode_vlq('/f'), -511)
+        self.assertEqual(vlq.decode_vlq('ggB'), 512)
+        # won't decode more than one
+        self.assertEqual(vlq.decode_vlq('ABCDE'), 0)
+
     def test_encode_vlqs(self):
         self.assertEqual(vlq.encode_vlqs((0, 1, 2, 3, 4)), 'ACEGI')
         self.assertEqual(vlq.encode_vlqs((123, 456, 789)), '2HwcqxB')

src/calmjs/parse/vlq.py

@@ -3,7 +3,7 @@
 Module for dealing with VLQ encodings
 """
 
-# While base64 is used, it not used for the string encoding but the
+# While base64 is used, it's not used for the string encoding but the
 # characters map to corresponding bits.  The lowest 5 bits map to the
 # actual number, with the highest (6th) bit being the continuation mark;
 # if set, denote the next character is to build on the current.  Do note
@@ -17,13 +17,13 @@
 # 0 | 0 0 0 0 0 0 | = 0           5 | 0 0 0 1 0 0 | = 2
 #
 # (F) F has sign bit, negative   (2H) 2 has continuation bit, H does not
-# F | c - - - - s |             2 H | c - - - - s | c - - - - s |
+# F | c - - - - s |             2 H | c - - - - s | c - - - - - |
 # 0 | 0 0 0 1 0 1 | = -2       54 7 | 1 1 0 1 1 0 | 0 0 0 1 1 1 | = 123
 #
 # For the 2H example, note that it's only the `2` character that carry
 # the sign bit as it has the lowest bit, the other characters form
 # further higher bits until the last one without one.  The bits would
-# look like ( 0 0 1 1 1, 1 0 1 1 + ) for the conversion to the interger
+# look like ( 0 0 1 1 1 | 1 0 1 1 + ) for the conversion to the interger
 # value of +123.
 #
 # Thus arbitrary long integers can be represented by arbitrary long
@@ -87,12 +87,12 @@ def encode_vlqs(ints):
     return ''.join(encode_vlq(i) for i in ints)
 
 
-def decode_vlqs(s):
+def vlq_decoder(s):
     """
-    Decode str `s` into a list of integers.
+    A generator that accepts a VLQ encoded string as input to produce
+    integers.
     """
 
-    ints = []
     i = 0
     shift = 0
 
@@ -103,11 +103,25 @@ def decode_vlqs(s):
         shift += VLQ_SHIFT
         if not cont:
             sign = -1 if 1 & i else 1
-            ints.append((i >> 1) * sign)
+            yield (i >> 1) * sign
             i = 0
             shift = 0
 
-    return tuple(ints)
+
+def decode_vlq(s):
+    """
+    Decode the first integer from a vlq encoded string.
+    """
+
+    return next(vlq_decoder(s))
+
+
+def decode_vlqs(s):
+    """
+    Decode str `s` into a tuple of integers.
+    """
+
+    return tuple(vlq_decoder(s))
 
 
 def encode_mappings(mappings):