Merge branch 'release/0.5.3'

CHANGES

@@ -1,6 +1,10 @@
 Change History
 ==============
 
+0.5.3 (2011-06-29)
+------------------
+- Bugfix: https://github.com/rspivak/slimit/issues/5
+
 0.5.2 (2011-06-14)
 ------------------
 - Bugfix: https://github.com/rspivak/slimit/issues/4

CREDIT

@@ -3,4 +3,5 @@ Bug reports
 -----------
 
 - Rui Pereira
-- d-m-k http://github.com/d-m-k
+- Dima Kozlov
+- BadKnees https://github.com/BadKnees

README.rst

@@ -130,5 +130,29 @@ Benchmarks
 
 Roadmap
 -------
-- More minifications
-- Speed improvements
+- when doing name mangling handle cases with 'eval' and 'with'
+- foo["bar"] ==> foo.bar
+- consecutive declarations: var a = 10; var b = 20; ==> var a=10,b=20;
+- reduce simple constant expressions if the result takes less space:
+  1 +2 * 3 ==> 7
+- IF statement optimizations
+
+  1. if (foo) bar(); else baz(); ==> foo?bar():baz();
+  2. if (!foo) bar(); else baz(); ==> foo?baz():bar();
+  3. if (foo) bar(); ==> foo&&bar();
+  4. if (!foo) bar(); ==> foo||bar();
+  5. if (foo) return bar(); else return baz(); ==> return foo?bar():baz();
+  6. if (foo) return bar(); else something(); ==> {if(foo)return bar();something()}
+
+- remove unreachable code that follows a return, throw, break or
+  continue statement, except function/variable declarations
+- parsing speed improvements
+
+Acknowledgments
+---------------
+- The lexer and parser are built with `PLY <http://www.dabeaz.com/ply/>`_
+- Several test cases and regexes from `jslex <https://bitbucket.org/ned/jslex>`_
+- Some visitor ideas - `pycparser <http://code.google.com/p/pycparser/>`_
+- Many grammar rules are taken from `rkelly <https://github.com/tenderlove/rkelly>`_
+- Name mangling and different optimization ideas - `UglifyJS <https://github.com/mishoo/UglifyJS>`_
+- ASI implementation was inspired by `pyjsparser <http://bitbucket.org/mvantellingen/pyjsparser>`_

setup.py

@@ -17,7 +17,7 @@ def read(*rel_names):
 
 setup(
     name='slimit',
-    version='0.5.2',
+    version='0.5.3',
     url='http://slimit.org',
     license='MIT',
     description='SlimIt - JavaScript minifier',

src/slimit/ast.py

@@ -153,6 +153,7 @@ class VarStatement(Node):
 class VarDecl(Node):
     def __init__(self, identifier, initializer=None):
         self.identifier = identifier
+        self.identifier._mangle_candidate = True
         self.initializer = initializer
 
     def children(self):
@@ -329,23 +330,32 @@ def __init__(self, value):
     def children(self):
         return []
 
-class FuncDecl(Node):
+
+class FuncBase(Node):
     def __init__(self, identifier, parameters, elements):
         self.identifier = identifier
         self.parameters = parameters if parameters is not None else []
         self.elements = elements if elements is not None else []
+        self._init_ids()
+
+    def _init_ids(self):
+        # function declaration/expression name and parameters are identifiers
+        # and therefore are subject to name mangling. we need to mark them.
+        if self.identifier is not None:
+            self.identifier._mangle_candidate = True
+        for param in self.parameters:
+            param._mangle_candidate = True
 
     def children(self):
         return [self.identifier] + self.parameters + self.elements
 
-class FuncExpr(Node):
-    def __init__(self, identifier, parameters, elements):
-        self.identifier = identifier
-        self.parameters = parameters if parameters is not None else []
-        self.elements = elements if elements is not None else []
+class FuncDecl(FuncBase):
+    pass
+
+# The only difference is that function expression might not have an identifier
+class FuncExpr(FuncBase):
+    pass
 
-    def children(self):
-        return [self.identifier] + self.parameters + self.elements
 
 class Comma(Node):
     def __init__(self, left, right):

src/slimit/lexer.py

@@ -360,22 +360,22 @@ def t_regex_error(self, token):
     t_STRING = r"""
     (?:
         # double quoted string
-        (?:"                           # opening double quote
-            (?: [^"\\\n\r]             # no escape chars, line terminators or "
-                | \\[a-zA-Z\\\'"?]     # or escaped characters
-                | \\x[0-9a-fA-F]{2}    # or hex_escape_sequence
-                | \\u[0-9a-fA-F]{4}    # or unicode_escape_sequence
-            )*?                        # zero or many times
-        ")                             # closing double quote
+        (?:"                               # opening double quote
+            (?: [^"\\\n\r]                 # no \, line terminators or "
+                | \\[a-zA-Z!-\/:-@\[-`{-~] # or escaped characters
+                | \\x[0-9a-fA-F]{2}        # or hex_escape_sequence
+                | \\u[0-9a-fA-F]{4}        # or unicode_escape_sequence
+            )*?                            # zero or many times
+        ")                                 # closing double quote
         |
         # single quoted string
-        (?:'                           # opening single quote
-            (?: [^'\\\n\r]             # no escape chars, line terminators or '
-                | \\[a-zA-Z\\'"?]      # or escaped characters
-                | \\x[0-9a-fA-F]{2}    # or hex_escape_sequence
-                | \\u[0-9a-fA-F]{4}    # or unicode_escape_sequence
-            )*?                        # zero or many times
-        ')                             # closing single quote
+        (?:'                               # opening single quote
+            (?: [^'\\\n\r]                 # no \, line terminators or '
+                | \\[a-zA-Z!-\/:-@\[-`{-~] # or escaped characters
+                | \\x[0-9a-fA-F]{2}        # or hex_escape_sequence
+                | \\u[0-9a-fA-F]{4}        # or unicode_escape_sequence
+            )*?                            # zero or many times
+        ')                                 # closing single quote
     )
     """  # "
 

src/slimit/parser.py

@@ -191,20 +191,26 @@ def p_primary_expr(self, p):
         """
         p[0] = p[1]
 
-    def p_primary_expr_no_brace(self, p):
-        """primary_expr_no_brace : THIS
-                                 | identifier
-                                 | literal
+    def p_primary_expr_no_brace_1(self, p):
+        """primary_expr_no_brace : identifier"""
+        p[1]._mangle_candidate = True
+        p[1]._in_expression = True
+        p[0] = p[1]
+
+    def p_primary_expr_no_brace_2(self, p):
+        """primary_expr_no_brace : THIS"""
+        p[0] = ast.This()
+
+    def p_primary_expr_no_brace_3(self, p):
+        """primary_expr_no_brace : literal
                                  | array_literal
-                                 | LPAREN expr RPAREN
         """
-        if p[1] == 'this':
-            p[0] = ast.This()
-        elif len(p) == 2:
-            p[0] = p[1]
-        else:
-            p[2]._parens = True
-            p[0] = p[2]
+        p[0] = p[1]
+
+    def p_primary_expr_no_brace_4(self, p):
+        """primary_expr_no_brace : LPAREN expr RPAREN"""
+        p[2]._parens = True
+        p[0] = p[2]
 
     def p_array_literal_1(self, p):
         """array_literal : LBRACKET elision_opt RBRACKET"""

src/slimit/tests/test_lexer.py

@@ -133,6 +133,16 @@ def test_illegal_unicode_char_in_identifier(self):
          [r"STRING '\u0001'", r'STRING "\uFCEF"', r"STRING 'a\\\b\n'"]
          ),
         (ur'"тест строки\""', [ur'STRING "тест строки\""']),
+        # Bug - https://github.com/rspivak/slimit/issues/5
+        (r"var tagRegExp = new RegExp('<(\/*)(FooBar)', 'gi');",
+         ['VAR var', 'ID tagRegExp', 'EQ =',
+          'NEW new', 'ID RegExp', 'LPAREN (',
+          r"STRING '<(\/*)(FooBar)'", 'COMMA ,', "STRING 'gi'",
+          'RPAREN )', 'SEMI ;']
+        ),
+        # same as above but inside double quotes
+        (r'"<(\/*)(FooBar)"', [r'STRING "<(\/*)(FooBar)"']),
+
 
         # # Comments
         # ("""

src/slimit/visitors/scopevisitor.py

@@ -51,15 +51,6 @@ def __init__(self, sym_table):
         self.sym_table = sym_table
         self.current_scope = sym_table.globals
 
-    def visit_Assign(self, node):
-        # property_assignment
-        # skip identifier
-        if node.op == ':' and isinstance(node.left, ast.Identifier):
-            self.visit(node.right)
-        else:
-            self.visit(node.left)
-            self.visit(node.right)
-
     def visit_VarDecl(self, node):
         ident = node.identifier
         symbol = VarSymbol(name=ident.value)
@@ -107,26 +98,20 @@ def visit_FuncDecl(self, node):
 class RefVisitor(Visitor):
     """Fill 'ref' attribute in scopes."""
 
-    def visit_VarDecl(self, node):
-        # we skip Identifier node because it's not part of an expression
-        self.visit(node.initializer)
-
-    def visit_FuncDecl(self, node):
-        # we skip all Identifier nodes because they are not part of expressions
-        for element in node.elements:
-            self.visit(element)
-    # alias
-    visit_FuncExpr = visit_FuncDecl
-
-    def visit_DotAccessor(self, node):
-        # we skip identifier
-        self.visit(node.node)
-
     def visit_Identifier(self, node):
-        if getattr(node, 'scope', None) is not None:
+        if self._is_id_in_expr(node):
             self._fill_scope_refs(node.value, node.scope)
 
-    def _fill_scope_refs(self, name, scope):
+    @staticmethod
+    def _is_id_in_expr(node):
+        """Return True if Identifier node is part of an expression."""
+        return (
+            getattr(node, 'scope', None) is not None and
+            getattr(node, '_in_expression', False)
+            )
+
+    @staticmethod
+    def _fill_scope_refs(name, scope):
         """Put referenced name in 'ref' dictionary of a scope.
 
         Walks up the scope tree and adds the name to 'ref' of every scope
@@ -172,37 +157,22 @@ class NameManglerVisitor(Visitor):
     mangled names.
     """
 
-    def visit_VarDecl(self, node):
-        self.visit(node.identifier)
-        self.visit(node.initializer)
-
-    def visit_FuncDecl(self, node):
-        if node.identifier is not None:
-            self.visit(node.identifier)
+    @staticmethod
+    def _is_mangle_candidate(id_node):
+        """Return True if Identifier node is a candidate for mangling.
 
-        for param in node.parameters:
-            self.visit(param)
-
-        for element in node.elements:
-            self.visit(element)
-
-    visit_FuncExpr = visit_FuncDecl
-
-    def visit_Assign(self, node):
-        # property_assignment
-        # skip identifier
-        if node.op == ':' and isinstance(node.left, ast.Identifier):
-            self.visit(node.right)
-        else:
-            self.visit(node.left)
-            self.visit(node.right)
-
-    def visit_DotAccessor(self, node):
-        self.visit(node.node)
+        There are 5 cases when Identifier is a mangling candidate:
+        1. Function declaration identifier
+        2. Function expression identifier
+        3. Function declaration/expression parameter
+        4. Variable declaration identifier
+        5. Identifier is a part of an expression (primary_expr_no_brace rule)
+        """
+        return getattr(id_node, '_mangle_candidate', False)
 
     def visit_Identifier(self, node):
         """Mangle names."""
-        if getattr(node, 'scope', None) is None:
+        if not self._is_mangle_candidate(node):
             return
         name = node.value
         symbol = node.scope.resolve(node.value)