compiler (#6)

* Moved compile logic in compile class

* Removed unnecessary comment

* Removed compile method from syntax tree

* Moved compiler class to a separate file

* Add docstrings to syntax tree file

* Fix formatting of docstrings

* Moved compile test to a new file

* Removed unused imports

* Added test for base snippet visitor

* Few renames

* Implemented more complete compile functionality

* Added tests for compiler

* Added cleanup after compile

* Compiler documentation added

docs/contributing.rst

@@ -38,6 +38,7 @@ The code is separated in several components:
 * `Parser`_
 * `Syntax tree`_
 * `Snippet provider`_
+* `Compiler`_
 * `Utilities`_
 * `Application frontend`_
 
@@ -93,6 +94,61 @@ Using this class should be straightforward. Look.
     snippet = "for"
     snippetExpansion = provider[snippet] # snippetExpansion == "for(#){$}" if used json from above
 
+Compiler
+^^^^^^^^
+
+Compiler is responsible for turning a syntax tree into output code. It uses `snippet provider`_ for simple snippets.
+General rules used by compiler will be discussed in this section.
+
+Simple substitution
+"""""""""""""""""""
+
+Consider the following snippet definition.
+
+.. code-block:: json
+
+    {"name": "if","language": "C++","snippet": "if(#){$}"}
+
+Snippet :code:`if#true$i=3;` is expanded in the following way:
+
+* :code:`if` becomes :code:`if(#){$}` from the definition.
+* :code:`#` gets replaced by :code:`true` to get :code:`if(true){$}`.
+* :code:`$` gets replaced by :code:`i=3;` to get the final output :code:`if(true){i=3;}`.
+
+The value of an operator gets replace by the value provided in the snippet.
+This is done for every operator to get the final result.
+
+Definition expansion
+""""""""""""""""""""
+
+Consider a snippet for a function call. Writing :code:`fun!foo#a#b#c` should return :code:`foo(a,b,c)`.
+To write a single snippet definition for all functions would mean supporting variable number of parameters.
+
+This is possible using snippet definitions inside snippets.
+
+.. code-block:: json
+
+    [
+    {"name": "fun","language": "python","snippet": "!({{params}})"},
+    {"name": "params","language": "python","snippet": "#{{opt_params}}"},
+    {"name": "opt_params","language": "python","snippet": ", #{{opt_params}}"}
+    ]
+
+Snippet :code:`fun!foo#a#b` is expanded in the following way:
+
+* :code:`fun` becomes :code:`!({{params}})` from the definition.
+* :code:`!` gets replaced by :code:`foo` to get :code:`foo({{params}})`.
+* :code:`#` does not exist in :code:`foo({{params}})` so :code:`{{params}}` get expanded to :code:`#{{opt_params}}`.
+* :code:`#` gets replaced by :code:`a;` to get :code:`foo(a{{opt_params}})`
+* :code:`#` does not exist in :code:`foo(a{{opt_params}})` so :code:`{{opt_params}}`
+  get expanded to :code:`, #{{opt_params}}`.
+* :code:`#` gets replaced by :code:`b;` to get :code:`foo(a, b{{opt_params}})`.
+* :code:`{{opt_params}}` gets removed from final result to get :code:`foo(a,b)`.
+
+Expansion is done in case `simple substitution`_ can't be done.
+This enables recursive constructs as shown in the example above.
+Number of expansion performed is capped to prevent infinite recursions.
+
 Utilities
 ^^^^^^^^^
 

homotopy/__main__.py

@@ -1,14 +1,17 @@
 import argparse
 import homotopy.parser
 
+from homotopy.compiler import Compiler
+
 
 def main():
     parser = argparse.ArgumentParser(description="Compile a snippet.")
     parser.add_argument('snippet', nargs=1, type=str, help='a snippet to be compiled')
 
     snippet = parser.parse_args().snippet[0]
 
-    print(homotopy.parser.parser.parse(snippet).compile())
+    print(Compiler().compile(homotopy.parser.parser.parse(snippet)))
+
 
 if __name__ == "__main__":
     main()

homotopy/compiler.py

@@ -0,0 +1,41 @@
+from homotopy.syntax_tree import SnippetVisitor
+from homotopy.snippet_provider import snippetProvider
+
+import re
+import logging
+
+
+class Compiler(SnippetVisitor):
+    def __init__(self, expansion_level_count=2):
+        self.expansion_level_count = expansion_level_count
+
+    def visit_composite_snippet(self, composite_snippet):
+        left_side = snippetProvider[self.visit(composite_snippet.left)]
+        right_side = snippetProvider[self.visit(composite_snippet.right)]
+
+        if composite_snippet.operation in left_side:
+            return left_side.replace(composite_snippet.operation, right_side)
+        else:
+            expanded_left_side = left_side
+
+            for _ in range(self.expansion_level_count):
+                expanded_left_side = re.sub(
+                    r'{{(.*?)}}',
+                    lambda match_object: snippetProvider[match_object.group(1)],
+                    expanded_left_side,
+                    count=1)
+
+                if composite_snippet.operation in expanded_left_side:
+                    return expanded_left_side.replace(composite_snippet.operation, right_side)
+
+        logging.warning("No match found. Ignoring right side of the snippet.")
+        return left_side
+
+    def visit_simple_snippet(self, simple_snippet):
+        return snippetProvider[simple_snippet.value]
+
+    def compile(self, snippet):
+        compiled_snippet = self.visit(snippet)
+
+        return re.sub(r'({{.*?}})', "", compiled_snippet)
+

homotopy/syntax_tree.py

@@ -1,16 +1,29 @@
-from homotopy.snippet_provider import snippetProvider
-
-
 class Snippet:
-    def compile(self):
-        pass
+    """
+    Base class for snippet syntax tree.
+    """
+    def accept(self, visitor):
+        """
+        Accepts a visitor
+
+        :param visitor: Visitor instance
+        :return: Visitor result
+        """
+        raise NotImplementedError("You should not be here.")
 
 
 class CompositeSnippet(Snippet):
     """
     CompositeSnippet compose two snippets with the operand
     """
     def __init__(self, left, operation, right):
+        """
+        Initialize CompositeSnippet instance.
+
+        :param left: Left subtree
+        :param operation: Operation
+        :param right: Right subtree
+        """
         self.left = left
         self.operation = operation
         self.right = right
@@ -25,16 +38,20 @@ def __eq__(self, other):
                    self.right == other.right
         return False
 
-    def compile(self):
-        return snippetProvider[self.left.compile()].replace(self.operation, snippetProvider[self.right.compile()])
-
+    def accept(self, visitor):
+        return visitor.visit_composite_snippet(self)
 
 
 class SimpleSnippet(Snippet):
     """
     BasicSnippet can be directly compiled.
     """
     def __init__(self, value):
+        """
+        Initialize SnipleSnippet instance.
+
+        :param value: Value of the snippet
+        """
         self.value = value
 
     def __repr__(self):
@@ -45,5 +62,32 @@ def __eq__(self, other):
             return self.value == other.value
         return False
 
-    def compile(self):
-        return snippetProvider[self.value]
+    def accept(self, visitor):
+        return visitor.visit_simple_snippet(self)
+
+
+class SnippetVisitor:
+    """
+    Base class for visitors working on syntax tree.
+    """
+    def visit(self, snippet):
+        return snippet.accept(self)
+
+    def visit_composite_snippet(self, composite_snippet):
+        """
+        Base visit logic for composite snippets. Process right and left subtrees recursively.
+
+        :param composite_snippet: CompositeSnippet instance
+        :return: None
+        """
+        composite_snippet.left.accept(self)
+        composite_snippet.right.accept(self)
+
+    def visit_simple_snippet(self, simple_snippet):
+        """
+        Base visit logic for simple snippets. Does nothing.
+
+        :param simple_snippet: SimpleSnippet instance
+        :return: None
+        """
+        pass

test/testCompiler.py

@@ -0,0 +1,80 @@
+from unittest import TestCase
+from unittest.mock import patch, MagicMock
+
+import homotopy.syntax_tree as st
+from homotopy.compiler import Compiler
+
+
+class TestCompiler(TestCase):
+    @patch('homotopy.snippet_provider.SnippetProvider.__getitem__')
+    def test_compile(self, mock_provider):
+        with self.assertRaises(NotImplementedError):
+            Compiler().compile(st.Snippet())
+
+        data = {
+            "for": "for # in !:\n\tpass",
+            "def": "def !({{params}}):\n\tpass",
+            "params": "#{{opt_params}}",
+            "opt_params": ", #{{opt_params}}",
+            "a1": "{{a2}}",
+            "a2": "{{a3}}",
+            "a3": "{{a4}}",
+            "a4": "#"
+        }
+
+        mock_provider.side_effect = lambda x: x if x not in data else data[x]
+
+        self.assertEqual(
+            Compiler().compile(st.CompositeSnippet(
+                st.CompositeSnippet(st.SimpleSnippet('for'), '#', st.SimpleSnippet('i')),
+                '!',
+                st.SimpleSnippet('data')
+            )),
+            'for i in data:\n\tpass'
+        )
+
+        with patch('logging.warning', MagicMock()) as m:
+            self.assertEqual(
+                Compiler().compile(st.CompositeSnippet(
+                    st.CompositeSnippet(st.SimpleSnippet('for'), '#', st.SimpleSnippet('i')),
+                    '%',
+                    st.SimpleSnippet('data')
+                )),
+                'for i in !:\n\tpass'
+            )
+
+            m.assert_called_once_with("No match found. Ignoring right side of the snippet.")
+
+        with patch('logging.warning', MagicMock()) as m:
+            self.assertEqual(
+                Compiler().compile(st.CompositeSnippet(st.SimpleSnippet('a1'), '#', st.SimpleSnippet('i'))),
+                ''
+            )
+
+            self.assertEqual(
+                Compiler(expansion_level_count=3).compile(st.CompositeSnippet(st.SimpleSnippet('a1'), '#', st.SimpleSnippet('i'))),
+                'i'
+            )
+
+            m.assert_called_once_with("No match found. Ignoring right side of the snippet.")
+
+        self.assertEqual(
+            Compiler().compile(st.CompositeSnippet(
+                st.CompositeSnippet(st.SimpleSnippet('def'), '!', st.SimpleSnippet('foo')),
+                '#',
+                st.SimpleSnippet('a')
+            )),
+            'def foo(a):\n\tpass'
+        )
+
+        self.assertEqual(
+            Compiler().compile(st.CompositeSnippet(
+                st.CompositeSnippet(
+                    st.CompositeSnippet(st.SimpleSnippet('def'), '!', st.SimpleSnippet('foo')),
+                    '#',
+                    st.SimpleSnippet('a')
+                ),
+                '#',
+                st.SimpleSnippet('b'))),
+            'def foo(a, b):\n\tpass'
+        )

test/testSyntax_tree.py

@@ -1,5 +1,4 @@
 from unittest import TestCase
-from unittest.mock import patch
 
 import homotopy.syntax_tree as st
 
@@ -22,17 +21,6 @@ def test_eq(self):
 
         self.assertFalse(st.SimpleSnippet('if') != st.SimpleSnippet('if'))
 
-    @patch('homotopy.snippet_provider.SnippetProvider.__getitem__')
-    def test_compile(self, mockProvider):
-        self.assertIsNone(st.Snippet().compile())
-
-        mockProvider.side_effect = lambda x: x if x != "for" else "for # in !:\n\tpass"
-
-        self.assertEqual(
-            st.CompositeSnippet(
-                st.CompositeSnippet(st.SimpleSnippet('for'), '#', st.SimpleSnippet('i')),
-                '!',
-                st.SimpleSnippet('data')
-            ).compile(),
-            'for i in data:\n\tpass'
-        )
+    def test_visitor(self):
+        self.assertIsNone(
+            st.SnippetVisitor().visit(st.CompositeSnippet(st.SimpleSnippet('if'), '$', st.SimpleSnippet('i==1'))))