Align transformers with Kolasu

.github/workflows/pythonapp.yml

@@ -36,4 +36,4 @@ jobs:
       - name: Test with pytest
         run: |
           pip install pytest pytest-cov pyecore==0.12.2
-          pytest --cov=pylasu --cov-fail-under=40 tests
+          pytest --cov=pylasu --cov-fail-under=50 tests

pylasu/emf/model.py

@@ -40,12 +40,13 @@ def to_eobject(self: Node, resource: Resource, mappings=None):
         raise Exception("Unknown classifier for " + str(type(self)))
     eobject = eclass()
     mappings[id(self)] = eobject
-    for (p, v) in self.properties:
+    for p in self.properties:
+        v = p.value
         ev = translate_value(v, resource, mappings)
         if isinstance(v, list):
-            eobject.eGet(p).extend(ev)
+            eobject.eGet(p.name).extend(ev)
         else:
-            eobject.eSet(p, ev)
+            eobject.eSet(p.name, ev)
     return eobject
 
 

pylasu/model/model.py

@@ -1,10 +1,10 @@
-import enum
 import inspect
 from abc import ABC, abstractmethod, ABCMeta
 from dataclasses import Field, MISSING, dataclass, field
 from typing import Optional, Callable, List
 
 from .position import Position, Source
+from .reflection import Multiplicity, PropertyDescription
 from ..reflection import getannotations
 from ..reflection.reflection import is_sequence_type, get_type_arguments
 
@@ -73,22 +73,6 @@ def is_internal_property_or_method(value):
     return isinstance(value, internal_property) or isinstance(value, InternalField) or isinstance(value, Callable)
 
 
-class Multiplicity(enum.Enum):
-    OPTIONAL = 0
-    SINGULAR = 1
-    MANY = 2
-
-
-@dataclass
-class PropertyDescriptor:
-    name: str
-    provides_nodes: bool
-    multiplicity: Multiplicity = Multiplicity.SINGULAR
-
-    def multiple(self):
-        return self.multiplicity == Multiplicity.MANY
-
-
 def provides_nodes(decl_type):
     return isinstance(decl_type, type) and issubclass(decl_type, Node)
 
@@ -123,11 +107,11 @@ def _direct_node_properties(cls, cl, known_property_names):
                     else:
                         is_child_property = provides_nodes(decl_type)
                 known_property_names.add(name)
-                yield PropertyDescriptor(name, is_child_property, multiplicity)
+                yield PropertyDescription(name, is_child_property, multiplicity)
         for name in dir(cl):
             if name not in known_property_names and cls.is_node_property(name):
                 known_property_names.add(name)
-                yield PropertyDescriptor(name, False)
+                yield PropertyDescription(name, False)
 
     def is_node_property(cls, name):
         return not name.startswith('_') \
@@ -178,10 +162,8 @@ def source(self) -> Optional[Source]:
 
     @internal_property
     def properties(self):
-        return ((name, getattr(self, name)) for name in dir(self)
-                if not name.startswith('_')
-                and name not in self.__internal_properties__
-                and name not in [n for n, v in inspect.getmembers(type(self), is_internal_property_or_method)])
+        return (PropertyDescription(p.name, p.provides_nodes, p.multiplicity, getattr(self, p.name))
+                for p in self.__class__.node_properties)
 
     @internal_property
     def _fields(self):

pylasu/model/processing.py

@@ -21,7 +21,7 @@ def assign_parents(self: Node):
 
 
 def children(self: Node):
-    yield from nodes_in(v for _, v in self.properties)
+    yield from nodes_in(p.value for p in self.properties)
 
 
 Node.children = internal_property(children)
@@ -44,7 +44,9 @@ def search_by_type(self: Node, target_type, walker=walk):
 
 @extension_method(Node)
 def transform_children(self: Node, operation: Callable[[Node], Node]):
-    for name, value in self.properties:
+    for prop in self.properties:
+        name = prop.name
+        value = prop.value
         if isinstance(value, Node):
             new_value = operation(value)
             if new_value != value:

pylasu/model/reflection.py

@@ -0,0 +1,20 @@
+import enum
+from dataclasses import dataclass
+
+
+class Multiplicity(enum.Enum):
+    OPTIONAL = 0
+    SINGULAR = 1
+    MANY = 2
+
+
+@dataclass
+class PropertyDescription:
+    name: str
+    provides_nodes: bool
+    multiplicity: Multiplicity = Multiplicity.SINGULAR
+    value: object = None
+
+    @property
+    def multiple(self):
+        return self.multiplicity == Multiplicity.MANY

pylasu/testing/testing.py

@@ -1,5 +1,67 @@
+import unittest
+
 from pylasu.model import Node
 
 
-def assert_asts_are_equal(expected: Node, actual: Node, context: str = "<root>", consider_position: bool = False):
-    raise NotImplementedError("TODO implement this. Transformers tests don't use this yet.")
+def assert_asts_are_equal(
+        case: unittest.TestCase,
+        expected: Node, actual: Node,
+        context: str = "<root>", consider_position: bool = False
+):
+    if expected.node_type != actual.node_type:
+        case.fail(f"{context}: expected node of type {expected.node_type}, "
+                  f"but found {actual.node_type}")
+    if consider_position:
+        case.assertEqual(expected.position, actual.position, f"{context}.position")
+    for expected_property in expected.properties:
+        try:
+            actual_property = next(filter(lambda p: p.name == expected_property.name, actual.properties))
+        except StopIteration:
+            case.fail(f"No property {expected_property.name} found at {context}")
+        actual_prop_value = actual_property.value
+        expected_prop_value = expected_property.value
+        if expected_property.provides_nodes:
+            if expected_property.multiple:
+                assert_multi_properties_are_equal(
+                    case, expected_property, expected_prop_value, actual_prop_value, context, consider_position)
+            else:
+                assert_single_properties_are_equal(case, expected_property, expected_prop_value, actual_prop_value,
+                                                   context, consider_position)
+        # TODO not yet supported elif expected_property.property_type == PropertyType.REFERENCE:
+        else:
+            case.assertEqual(
+                expected_prop_value, actual_prop_value,
+                f"{context}, comparing property {expected_property.name} of {expected.node_type}")
+
+
+def assert_single_properties_are_equal(case, expected_property, expected_prop_value, actual_prop_value, context,
+                                       consider_position):
+    if expected_prop_value is None and actual_prop_value is not None:
+        case.assertEqual(expected_prop_value, actual_prop_value,
+                         f"{context}.{expected_property.name}")
+    elif expected_prop_value is not None and actual_prop_value is None:
+        case.assertEqual(expected_prop_value, actual_prop_value,
+                         f"{context}.{expected_property.name}")
+    elif expected_prop_value is None and actual_prop_value is None:
+        # that is ok
+        pass
+    else:
+        case.assertIsInstance(actual_prop_value, Node)
+        assert_asts_are_equal(
+            case, expected_prop_value, actual_prop_value,
+            context=f"{context}.{expected_property.name}",
+            consider_position=consider_position)
+
+
+def assert_multi_properties_are_equal(case, expected_property, expected_prop_value, actual_prop_value, context,
+                                      consider_position):
+    # TODO IgnoreChildren
+    case.assertEquals(actual_prop_value is None, expected_prop_value is None,
+                      f"{context}.{expected_property.name} nullness")
+    if actual_prop_value is not None and expected_prop_value is not None:
+        case.assertEquals(len(actual_prop_value), len(expected_prop_value),
+                          f"{context}.{expected_property.name} length")
+    for expected_it, actual_it, i in \
+            zip(expected_prop_value, actual_prop_value, range(len(expected_prop_value))):
+        assert_asts_are_equal(case, expected_it, actual_it, f"{context}[{i}]",
+                              consider_position=consider_position)

pylasu/transformation/generic_nodes.py

@@ -6,4 +6,4 @@
 @dataclass
 class GenericNode(Node):
     """A generic AST node. We use it to represent parts of a source tree that we don't know how to translate yet."""
-    parent: Node
+    parent: Node = None

pylasu/transformation/transformation.py

@@ -5,7 +5,7 @@
 
 from pylasu.model import Node, Origin
 from pylasu.model.errors import GenericErrorNode
-from pylasu.model.model import PropertyDescriptor
+from pylasu.model.reflection import PropertyDescription
 from pylasu.transformation.generic_nodes import GenericNode
 from pylasu.validation import Issue, IssueSeverity
 
@@ -36,8 +36,8 @@ class NodeFactory(Generic[Source, Output]):
 
     def with_child(
             self,
-            getter: Union[Callable[[Source], Optional[Any]], PropertyRef],
             setter: Union[Callable[[Target, Optional[Child]], None], PropertyRef],
+            getter: Union[Callable[[Source], Optional[Any]], PropertyRef],
             name: Optional[str] = None,
             target_type: Optional[type] = None
     ) -> "NodeFactory[Source, Output]":
@@ -140,10 +140,12 @@ def process_child(self, source, node, pd, factory):
     def as_origin(self, source: Any) -> Optional[Origin]:
         return source if isinstance(source, Origin) else None
 
-    def set_child(self, child_node_factory: ChildNodeFactory, source: Any, node: Node, pd: PropertyDescriptor):
+    def set_child(self, child_node_factory: ChildNodeFactory, source: Any, node: Node, pd: PropertyDescription):
         src = child_node_factory.get(self.get_source(node, source))
-        if pd.multiple():
-            child = [self.transform(it, node) for it in src or [] if it is not None]
+        if pd.multiple:
+            child = []
+            for child_src in src:
+                child.extend(self.transform_into_nodes(child_src, node))
         else:
             child = self.transform(src, node)
         try:
@@ -191,13 +193,16 @@ def register_identity_transformation(self, node_class: Type[Target]):
         self.register_node_factory(node_class, lambda node: node)
 
 
-def get_node_constructor_wrapper(decorated_function):
-    def ensure_list(obj):
-        if isinstance(obj, list):
-            return obj
-        else:
-            return [obj]
+def ensure_list(obj):
+    if isinstance(obj, list):
+        return obj
+    elif obj is not None:
+        return [obj]
+    else:
+        return []
+
 
+def get_node_constructor_wrapper(decorated_function):  # noqa C901
     try:
         sig = signature(decorated_function)
         try:

tests/transformation/test_parse_tree_to_ast_transformers.py → tests/mapping/test_parse_tree_to_ast_transformers.py

@@ -49,8 +49,8 @@ class ParseTreeToASTTransformerTest(unittest.TestCase):
 
     def test_simple_entities_transformer(self):
         transformer = ParseTreeToASTTransformer(allow_generic_node=False)
-        transformer.register_node_factory(AntlrEntityParser.ModuleContext, lambda ctx: EModule(name=ctx.name.text))\
-            .with_child(AntlrEntityParser.ModuleContext.entity, PropertyRef("entities"))
+        transformer.register_node_factory(AntlrEntityParser.ModuleContext, lambda ctx: EModule(name=ctx.name.text)) \
+            .with_child(PropertyRef("entities"), AntlrEntityParser.ModuleContext.entity)
         transformer.register_node_factory(AntlrEntityParser.EntityContext, lambda ctx: EEntity(name=ctx.name.text))
         expected_ast = EModule("M", [EEntity("FOO", []), EEntity("BAR", [])])
         actual_ast = transformer.transform(self.parse_entities("""
@@ -64,11 +64,11 @@ def test_simple_entities_transformer(self):
     def test_entities_with_features_transformer(self):
         transformer = ParseTreeToASTTransformer(allow_generic_node=False)
         transformer.register_node_factory(AntlrEntityParser.ModuleContext, lambda ctx: EModule(name=ctx.name.text)) \
-            .with_child(AntlrEntityParser.ModuleContext.entity, PropertyRef("entities"))
+            .with_child(PropertyRef("entities"), AntlrEntityParser.ModuleContext.entity)
         transformer.register_node_factory(AntlrEntityParser.EntityContext, lambda ctx: EEntity(name=ctx.name.text)) \
-            .with_child(AntlrEntityParser.EntityContext.feature, PropertyRef("features"))
-        transformer.register_node_factory(AntlrEntityParser.FeatureContext, lambda ctx: EFeature(name=ctx.name.text))\
-            .with_child(AntlrEntityParser.FeatureContext.type_spec, PropertyRef("type"))
+            .with_child(PropertyRef("features"), AntlrEntityParser.EntityContext.feature)
+        transformer.register_node_factory(AntlrEntityParser.FeatureContext, lambda ctx: EFeature(name=ctx.name.text)) \
+            .with_child(PropertyRef("type"), AntlrEntityParser.FeatureContext.type_spec)
         transformer.register_node_factory(AntlrEntityParser.Boolean_typeContext, EBooleanType)
         transformer.register_node_factory(AntlrEntityParser.String_typeContext, EStringType)
         transformer.register_node_factory(

tests/test_model.py

@@ -3,13 +3,14 @@
 from typing import List
 
 from pylasu.model import Node, Position, Point
-from pylasu.model.model import Multiplicity
+from pylasu.model.reflection import Multiplicity
 from pylasu.model.naming import ReferenceByName, Named, Scope, Symbol
 
 
 @dataclasses.dataclass
 class SomeNode(Node, Named):
     foo = 3
+    bar: int = dataclasses.field(init=False)
     __private__ = 4
     ref: Node = None
     multiple: List[Node] = dataclasses.field(default_factory=list)
@@ -66,17 +67,17 @@ def test_node_with_position(self):
 
     def test_node_properties(self):
         node = SomeNode("n").with_position(Position(Point(1, 0), Point(2, 1)))
-        self.assertIsNotNone(next(n for n, _ in node.properties if n == 'foo'))
-        self.assertIsNotNone(next(n for n, _ in node.properties if n == 'bar'))
-        self.assertIsNotNone(next(n for n, _ in node.properties if n == "name"))
+        self.assertIsNotNone(next(n for n in node.properties if n.name == 'foo'))
+        self.assertIsNotNone(next(n for n in node.properties if n.name == 'bar'))
+        self.assertIsNotNone(next(n for n in node.properties if n.name == "name"))
         with self.assertRaises(StopIteration):
-            next(n for n, _ in node.properties if n == '__private__')
+            next(n for n in node.properties if n.name == '__private__')
         with self.assertRaises(StopIteration):
-            next(n for n, _ in node.properties if n == 'non_existent')
+            next(n for n in node.properties if n.name == 'non_existent')
         with self.assertRaises(StopIteration):
-            next(n for n, _ in node.properties if n == 'properties')
+            next(n for n in node.properties if n.name == 'properties')
         with self.assertRaises(StopIteration):
-            next(n for n, _ in node.properties if n == "origin")
+            next(n for n in node.properties if n.name == "origin")
 
     def test_scope_lookup_0(self):
         """Symbol found in local scope with name and default type"""
@@ -137,13 +138,15 @@ def test_scope_lookup_7(self):
 
     def test_node_properties_meta(self):
         pds = [pd for pd in sorted(SomeNode.node_properties, key=lambda x: x.name)]
-        self.assertEqual(4, len(pds))
-        self.assertEqual("foo", pds[0].name)
+        self.assertEqual(5, len(pds))
+        self.assertEqual("bar", pds[0].name)
         self.assertFalse(pds[0].provides_nodes)
-        self.assertEqual("multiple", pds[1].name)
-        self.assertTrue(pds[1].provides_nodes)
-        self.assertEqual(Multiplicity.MANY, pds[1].multiplicity)
-        self.assertEqual("name", pds[2].name)
-        self.assertFalse(pds[2].provides_nodes)
-        self.assertEqual("ref", pds[3].name)
-        self.assertTrue(pds[3].provides_nodes)
+        self.assertEqual("foo", pds[1].name)
+        self.assertFalse(pds[1].provides_nodes)
+        self.assertEqual("multiple", pds[2].name)
+        self.assertTrue(pds[2].provides_nodes)
+        self.assertEqual(Multiplicity.MANY, pds[2].multiplicity)
+        self.assertEqual("name", pds[3].name)
+        self.assertFalse(pds[3].provides_nodes)
+        self.assertEqual("ref", pds[4].name)
+        self.assertTrue(pds[4].provides_nodes)