diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 3e51ef6..5e358ba 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -36,4 +36,4 @@ jobs: - name: Test with pytest run: | pip install pytest pytest-cov pyecore==0.12.2 - pytest --cov=pylasu --cov-fail-under=40 tests + pytest --cov=pylasu --cov-fail-under=50 tests diff --git a/pylasu/emf/model.py b/pylasu/emf/model.py index 60baa92..ca840bf 100644 --- a/pylasu/emf/model.py +++ b/pylasu/emf/model.py @@ -40,12 +40,13 @@ def to_eobject(self: Node, resource: Resource, mappings=None): raise Exception("Unknown classifier for " + str(type(self))) eobject = eclass() mappings[id(self)] = eobject - for (p, v) in self.properties: + for p in self.properties: + v = p.value ev = translate_value(v, resource, mappings) if isinstance(v, list): - eobject.eGet(p).extend(ev) + eobject.eGet(p.name).extend(ev) else: - eobject.eSet(p, ev) + eobject.eSet(p.name, ev) return eobject diff --git a/pylasu/model/model.py b/pylasu/model/model.py index ca966da..d0452d6 100644 --- a/pylasu/model/model.py +++ b/pylasu/model/model.py @@ -1,10 +1,10 @@ -import enum import inspect from abc import ABC, abstractmethod, ABCMeta from dataclasses import Field, MISSING, dataclass, field from typing import Optional, Callable, List from .position import Position, Source +from .reflection import Multiplicity, PropertyDescription from ..reflection import getannotations from ..reflection.reflection import is_sequence_type, get_type_arguments @@ -73,22 +73,6 @@ def is_internal_property_or_method(value): return isinstance(value, internal_property) or isinstance(value, InternalField) or isinstance(value, Callable) -class Multiplicity(enum.Enum): - OPTIONAL = 0 - SINGULAR = 1 - MANY = 2 - - -@dataclass -class PropertyDescriptor: - name: str - provides_nodes: bool - multiplicity: Multiplicity = Multiplicity.SINGULAR - - def multiple(self): - return self.multiplicity == Multiplicity.MANY - - def provides_nodes(decl_type): return isinstance(decl_type, type) and issubclass(decl_type, Node) @@ -123,11 +107,11 @@ def _direct_node_properties(cls, cl, known_property_names): else: is_child_property = provides_nodes(decl_type) known_property_names.add(name) - yield PropertyDescriptor(name, is_child_property, multiplicity) + yield PropertyDescription(name, is_child_property, multiplicity) for name in dir(cl): if name not in known_property_names and cls.is_node_property(name): known_property_names.add(name) - yield PropertyDescriptor(name, False) + yield PropertyDescription(name, False) def is_node_property(cls, name): return not name.startswith('_') \ @@ -178,10 +162,8 @@ def source(self) -> Optional[Source]: @internal_property def properties(self): - return ((name, getattr(self, name)) for name in dir(self) - if not name.startswith('_') - and name not in self.__internal_properties__ - and name not in [n for n, v in inspect.getmembers(type(self), is_internal_property_or_method)]) + return (PropertyDescription(p.name, p.provides_nodes, p.multiplicity, getattr(self, p.name)) + for p in self.__class__.node_properties) @internal_property def _fields(self): diff --git a/pylasu/model/processing.py b/pylasu/model/processing.py index 6598dfe..cc170a7 100644 --- a/pylasu/model/processing.py +++ b/pylasu/model/processing.py @@ -21,7 +21,7 @@ def assign_parents(self: Node): def children(self: Node): - yield from nodes_in(v for _, v in self.properties) + yield from nodes_in(p.value for p in self.properties) Node.children = internal_property(children) @@ -44,7 +44,9 @@ def search_by_type(self: Node, target_type, walker=walk): @extension_method(Node) def transform_children(self: Node, operation: Callable[[Node], Node]): - for name, value in self.properties: + for prop in self.properties: + name = prop.name + value = prop.value if isinstance(value, Node): new_value = operation(value) if new_value != value: diff --git a/pylasu/model/reflection.py b/pylasu/model/reflection.py new file mode 100644 index 0000000..f9d7def --- /dev/null +++ b/pylasu/model/reflection.py @@ -0,0 +1,20 @@ +import enum +from dataclasses import dataclass + + +class Multiplicity(enum.Enum): + OPTIONAL = 0 + SINGULAR = 1 + MANY = 2 + + +@dataclass +class PropertyDescription: + name: str + provides_nodes: bool + multiplicity: Multiplicity = Multiplicity.SINGULAR + value: object = None + + @property + def multiple(self): + return self.multiplicity == Multiplicity.MANY diff --git a/pylasu/testing/testing.py b/pylasu/testing/testing.py index e9aec99..6c43916 100644 --- a/pylasu/testing/testing.py +++ b/pylasu/testing/testing.py @@ -1,5 +1,67 @@ +import unittest + from pylasu.model import Node -def assert_asts_are_equal(expected: Node, actual: Node, context: str = "", consider_position: bool = False): - raise NotImplementedError("TODO implement this. Transformers tests don't use this yet.") +def assert_asts_are_equal( + case: unittest.TestCase, + expected: Node, actual: Node, + context: str = "", consider_position: bool = False +): + if expected.node_type != actual.node_type: + case.fail(f"{context}: expected node of type {expected.node_type}, " + f"but found {actual.node_type}") + if consider_position: + case.assertEqual(expected.position, actual.position, f"{context}.position") + for expected_property in expected.properties: + try: + actual_property = next(filter(lambda p: p.name == expected_property.name, actual.properties)) + except StopIteration: + case.fail(f"No property {expected_property.name} found at {context}") + actual_prop_value = actual_property.value + expected_prop_value = expected_property.value + if expected_property.provides_nodes: + if expected_property.multiple: + assert_multi_properties_are_equal( + case, expected_property, expected_prop_value, actual_prop_value, context, consider_position) + else: + assert_single_properties_are_equal(case, expected_property, expected_prop_value, actual_prop_value, + context, consider_position) + # TODO not yet supported elif expected_property.property_type == PropertyType.REFERENCE: + else: + case.assertEqual( + expected_prop_value, actual_prop_value, + f"{context}, comparing property {expected_property.name} of {expected.node_type}") + + +def assert_single_properties_are_equal(case, expected_property, expected_prop_value, actual_prop_value, context, + consider_position): + if expected_prop_value is None and actual_prop_value is not None: + case.assertEqual(expected_prop_value, actual_prop_value, + f"{context}.{expected_property.name}") + elif expected_prop_value is not None and actual_prop_value is None: + case.assertEqual(expected_prop_value, actual_prop_value, + f"{context}.{expected_property.name}") + elif expected_prop_value is None and actual_prop_value is None: + # that is ok + pass + else: + case.assertIsInstance(actual_prop_value, Node) + assert_asts_are_equal( + case, expected_prop_value, actual_prop_value, + context=f"{context}.{expected_property.name}", + consider_position=consider_position) + + +def assert_multi_properties_are_equal(case, expected_property, expected_prop_value, actual_prop_value, context, + consider_position): + # TODO IgnoreChildren + case.assertEquals(actual_prop_value is None, expected_prop_value is None, + f"{context}.{expected_property.name} nullness") + if actual_prop_value is not None and expected_prop_value is not None: + case.assertEquals(len(actual_prop_value), len(expected_prop_value), + f"{context}.{expected_property.name} length") + for expected_it, actual_it, i in \ + zip(expected_prop_value, actual_prop_value, range(len(expected_prop_value))): + assert_asts_are_equal(case, expected_it, actual_it, f"{context}[{i}]", + consider_position=consider_position) diff --git a/pylasu/transformation/generic_nodes.py b/pylasu/transformation/generic_nodes.py index 11361d2..8c0862e 100644 --- a/pylasu/transformation/generic_nodes.py +++ b/pylasu/transformation/generic_nodes.py @@ -6,4 +6,4 @@ @dataclass class GenericNode(Node): """A generic AST node. We use it to represent parts of a source tree that we don't know how to translate yet.""" - parent: Node + parent: Node = None diff --git a/pylasu/transformation/transformation.py b/pylasu/transformation/transformation.py index 4000176..fe06fdd 100644 --- a/pylasu/transformation/transformation.py +++ b/pylasu/transformation/transformation.py @@ -5,7 +5,7 @@ from pylasu.model import Node, Origin from pylasu.model.errors import GenericErrorNode -from pylasu.model.model import PropertyDescriptor +from pylasu.model.reflection import PropertyDescription from pylasu.transformation.generic_nodes import GenericNode from pylasu.validation import Issue, IssueSeverity @@ -36,8 +36,8 @@ class NodeFactory(Generic[Source, Output]): def with_child( self, - getter: Union[Callable[[Source], Optional[Any]], PropertyRef], setter: Union[Callable[[Target, Optional[Child]], None], PropertyRef], + getter: Union[Callable[[Source], Optional[Any]], PropertyRef], name: Optional[str] = None, target_type: Optional[type] = None ) -> "NodeFactory[Source, Output]": @@ -140,10 +140,12 @@ def process_child(self, source, node, pd, factory): def as_origin(self, source: Any) -> Optional[Origin]: return source if isinstance(source, Origin) else None - def set_child(self, child_node_factory: ChildNodeFactory, source: Any, node: Node, pd: PropertyDescriptor): + def set_child(self, child_node_factory: ChildNodeFactory, source: Any, node: Node, pd: PropertyDescription): src = child_node_factory.get(self.get_source(node, source)) - if pd.multiple(): - child = [self.transform(it, node) for it in src or [] if it is not None] + if pd.multiple: + child = [] + for child_src in src: + child.extend(self.transform_into_nodes(child_src, node)) else: child = self.transform(src, node) try: @@ -191,13 +193,16 @@ def register_identity_transformation(self, node_class: Type[Target]): self.register_node_factory(node_class, lambda node: node) -def get_node_constructor_wrapper(decorated_function): - def ensure_list(obj): - if isinstance(obj, list): - return obj - else: - return [obj] +def ensure_list(obj): + if isinstance(obj, list): + return obj + elif obj is not None: + return [obj] + else: + return [] + +def get_node_constructor_wrapper(decorated_function): # noqa C901 try: sig = signature(decorated_function) try: diff --git a/tests/mapping/__init__.py b/tests/mapping/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/transformation/test_parse_tree_to_ast_transformers.py b/tests/mapping/test_parse_tree_to_ast_transformers.py similarity index 87% rename from tests/transformation/test_parse_tree_to_ast_transformers.py rename to tests/mapping/test_parse_tree_to_ast_transformers.py index d15bb69..42e7c55 100644 --- a/tests/transformation/test_parse_tree_to_ast_transformers.py +++ b/tests/mapping/test_parse_tree_to_ast_transformers.py @@ -49,8 +49,8 @@ class ParseTreeToASTTransformerTest(unittest.TestCase): def test_simple_entities_transformer(self): transformer = ParseTreeToASTTransformer(allow_generic_node=False) - transformer.register_node_factory(AntlrEntityParser.ModuleContext, lambda ctx: EModule(name=ctx.name.text))\ - .with_child(AntlrEntityParser.ModuleContext.entity, PropertyRef("entities")) + transformer.register_node_factory(AntlrEntityParser.ModuleContext, lambda ctx: EModule(name=ctx.name.text)) \ + .with_child(PropertyRef("entities"), AntlrEntityParser.ModuleContext.entity) transformer.register_node_factory(AntlrEntityParser.EntityContext, lambda ctx: EEntity(name=ctx.name.text)) expected_ast = EModule("M", [EEntity("FOO", []), EEntity("BAR", [])]) actual_ast = transformer.transform(self.parse_entities(""" @@ -64,11 +64,11 @@ def test_simple_entities_transformer(self): def test_entities_with_features_transformer(self): transformer = ParseTreeToASTTransformer(allow_generic_node=False) transformer.register_node_factory(AntlrEntityParser.ModuleContext, lambda ctx: EModule(name=ctx.name.text)) \ - .with_child(AntlrEntityParser.ModuleContext.entity, PropertyRef("entities")) + .with_child(PropertyRef("entities"), AntlrEntityParser.ModuleContext.entity) transformer.register_node_factory(AntlrEntityParser.EntityContext, lambda ctx: EEntity(name=ctx.name.text)) \ - .with_child(AntlrEntityParser.EntityContext.feature, PropertyRef("features")) - transformer.register_node_factory(AntlrEntityParser.FeatureContext, lambda ctx: EFeature(name=ctx.name.text))\ - .with_child(AntlrEntityParser.FeatureContext.type_spec, PropertyRef("type")) + .with_child(PropertyRef("features"), AntlrEntityParser.EntityContext.feature) + transformer.register_node_factory(AntlrEntityParser.FeatureContext, lambda ctx: EFeature(name=ctx.name.text)) \ + .with_child(PropertyRef("type"), AntlrEntityParser.FeatureContext.type_spec) transformer.register_node_factory(AntlrEntityParser.Boolean_typeContext, EBooleanType) transformer.register_node_factory(AntlrEntityParser.String_typeContext, EStringType) transformer.register_node_factory( diff --git a/tests/test_model.py b/tests/test_model.py index a5773af..542071b 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -3,13 +3,14 @@ from typing import List from pylasu.model import Node, Position, Point -from pylasu.model.model import Multiplicity +from pylasu.model.reflection import Multiplicity from pylasu.model.naming import ReferenceByName, Named, Scope, Symbol @dataclasses.dataclass class SomeNode(Node, Named): foo = 3 + bar: int = dataclasses.field(init=False) __private__ = 4 ref: Node = None multiple: List[Node] = dataclasses.field(default_factory=list) @@ -66,17 +67,17 @@ def test_node_with_position(self): def test_node_properties(self): node = SomeNode("n").with_position(Position(Point(1, 0), Point(2, 1))) - self.assertIsNotNone(next(n for n, _ in node.properties if n == 'foo')) - self.assertIsNotNone(next(n for n, _ in node.properties if n == 'bar')) - self.assertIsNotNone(next(n for n, _ in node.properties if n == "name")) + self.assertIsNotNone(next(n for n in node.properties if n.name == 'foo')) + self.assertIsNotNone(next(n for n in node.properties if n.name == 'bar')) + self.assertIsNotNone(next(n for n in node.properties if n.name == "name")) with self.assertRaises(StopIteration): - next(n for n, _ in node.properties if n == '__private__') + next(n for n in node.properties if n.name == '__private__') with self.assertRaises(StopIteration): - next(n for n, _ in node.properties if n == 'non_existent') + next(n for n in node.properties if n.name == 'non_existent') with self.assertRaises(StopIteration): - next(n for n, _ in node.properties if n == 'properties') + next(n for n in node.properties if n.name == 'properties') with self.assertRaises(StopIteration): - next(n for n, _ in node.properties if n == "origin") + next(n for n in node.properties if n.name == "origin") def test_scope_lookup_0(self): """Symbol found in local scope with name and default type""" @@ -137,13 +138,15 @@ def test_scope_lookup_7(self): def test_node_properties_meta(self): pds = [pd for pd in sorted(SomeNode.node_properties, key=lambda x: x.name)] - self.assertEqual(4, len(pds)) - self.assertEqual("foo", pds[0].name) + self.assertEqual(5, len(pds)) + self.assertEqual("bar", pds[0].name) self.assertFalse(pds[0].provides_nodes) - self.assertEqual("multiple", pds[1].name) - self.assertTrue(pds[1].provides_nodes) - self.assertEqual(Multiplicity.MANY, pds[1].multiplicity) - self.assertEqual("name", pds[2].name) - self.assertFalse(pds[2].provides_nodes) - self.assertEqual("ref", pds[3].name) - self.assertTrue(pds[3].provides_nodes) + self.assertEqual("foo", pds[1].name) + self.assertFalse(pds[1].provides_nodes) + self.assertEqual("multiple", pds[2].name) + self.assertTrue(pds[2].provides_nodes) + self.assertEqual(Multiplicity.MANY, pds[2].multiplicity) + self.assertEqual("name", pds[3].name) + self.assertFalse(pds[3].provides_nodes) + self.assertEqual("ref", pds[4].name) + self.assertTrue(pds[4].provides_nodes) diff --git a/tests/transformation/__init__.py b/tests/transformation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/transformation/test_ast_transformers.py b/tests/transformation/test_ast_transformers.py index e3a319a..7917379 100644 --- a/tests/transformation/test_ast_transformers.py +++ b/tests/transformation/test_ast_transformers.py @@ -4,6 +4,8 @@ from typing import List from pylasu.model import Node +from pylasu.testing.testing import assert_asts_are_equal +from pylasu.transformation.generic_nodes import GenericNode from pylasu.transformation.transformation import ASTTransformer, PropertyRef, ast_transformer @@ -112,6 +114,45 @@ def test_translate_across_languages(self): ), ALangIntLiteral(4)))) + def test_dropping_nodes(self): + prop = PropertyRef("statements") + transformer = ASTTransformer() + transformer.register_node_factory(CU, CU).with_child(prop, prop) + transformer.register_node_factory(DisplayIntStatement, lambda _: None) + transformer.register_identity_transformation(SetStatement) + cu = CU(statements=[DisplayIntStatement(value=456), SetStatement(variable="foo", value=123)]) + transformed_cu = transformer.transform(cu) + # TODO not yet supported self.assertTrue(transformed_cu.hasValidParents()) + self.assertEqual(transformed_cu.origin, cu) + self.assertEqual(1, len(transformed_cu.statements)) + assert_asts_are_equal(self, cu.statements[1], transformed_cu.statements[0]) + + def test_nested_origin(self): + prop = PropertyRef("statements") + transformer = ASTTransformer() + transformer.register_node_factory(CU, CU).with_child(prop, prop) + transformer.register_node_factory(DisplayIntStatement, lambda s: s.with_origin(GenericNode())) + cu = CU(statements=[DisplayIntStatement(value=456)]) + transformed_cu = transformer.transform(cu) + # TODO not yet supported self.assertTrue(transformed_cu.hasValidParents()) + self.assertEqual(transformed_cu.origin, cu) + self.assertIsInstance(transformed_cu.statements[0].origin, GenericNode) + + def test_transforming_one_node_to_many(self): + prop = PropertyRef("stmts") + transformer = ASTTransformer(allow_generic_node=False) + transformer.register_node_factory(BarRoot, BazRoot).with_child(prop, prop) + transformer.register_node_factory(BarStmt, lambda s: [BazStmt(f"{s.desc}-1"), BazStmt(f"{s.desc}-2")]) + original = BarRoot([BarStmt("a"), BarStmt("b")]) + transformed = transformer.transform(original) + # TODO not yet supported assertTrue { transformed.hasValidParents() } + self.assertEquals(transformed.origin, original) + assert_asts_are_equal( + self, + BazRoot([BazStmt("a-1"), BazStmt("a-2"), BazStmt("b-1"), BazStmt("b-2")]), + transformed + ) + @dataclass class ALangExpression(Node): @@ -157,5 +198,25 @@ class BLangMult(BLangExpression): right: BLangExpression +@dataclass +class BarStmt(Node): + desc: str + + +@dataclass +class BarRoot(Node): + stmts: List[BarStmt] = field(default_factory=list) + + +@dataclass +class BazStmt(Node): + desc: str + + +@dataclass +class BazRoot(Node): + stmts: List[BazStmt] = field(default_factory=list) + + if __name__ == '__main__': unittest.main()