Add node attribute bracket syntax (#112)

* Update grammar and tests * Enforce quoted attributes * Update neuPrint and Neo4j executors * Update documentation with examples * Update changelog * Update version for 0.11.0 deploy * Fix tests to match new attribute escapes * Add quote escaping tests
aplbrain · Apr 13, 2022 · ec6f653 · ec6f653
1 parent 6f30f24
commit ec6f653
Show file tree

Hide file tree

Showing 9 changed files with 156 additions and 26 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+-   **0.11.0** (April 12 2022)
+    -   Features:
+        -   Added support for attributes that include spaces and other non-word/variable characters, and updated Cypher syntax for these attributes when searching with Neo4j and neuPrint (#112)
 -   **0.10.1** (January 26 2022)
     -   Bugfixes: Fixed compatibility with grandiso 2.1 and up
 -   **0.10.0** (September 30 2021)

diff --git a/docs/examples/Searching in neuPrint.py b/docs/examples/Searching in neuPrint.py
@@ -0,0 +1,17 @@
+import dotmotif
+from dotmotif.executors.NeuPrintExecutor import NeuPrintExecutor
+
+HOSTNAME = "neuprint.janelia.org"
+DATASET = "hemibrain:v1.2.1"
+TOKEN = "[YOUR TOKEN HERE]"
+
+motif = dotmotif.Motif(
+    """
+    A -> B
+    A['AVLP(R)'] = True
+    """
+)
+
+E = NeuPrintExecutor(HOSTNAME, DATASET, TOKEN)
+
+E.find(motif, limit=2)
diff --git a/dotmotif/__init__.py b/dotmotif/__init__.py
@@ -30,7 +30,7 @@
 from .executors.GrandIsoExecutor import GrandIsoExecutor
 from .executors.Neo4jExecutor import Neo4jExecutor
 
-__version__ = "0.10.1"
+__version__ = "0.11.0"
 
 DEFAULT_MOTIF_PARSER = ParserV2
 

diff --git a/dotmotif/executors/Neo4jExecutor.py b/dotmotif/executors/Neo4jExecutor.py
@@ -1,5 +1,5 @@
 """
-Copyright 2021 The Johns Hopkins University Applied Physics Laboratory.
+Copyright 2022 The Johns Hopkins University Applied Physics Laboratory.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -67,6 +67,29 @@ def _operator_negation_infix(op):
     }[op]
 
 
+def _quoted_if_necessary(val: str) -> str:
+    """
+    If the value is already quoted, return it.
+
+    If it has single quotes in it, nest it in double quotes.
+    If it has double quotes in it, nest it in single quotes.
+    If it has both, nest it in double quotes and escape the double quotes in
+    the existing text (i.e., [foo"bar] becomes ["foo\"bar"])
+    """
+    if val.startswith('"') and val.endswith('"'):
+        return val
+    elif val.startswith("'") and val.endswith("'"):
+        return val
+    elif '"' in val and "'" in val:
+        return '"' + val.replace('"', '\\"') + '"'
+    elif '"' in val:
+        return "'" + val + "'"
+    elif "'" in val:
+        return '"' + val + '"'
+    else:
+        return '"' + val + '"'
+
+
 _LOOKUP = {
     "INHIBITS": "INH",
     "EXCITES": "EXC",
@@ -312,7 +335,9 @@ def find(self, motif: "dotmotif.Motif", limit=None, cursor=True):
 
     @staticmethod
     def motif_to_cypher(
-        motif: "dotmotif.Motif", count_only: bool = False, static_entity_labels: dict = None,
+        motif: "dotmotif.Motif",
+        count_only: bool = False,
+        static_entity_labels: dict = None,
     ) -> str:
         """
         Output a query suitable for Cypher-compatible engines (e.g. Neo4j).
@@ -396,12 +421,12 @@ def motif_to_cypher(
                     for value in values:
                         cypher_edge_constraints.append(
                             (
-                                "NOT ({}.{} {} {})"
+                                "NOT ({}[{}] {} {})"
                                 if _operator_negation_infix(operator)
-                                else "{}.{} {} {}"
+                                else "{}[{}] {} {}"
                             ).format(
                                 edge_mapping[(u, v)],
-                                key,
+                                _quoted_if_necessary(key),
                                 _remapped_operator(operator),
                                 f'"{value}"' if isinstance(value, str) else value,
                             )
@@ -415,12 +440,12 @@ def motif_to_cypher(
                     for value in values:
                         cypher_node_constraints.append(
                             (
-                                "NOT ({}.{} {} {})"
+                                "NOT ({}[{}] {} {})"
                                 if _operator_negation_infix(operator)
-                                else "{}.{} {} {}"
+                                else "{}[{}] {} {}"
                             ).format(
                                 n,
-                                key,
+                                _quoted_if_necessary(key),
                                 _remapped_operator(operator),
                                 f'"{value}"' if isinstance(value, str) else value,
                             )
@@ -433,15 +458,15 @@ def motif_to_cypher(
                     for value in values:
                         cypher_node_constraints.append(
                             (
-                                "NOT ({}.{} {} {}.{})"
+                                "NOT ({}[{}] {} {}[{}])"
                                 if _operator_negation_infix(operator)
-                                else "{}.{} {} {}.{}"
+                                else "{}[{}] {} {}[{}]"
                             ).format(
                                 n,
-                                key,
+                                _quoted_if_necessary(key),
                                 _remapped_operator(operator),
                                 value[0],
-                                value[1],
+                                _quoted_if_necessary(value[1]),
                             )
                         )
 

diff --git a/dotmotif/executors/test_dm_cypher.py b/dotmotif/executors/test_dm_cypher.py
@@ -25,22 +25,22 @@
 _DEMO_EDGE_ATTR_CYPHER = """
 MATCH (A:Neuron)-[A_B:SYN]->(B:Neuron)
 MATCH (X:Neuron)-[X_Y:SYN]->(Y:Neuron)
-WHERE A_B.weight = 4 AND A_B.area <= 10 AND X_Y.weight = 2
+WHERE A_B["weight"] = 4 AND A_B["area"] <= 10 AND X_Y["weight"] = 2
 RETURN DISTINCT A,B,X,Y
 """
 
 
 _DEMO_EDGE_ATTR_CYPHER_2 = """
 MATCH (A:Neuron)-[A_B:SYN]->(B:Neuron)
 MATCH (X:Neuron)-[X_Y:SYN]->(Y:Neuron)
-WHERE A_B.weight = 4 AND A_B.area <= 10 AND A_B.area <= 20 AND X_Y.weight = 2
+WHERE A_B["weight"] = 4 AND A_B["area"] <= 10 AND A_B["area"] <= 20 AND X_Y["weight"] = 2
 RETURN DISTINCT A,B,X,Y
 """
 
 _DEMO_EDGE_ATTR_CYPHER_2_ENF_INEQ = """
 MATCH (A:Neuron)-[A_B:SYN]->(B:Neuron)
 MATCH (X:Neuron)-[X_Y:SYN]->(Y:Neuron)
-WHERE A_B.weight = 4 AND A_B.area <= 10 AND A_B.area <= 20 AND X_Y.weight = 2 AND B<>X AND B<>Y AND A<>X AND A<>B AND A<>Y AND X<>Y
+WHERE A_B["weight"] = 4 AND A_B["area"] <= 10 AND A_B["area"] <= 20 AND X_Y["weight"] = 2 AND B<>X AND B<>Y AND A<>X AND A<>B AND A<>Y AND X<>Y
 RETURN DISTINCT A,B,X,Y
 """
 
@@ -109,7 +109,7 @@ def test_cypher_node_attributes(self):
 
         self.assertEqual(
             Neo4jExecutor.motif_to_cypher(dm).strip(),
-            """MATCH (A:Neuron)-[A_B:SYN]->(B:Neuron)\nWHERE A.size = "big"\nRETURN DISTINCT A,B""".strip(),
+            """MATCH (A:Neuron)-[A_B:SYN]->(B:Neuron)\nWHERE A["size"] = "big"\nRETURN DISTINCT A,B""".strip(),
         )
 
     def test_cypher_node_many_attributes(self):
@@ -124,7 +124,7 @@ def test_cypher_node_many_attributes(self):
 
         self.assertEqual(
             Neo4jExecutor.motif_to_cypher(dm).strip(),
-            """MATCH (A:Neuron)-[A_B:SYN]->(B:Neuron)\nWHERE A.size = "big" AND A.type = "funny"\nRETURN DISTINCT A,B""".strip(),
+            """MATCH (A:Neuron)-[A_B:SYN]->(B:Neuron)\nWHERE A["size"] = "big" AND A["type"] = "funny"\nRETURN DISTINCT A,B""".strip(),
         )
 
     def test_cypher_node_same_node_many_attributes(self):
@@ -139,7 +139,7 @@ def test_cypher_node_same_node_many_attributes(self):
 
         self.assertEqual(
             Neo4jExecutor.motif_to_cypher(dm).strip(),
-            """MATCH (A:Neuron)-[A_B:SYN]->(B:Neuron)\nWHERE A.personality <> "exciting" AND A.personality <> "funny"\nRETURN DISTINCT A,B""".strip(),
+            """MATCH (A:Neuron)-[A_B:SYN]->(B:Neuron)\nWHERE A["personality"] <> "exciting" AND A["personality"] <> "funny"\nRETURN DISTINCT A,B""".strip(),
         )
 
     def test_cypher_node_many_node_attributes(self):
@@ -154,7 +154,7 @@ def test_cypher_node_many_node_attributes(self):
 
         self.assertEqual(
             Neo4jExecutor.motif_to_cypher(dm).strip(),
-            """MATCH (A:Neuron)-[A_B:SYN]->(B:Neuron)\nWHERE A.area <= 10 AND B.area <= 10\nRETURN DISTINCT A,B""".strip(),
+            """MATCH (A:Neuron)-[A_B:SYN]->(B:Neuron)\nWHERE A["area"] <= 10 AND B["area"] <= 10\nRETURN DISTINCT A,B""".strip(),
         )
 
     def test_cypher_negative_edge_and_inequality(self):
@@ -193,7 +193,7 @@ def test_cypher_node_and_edge_attributes(self):
             Neo4jExecutor.motif_to_cypher(dm).strip(),
             (
                 "MATCH (A:Neuron)-[A_B:SYN]->(B:Neuron)\n"
-                "WHERE A.area <= 10 AND B.area <= 10 AND A_B.area <> 10\n"
+                'WHERE A["area"] <= 10 AND B["area"] <= 10 AND A_B["area"] <> 10\n'
                 "RETURN DISTINCT A,B"
             ).strip(),
         )
@@ -210,7 +210,7 @@ def test_dynamic_constraints_in_cypher(self):
         """
         )
         self.assertIn(
-            "WHERE A.radius >= B.radius AND A.zorf <> B.zorf",
+            """WHERE A["radius"] >= B["radius"] AND A["zorf"] <> B["zorf"]""",
             Neo4jExecutor.motif_to_cypher(dm).strip(),
         )
 
@@ -224,4 +224,3 @@ def test_fix_where_clause__github_35(self):
         """
         )
         self.assertIn("WHERE", Neo4jExecutor.motif_to_cypher(dm).strip())
-
diff --git a/dotmotif/executors/test_neo4jexecutor.py b/dotmotif/executors/test_neo4jexecutor.py
@@ -1,5 +1,5 @@
 import dotmotif
-from dotmotif.executors.Neo4jExecutor import Neo4jExecutor
+from dotmotif.executors.Neo4jExecutor import Neo4jExecutor, _quoted_if_necessary
 import unittest
 import networkx as nx
 
@@ -31,3 +31,14 @@ def test_automatic_autos(self):
         dm = dotmotif.Motif(exp, exclude_automorphisms=True)
         cypher = Neo4jExecutor.motif_to_cypher(dm)
         self.assertIn("id(A) < id(B)", cypher)
+
+
+class TestQuotingIfNecessary(unittest.TestCase):
+    def test_quoting_if_necessary(self):
+        self.assertEqual(_quoted_if_necessary('"xyz"'), '"xyz"'),
+        self.assertEqual(_quoted_if_necessary("'xyz'"), "'xyz'"),
+        self.assertEqual(_quoted_if_necessary("""x'y"z"""), '''"x'y\\\"z"'''),
+        self.assertEqual(_quoted_if_necessary('foo "bar"'), "'foo \"bar\"'"),
+        self.assertEqual(_quoted_if_necessary("""don't break"""), '''"don't break"'''),
+        self.assertEqual(_quoted_if_necessary("foo bar"), '"foo bar"')
+        self.assertEqual(_quoted_if_necessary("foo"), '"foo"')
diff --git a/dotmotif/parsers/v2/grammar.lark b/dotmotif/parsers/v2/grammar.lark
@@ -88,11 +88,41 @@ edge_clause             : key op value
 
 
 // Node constraints:
+                        // Dot Notation
 node_constraint         : node_id "." key op value_or_quoted_value
                         | node_id "." key op node_id "." key
-
+                        // Left-Hand Bracket Notation
+                        | node_id "[" flex_key "]" op value_or_quoted_value
+                        | node_id "['" flex_key "']" op value_or_quoted_value
+                        | node_id "[\"" flex_key "\"]" op value_or_quoted_value
+                        // LH and RH Bracket Notation
+                        | node_id "[" flex_key "]" op node_id "[" flex_key "]"
+                        | node_id "['" flex_key "']" op node_id "[" flex_key "]"
+                        | node_id "[\"" flex_key "\"]" op node_id "[" flex_key "]"
+                        | node_id "[" flex_key "]" op node_id "['" flex_key "']"
+                        | node_id "['" flex_key "']" op node_id "['" flex_key "']"
+                        | node_id "[\"" flex_key "\"]" op node_id "['" flex_key "']"
+                        | node_id "[" flex_key "]" op node_id "[\"" flex_key "\"]"
+                        | node_id "['" flex_key "']" op node_id "[\"" flex_key "\"]"
+                        | node_id "[\"" flex_key "\"]" op node_id "[\"" flex_key "\"]"
+
+                        // Dot Notation
 macro_node_constraint   : node_id "." key op value_or_quoted_value
                         | node_id "." key op node_id "." key
+                        // Left-Hand Bracket Notation
+                        | node_id "[" flex_key "]" op value_or_quoted_value
+                        | node_id "['" flex_key "']" op value_or_quoted_value
+                        | node_id "[\"" flex_key "\"]" op value_or_quoted_value
+                        // LH and RH Bracket Notation
+                        | node_id "[" flex_key "]" op node_id "[" flex_key "]"
+                        | node_id "['" flex_key "']" op node_id "[" flex_key "]"
+                        | node_id "[\"" flex_key "\"]" op node_id "[" flex_key "]"
+                        | node_id "[" flex_key "]" op node_id "['" flex_key "']"
+                        | node_id "['" flex_key "']" op node_id "['" flex_key "']"
+                        | node_id "[\"" flex_key "\"]" op node_id "['" flex_key "']"
+                        | node_id "[" flex_key "]" op node_id "[\"" flex_key "\"]"
+                        | node_id "['" flex_key "']" op node_id "[\"" flex_key "\"]"
+                        | node_id "[\"" flex_key "\"]" op node_id "[\"" flex_key "\"]"
 
 
 // Automorphism notation:
@@ -102,6 +132,7 @@ automorphism_notation: node_id "===" node_id
 
 
 ?key            : WORD | variable
+?flex_key       : WORD | variable | FLEXIBLE_KEY
 ?value          : WORD | NUMBER
 ?op             : OPERATOR | iter_ops
 
@@ -114,13 +145,15 @@ iter_ops        : "contains"                        -> iter_op_contains
                 | "!in"                             -> iter_op_not_in
 
 NAME            : /[a-zA-Z_-]\w*/
+FLEXIBLE_KEY    : "\"" /.+?/ /(?<!\\)(\\\\)*?/ "\"" | "'" /.+?/ /(?<!\\)(\\\\)*?/ "'"
 OPERATOR        : /(?:[\!=\>\<][=]?)|(?:\<\>)/
 VAR_SEP         : /[\_\-]/
 COMMENT         : /#[^\n]+/
 DOUBLE_QUOTED_STRING  : /"[^"]*"/
 %ignore COMMENT
 
 %import common.WORD -> WORD
+%import common._STRING_ESC_INNER -> _STRING_ESC_INNER
 %import common.SIGNED_NUMBER  -> NUMBER
 %import common.WS
 %ignore WS
diff --git a/dotmotif/parsers/v2/test_v2_parser.py b/dotmotif/parsers/v2/test_v2_parser.py
@@ -349,6 +349,16 @@ def test_basic_node_attr(self):
         self.assertEqual(len(dm.list_node_constraints()), 1)
         self.assertEqual(list(dm.list_node_constraints().keys()), ["Aa"])
 
+    def test_basic_node_attr_bracket_keying(self):
+        exp = """\
+        Aa -> Ba
+
+        Aa['type'] = "excitatory"
+        """
+        dm = dotmotif.Motif(exp)
+        self.assertEqual(len(dm.list_node_constraints()), 1)
+        self.assertEqual(list(dm.list_node_constraints().keys()), ["Aa"])
+
     def test_node_multi_attr(self):
         exp = """\
         Aa -> Ba
@@ -416,6 +426,20 @@ def test_dynamic_constraints(self):
         dm = dotmotif.Motif(exp)
         self.assertEqual(len(dm.list_dynamic_node_constraints()), 1)
 
+    def test_dynamic_constraints_brackets(self):
+        """
+        Test that comparisons may be made between variables, e.g.:
+
+        A.type != B.type
+
+        """
+        exp = """\
+        A -> B
+        A['radius'] < B["radius"]
+        """
+        dm = dotmotif.Motif(exp)
+        self.assertEqual(len(dm.list_dynamic_node_constraints()), 1)
+
     def test_dynamic_constraints_in_macro(self):
         """
         Test that comparisons may be made between variables in a macro, e.g.:
@@ -432,3 +456,21 @@ def test_dynamic_constraints_in_macro(self):
         """
         dm = dotmotif.Motif(exp)
         self.assertEqual(len(dm.list_dynamic_node_constraints()), 1)
+
+    def test_dynamic_constraints_bracketed_in_macro(self):
+        """
+        Test that comparisons may be made between variables in a macro, e.g.:
+
+        A.type != B.type
+
+        """
+        exp = """\
+        macro(A, B) {
+            # A.radius > B.radius
+            A["radius"] > B['radius']
+        }
+        macro(A, B)
+        A -> B
+        """
+        dm = dotmotif.Motif(exp)
+        self.assertEqual(len(dm.list_dynamic_node_constraints()), 1)
diff --git a/setup.py b/setup.py
@@ -11,7 +11,7 @@
 twine upload dist/*
 """
 
-VERSION = "0.10.1"
+VERSION = "0.11.0"
 
 here = os.path.abspath(os.path.dirname(__file__))
 with io.open(os.path.join(here, "README.md"), encoding="utf-8") as f: