Fix query negation in MongoDB

INSTALL.md

@@ -38,6 +38,9 @@ conda activate optimade
 # Install package and dependencies in editable mode (including "dev" requirements).
 pip install -e ".[dev]"
 
+# Retrieve a list of providers. (Without this list some of the test will fail because providers.json cannot be found.)
+git submodule update --init
+
 # Run the tests with pytest
 py.test
 

optimade/filtertransformers/mongo.py

@@ -34,6 +34,17 @@ class MongoTransformer(BaseTransformer):
         "=": "$eq",
     }
 
+    inverse_operator_map = {
+        "$lt": "$gte",
+        "$lte": "$gt",
+        "$gt": "$lte",
+        "$gte": "$lt",
+        "$ne": "$eq",
+        "$eq": "$ne",
+        "$in": "$nin",
+        "$nin": "$in",
+    }
+
     def postprocess(self, query: Dict[str, Any]):
         """Used to post-process the nested dictionary of the parsed query."""
         query = self._apply_relationship_filtering(query)
@@ -208,30 +219,82 @@ def property_zip_addon(self, arg):
 
     def _recursive_expression_phrase(self, arg):
         """Helper function for parsing `expression_phrase`. Recursively sorts out
-        the correct precedence for `$and`, `$or` and `$nor`.
+        the correct precedence for `$not', '$and` and `$or`.
 
         """
-        if len(arg) == 1:
-            # without NOT
-            return arg[0]
 
-        # handle the case of {"$not": {"$or": [expr1, expr2]}} using {"$nor": [expr1, expr2]}.
-        if "$or" in arg[1]:
-            return {"$nor": self._recursive_expression_phrase([arg[1]["$or"]])}
+        def handle_not_and(arg):
+            """Handle the case of `{"$not": {"$and": [expr1, expr2]}}`
+            which is equal to `{"or": [{"$not": expr1}, {"$not": expr2}]}`}
+            We have to check for the special case in which the "and" was created by a previous NOT
+            e.g `NOT (NOT ({"a": {"$eq" : 6}})) -> NOT({$and:[{"a": {"$ne" : 6}},{"a": {"$ne": None}}]})`
+
+            """
+
+            expr1 = arg["$and"][0]
+            expr2 = arg["$and"][1]
+            key = list(expr1.keys())[0]
+            if expr1.keys() == expr2.keys():
+                if expr1.get(key) == {"$ne": None}:
+                    return self._recursive_expression_phrase(["NOT", expr2])
+                elif expr2.get(key) == {"$ne": None}:
+                    return self._recursive_expression_phrase(["NOT", expr1])
+            return {
+                "$or": [
+                    self._recursive_expression_phrase(["NOT", subdict])
+                    for subdict in arg["$and"]
+                ]
+            }
+
+        def handle_not_or(arg):
+            """Handle the case of {"$not": {"$or": [expr1, expr2]}} using:
+            {"$not": {"$or": [expr1, expr2]}} == {"$and": [(NOT, expr1), (NOT, expr2)]}}
+            {"$nor": [expr1, expr2]} is not convenient because nor will also return documents where the field in expr1 or exp2 is missing.
+
+            """
 
-        # handle the case of {"$not": {"$and": [expr1, expr2]}} using per-expression negation,
-        # e.g. {"$and": [{prop1: {"$not": expr1}}, {prop2: {"$not": ~expr2}}]}.
-        # Note that this is not the same as NOT (expr1 AND expr2)!
-        if "$and" in arg[1]:
             return {
                 "$and": [
                     self._recursive_expression_phrase(["NOT", subdict])
-                    for subdict in arg[1]["$and"]
+                    for subdict in arg["$or"]
                 ]
             }
 
-        # simple case of negating one expression, from NOT (expr) to ~expr.
-        return {prop: {"$not": expr} for prop, expr in arg[1].items()}
+        if len(arg) == 1:
+            # without NOT
+            return arg[0]
+
+        if "$or" in arg[1]:
+            return handle_not_or(arg[1])
+
+        if "$and" in arg[1]:
+            return handle_not_and(arg[1])
+
+        # Incase the NOT operator occurs at the lowest nesting level,
+        # the expression can be simplified by using the opposite operator and removing the not.
+
+        for prop, expr in arg[1].items():
+            for operator, value in expr.items():
+                if operator in self.inverse_operator_map:
+                    if operator in ("$in", "$eq"):
+                        return {
+                            "$and": [
+                                {
+                                    prop: {
+                                        self.inverse_operator_map.get(operator): value
+                                    }
+                                },
+                                {prop: {"$ne": None}},
+                            ]
+                        }
+
+                    else:
+                        return {prop: {self.inverse_operator_map.get(operator): value}}
+                else:
+                    if "#known" in expr:
+                        return {prop: {"$not": expr}}
+                    else:
+                        return {"$and": [{prop: {"$not": expr}}, {prop: {"$ne": None}}]}
 
     def _apply_length_operators(self, filter_: dict) -> dict:
         """Check for any invalid pymongo queries that involve applying a
@@ -275,9 +338,7 @@ def apply_length_op(subdict, prop, expr):
             return subdict
 
         return recursive_postprocessing(
-            filter_,
-            check_for_length_op_filter,
-            apply_length_op,
+            filter_, check_for_length_op_filter, apply_length_op
         )
 
     def _apply_relationship_filtering(self, filter_: dict) -> dict:
@@ -307,11 +368,7 @@ def replace_with_relationship(subdict, prop, expr):
                     subdict["$and"] = []
                 subdict["$and"].extend(
                     [
-                        {
-                            f"relationships.{_prop}.data": {
-                                "$size": expr.pop("$size"),
-                            }
-                        },
+                        {f"relationships.{_prop}.data": {"$size": expr.pop("$size")}},
                         {f"relationships.{_prop}.data.{_field}": expr},
                     ]
                 )