Align mongodb4.4

.github/workflows/python-package.yml

@@ -19,17 +19,16 @@ jobs:
       matrix:
         experimental: [ false ]
         monty-storage: [ memory, flatfile, sqlite ]
-        mongodb-version: [ "3.6", "4.0", "4.2" ]
+        mongodb-version: [ "3.6", "4.0", "4.2", "4.4", "5.0", "6.0" ]
         python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
-        poetry-version: [ "1.3" ]
+
         include:
           # run lmdb tests as experimental due to the seg fault in GitHub
           #  action is not reproducible on my Windows and Mac.
           - experimental: true
             monty-storage: lightning
             mongodb-version: "4.0"
             python-version: "3.7"
-            poetry-version: "1.3"
 
     steps:
     - uses: actions/checkout@v3
@@ -44,11 +43,11 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Setup Poetry ${{ matrix.poetry-version }}
+    - name: Setup Poetry 1.3
       uses: abatilo/actions-poetry@v2
       with:
-        poetry-version: ${{ matrix.poetry-version }}
-      
+        poetry-version: "1.3"
+
     - name: Install dependencies via poetry
       run: make install
 

Makefile

@@ -25,7 +25,7 @@ lint:  ## Run linting with flake8
 	poetry run flake8 . \
 	--count \
 	--ignore=F841,W503,E741 \
-	--max-complexity=26 \
+	--max-complexity=32 \
 	--max-line-length=88 \
 	--statistics \
 	--exclude .venv,venv

montydb/client.py

@@ -128,9 +128,9 @@ def get_database(self, name):
         """
         # verify database name
         if platform.system() == "Windows":
-            is_invalid = set('/\\. "$*<>:|?').intersection(set(name))
+            is_invalid = set(r'/\. "$*<>:|?').intersection(set(name))
         else:
-            is_invalid = set('/\\. "$').intersection(set(name))
+            is_invalid = set(r'/\. "$').intersection(set(name))
 
         if is_invalid or not name:
             raise errors.OperationFailure("Invalid database name.")

montydb/configure.py

@@ -18,7 +18,7 @@
 
 URI_SCHEME_PREFIX = "montydb://"
 
-MONGO_COMPAT_VERSIONS = ("3.6", "4.0", "4.2", "4.4")  # 4.4 is experimenting
+MONGO_COMPAT_VERSIONS = ("3.6", "4.0", "4.2", "4.4", "5.0", "6.0")
 
 
 _pinned_repository = {"_": None}
@@ -296,27 +296,43 @@ def _bson_init(use_bson):
 
 
 def _mongo_compat(version):
-    from .engine import queries
+    from .engine import queries, project
 
-    def patch(mod, func, ver_func):
-        setattr(mod, func, getattr(mod, ver_func))
+    def patch(mod, attr, ver):
+        setattr(mod, attr, getattr(mod, attr + ver))
 
     if version.startswith("3"):
-        patch(queries, "_is_comparable", "_is_comparable_ver3")
-        patch(queries, "_regex_options_check", "_regex_options_")
-        patch(queries, "_mod_remainder_not_num", "_mod_remainder_not_num_")
+        patch(queries, "_is_comparable", "_ver3")
+        patch(queries, "_regex_options", "_")
+        patch(queries, "_mod_check_numeric_remainder", "_")
+        patch(project, "_positional_mismatch", "_")
+        patch(project, "_check_positional_key", "_")
+        patch(project, "_check_path_collision", "_")
+        patch(project, "_include_positional_non_located_match", "_")
 
     elif version == "4.0":
-        patch(queries, "_is_comparable", "_is_comparable_ver4")
-        patch(queries, "_regex_options_check", "_regex_options_")
-        patch(queries, "_mod_remainder_not_num", "_mod_remainder_not_num_")
+        patch(queries, "_is_comparable", "_ver4")
+        patch(queries, "_regex_options", "_")
+        patch(queries, "_mod_check_numeric_remainder", "_")
+        patch(project, "_positional_mismatch", "_")
+        patch(project, "_check_positional_key", "_")
+        patch(project, "_check_path_collision", "_")
+        patch(project, "_include_positional_non_located_match", "_")
 
     elif version == "4.2":
-        patch(queries, "_is_comparable", "_is_comparable_ver4")
-        patch(queries, "_regex_options_check", "_regex_options_v42")
-        patch(queries, "_mod_remainder_not_num", "_mod_remainder_not_num_v42")
-
-    else:
-        patch(queries, "_is_comparable", "_is_comparable_ver4")
-        patch(queries, "_regex_options_check", "_regex_options_")
-        patch(queries, "_mod_remainder_not_num", "_mod_remainder_not_num_v42")
+        patch(queries, "_is_comparable", "_ver4")
+        patch(queries, "_regex_options", "_v42")
+        patch(queries, "_mod_check_numeric_remainder", "_v42_")
+        patch(project, "_positional_mismatch", "_")
+        patch(project, "_check_positional_key", "_")
+        patch(project, "_check_path_collision", "_")
+        patch(project, "_include_positional_non_located_match", "_")
+
+    else:  # 4.4+ (default)
+        patch(queries, "_is_comparable", "_ver4")
+        patch(queries, "_regex_options", "_")
+        patch(queries, "_mod_check_numeric_remainder", "_v42_")
+        patch(project, "_positional_mismatch", "_v44")
+        patch(project, "_check_positional_key", "_v44")
+        patch(project, "_check_path_collision", "_v44")
+        patch(project, "_include_positional_non_located_match", "_v44")

montydb/engine/project.py

@@ -35,6 +35,15 @@ def _is_positional_match(conditions, match_field):
         return None
 
 
+def _has_path_collision(path, parsed_paths):
+    path = path.split(".$")[0]
+    for parsed in parsed_paths:
+        if path == parsed \
+                or path.startswith(parsed + ".") \
+                or parsed.startswith(path + "."):
+            return parsed
+
+
 def _perr_doc(val):
     """
     For pretty error msg, same as Mongo
@@ -58,6 +67,15 @@ def _perr_doc(val):
     return "{ " + ", ".join(v_lis) + " }"
 
 
+_check_positional_key_ = False
+_check_positional_key_v44 = True
+_check_positional_key = _check_positional_key_v44
+
+_check_path_collision_ = False
+_check_path_collision_v44 = True
+_check_path_collision = _check_path_collision_v44
+
+
 class Projector(object):
     """ """
 
@@ -104,11 +122,10 @@ def __call__(self, fieldwalker):
                 fieldwalker.go(path).get()
 
             if self.include_flag:
-                located_match = None
-                if self.matched is not None:
-                    located_match = self.matched.located
-
-                projected = inclusion(fieldwalker, positioned, located_match, init_doc)
+                projected = inclusion(fieldwalker,
+                                      positioned,
+                                      self.matched,
+                                      init_doc)
             else:
                 projected = exclusion(fieldwalker, init_doc)
 
@@ -119,6 +136,22 @@ def parser(self, spec, qfilter):
         self.array_op_type = self.ARRAY_OP_NORMAL
 
         for key, val in spec.items():
+            # check path collision (mongo-4.4+)
+            if _check_path_collision:
+                collision = (
+                    _has_path_collision(key, self.regular_field)
+                    or _has_path_collision(key, self.array_field.keys())
+                )
+                if collision:
+                    remaining = key[len(collision + "."):]
+                    if remaining:
+                        raise OperationFailure(
+                            "Path collision at %s remaining portion %s"
+                            % (collision, remaining)
+                        )
+                    else:
+                        raise OperationFailure("Path collision at %s" % key)
+
             # Parsing options
             if is_duckument_type(val):
                 if not len(val) == 1:
@@ -180,6 +213,12 @@ def parser(self, spec, qfilter):
             elif key == "_id" and not _is_include(val):
                 self.proj_with_id = False
 
+            elif _check_positional_key and key.startswith("$."):
+                raise OperationFailure("FieldPath field names may not start "
+                                       "with '$'.")
+            elif _check_positional_key and key.endswith("."):
+                raise OperationFailure("FieldPath must not end with a '.'.")
+
             else:
                 # Normal field options, include or exclude.
                 flag = _is_include(val)
@@ -219,6 +258,17 @@ def parser(self, spec, qfilter):
                         "operator more than once.".format(key)
                     )
 
+                if _check_positional_key and ".$." in key:
+                    raise OperationFailure(
+                        "As of 4.4, it's illegal to specify positional "
+                        "operator in the middle of a path.Positional "
+                        "projection may only be used at the end, for example: "
+                        "a.b.$. If the query previously used a form like "
+                        "a.b.$.d, remove the parts following the '$' and the "
+                        "results will be equivalent.",
+                        code=31394
+                    )
+
                 path = key.split(".$", 1)[0]
                 conditions = qfilter.conditions
                 match_query = _is_positional_match(conditions, path)
@@ -305,10 +355,11 @@ def _positional(fieldwalker):
                             code=2,
                         )
 
-                    if int(
-                        matched_index
-                    ) >= elem_count and self.matched.full_path.startswith(
-                        node.full_path
+                    if _positional_mismatch(
+                            int(matched_index),
+                            elem_count,
+                            self.matched.full_path,
+                            node.full_path
                     ):
                         raise OperationFailure(
                             "Executor error during find command "
@@ -323,8 +374,20 @@ def _positional(fieldwalker):
         return _positional
 
 
-def inclusion(fieldwalker, positioned, located_match, init_doc):
+def _positional_mismatch_(matched, elem_count, matched_path, node_path):
+    return matched >= elem_count and matched_path.startswith(node_path)
+
+
+def _positional_mismatch_v44(matched, elem_count, matched_path, node_path):
+    return matched >= elem_count
+
+
+_positional_mismatch = _positional_mismatch_v44
+
+
+def inclusion(fieldwalker, positioned, matched, init_doc):
     _doc_type = fieldwalker.doc_type
+    located_match = False if matched is None else matched.located
 
     def _inclusion(node, init_doc=None):
         doc = node.value
@@ -358,7 +421,10 @@ def _inclusion(node, init_doc=None):
                         if isinstance(child.value, _doc_type):
                             new_doc.append(child.value)
                     else:
-                        new_doc.append(child.value)
+                        if _include_positional_non_located_match(matched, node):
+                            new_doc.append(child.value)
+                        else:
+                            new_doc.append(_doc_type())
 
                 return new_doc or _no_val
 
@@ -396,6 +462,18 @@ def _inclusion(node, init_doc=None):
     return _inclusion(fieldwalker.tree.root, init_doc)
 
 
+def _include_positional_non_located_match_(matched, node):
+    return True
+
+
+def _include_positional_non_located_match_v44(matched, node):
+    return matched.full_path.startswith(node.full_path)
+
+
+_include_positional_non_located_match = \
+    _include_positional_non_located_match_v44
+
+
 def exclusion(fieldwalker, init_doc):
     _doc_type = fieldwalker.doc_type
 

montydb/engine/queries.py

@@ -421,7 +421,7 @@ def _regex_options_v42(regex_flag, opt_flag):
         raise OperationFailure("options set in both $regex and $options")
 
 
-_regex_options_check = _regex_options_v42
+_regex_options = _regex_options_v42
 
 
 def _modify_regex_optins(sub_spec):
@@ -451,7 +451,7 @@ def _modify_regex_optins(sub_spec):
             _re = sub_spec["$regex"]
             sub_spec["$regex"] = None
 
-    _regex_options_check(regex_flags, opt_flags)
+    _regex_options(regex_flags, opt_flags)
 
     new_sub_spec = deepcopy(sub_spec)
     new_sub_spec["$regex"] = {
@@ -897,17 +897,11 @@ def _regex(fieldwalker):
     return _regex
 
 
-def _mod_remainder_not_num_():
-    pass
-
-
-def _mod_remainder_not_num_v42():
-    # mongo-4.2.19+
-    # https://jira.mongodb.org/browse/SERVER-23664
-    raise OperationFailure("malformed mod, remainder not a number")
-
-
-_mod_remainder_not_num = _mod_remainder_not_num_v42
+_mod_check_numeric_remainder_ = False
+_mod_check_numeric_remainder_v42_ = True
+_mod_check_numeric_remainder = _mod_check_numeric_remainder_v42_
+# mongo-4.2.19+
+# https://jira.mongodb.org/browse/SERVER-23664
 
 
 def parse_mod(query):
@@ -926,7 +920,8 @@ def parse_mod(query):
     if not isinstance(divisor, num_types):
         raise OperationFailure("malformed mod, divisor not a number")
     if not isinstance(remainder, num_types):
-        _mod_remainder_not_num()
+        if _mod_check_numeric_remainder:
+            raise OperationFailure("malformed mod, remainder not a number")
         remainder = 0
 
     if isinstance(divisor, bson.Decimal128):

pyproject.toml

@@ -40,6 +40,11 @@ codespell = "^2"
 black = {version = "*", python =">=3.6.2", allow-prereleases = true}
 bandit = "^1"
 
+[tool.pytest.ini_options]
+filterwarnings = [
+    "ignore::pytest.PytestUnraisableExceptionWarning",
+]
+
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"