Feature: optional batteries (#575)

* poetry -> flit * feature | merge indexes * version | 1.20.0b0
BeanieODM · Jun 9, 2023 · 2652c5a · 2652c5a
1 parent 02111b0
commit 2652c5a
Show file tree

Hide file tree

Showing 33 changed files with 296 additions and 180 deletions.
diff --git a/.github/workflows/github-actions-lint.yml b/.github/workflows/github-actions-lint.yml
@@ -9,7 +9,7 @@ jobs:
         python-version: [ 3.10.9 ]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}

diff --git a/.github/workflows/github-actions-mypy.yml b/.github/workflows/github-actions-mypy.yml
@@ -7,7 +7,7 @@ jobs:
       fail-fast: false
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-python@v2
         with:
           python-version: 3.10.9

diff --git a/.github/workflows/github-actions-publish-docs.yml b/.github/workflows/github-actions-publish-docs.yml
@@ -7,13 +7,11 @@ jobs:
   publish_docs:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-python@v2
         with:
           python-version: 3.10.9
-      - name: install poetry
-        run: pip install poetry
       - name: install dependencies
-        run: poetry install
+        run: pip3 instll .[doc]
       - name: publish docs
         run: bash scripts/publish_docs.sh
diff --git a/.github/workflows/github-actions-publish-project.yml b/.github/workflows/github-actions-publish-project.yml
@@ -7,13 +7,8 @@ jobs:
   publish_project:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.10.9
-      - name: install poetry
-        run: pip install poetry
-      - name: install dependencies
-        run: poetry install
+      - uses: actions/checkout@v3
+      - name: install flit
+        run: pip3 install flit
       - name: publish project
-        run: poetry publish --build --username __token__ --password ${{ secrets.PYPI_TOKEN }}
+        run: flit publish
diff --git a/.github/workflows/github-actions-tests.yml b/.github/workflows/github-actions-tests.yml
@@ -11,7 +11,7 @@ jobs:
         pydantic-version: [ 1.10.0 ]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
@@ -20,11 +20,9 @@ jobs:
         with:
           mongodb-version: ${{ matrix.mongodb-version }}
           mongodb-replica-set: test-rs
-      - name: install poetry
-        run: pip install poetry
       - name: install dependencies
-        run: poetry install
+        run: pip install .[test]
       - name: specify pydantic
-        run: poetry add pydantic==${{ matrix.pydantic-version }}
+        run: pip3 install pydantic==${{ matrix.pydantic-version }}
       - name: run tests
-        run: poetry run pytest
+        run: pytest
diff --git a/.gitignore b/.gitignore
@@ -184,4 +184,5 @@ docker-compose-aws.yml
 tilt_modules
 
 # Poetry stuff
-poetry.lock
+poetry.lock
+.pdm-python
diff --git a/.pypirc b/.pypirc
@@ -0,0 +1,14 @@
+[distutils]
+index-servers =
+    pypi
+    testpypi
+
+[pypi]
+repository = https://upload.pypi.org/legacy/
+username = __token__
+password = ${PYPI_TOKEN}
+
+[testpypi]
+repository = https://test.pypi.org/legacy/
+username = roman-right
+password = =$C[wT}^]5EWvX(p#9Po
diff --git a/beanie/__init__.py b/beanie/__init__.py
@@ -28,7 +28,7 @@
 from beanie.odm.views import View
 from beanie.odm.union_doc import UnionDoc
 
-__version__ = "1.19.2"
+__version__ = "1.20.0b0"
 __all__ = [
     # ODM
     "Document",

diff --git a/beanie/odm/bulk.py b/beanie/odm/bulk.py
@@ -42,7 +42,11 @@ async def __aenter__(self):
     async def __aexit__(self, exc_type, exc, tb):
         await self.commit()
 
-    async def commit(self) -> BulkWriteResult:
+    async def commit(self) -> Optional[BulkWriteResult]:
+        """
+        Commit all the operations to the database
+        :return:
+        """
         obj_class = None
         requests = []
         if self.operations:
@@ -55,16 +59,17 @@ async def commit(self) -> BulkWriteResult:
                         "All the operations should be for a single document model"
                     )
                 if op.operation in [InsertOne, DeleteOne]:
-                    query = op.operation(op.first_query, **op.pymongo_kwargs)
+                    query = op.operation(op.first_query, **op.pymongo_kwargs)  # type: ignore
                 else:
                     query = op.operation(
-                        op.first_query, op.second_query, **op.pymongo_kwargs
+                        op.first_query, op.second_query, **op.pymongo_kwargs  # type: ignore
                     )
                 requests.append(query)
 
             return await obj_class.get_motor_collection().bulk_write(  # type: ignore
                 requests, session=self.session
             )
+        return None
 
     def add_operation(self, operation: Operation):
         self.operations.append(operation)
diff --git a/beanie/odm/fields.py b/beanie/odm/fields.py
@@ -33,6 +33,7 @@
     In,
 )
 from beanie.odm.utils.parsing import parse_obj
+from pymongo import IndexModel
 
 if TYPE_CHECKING:
     from beanie.odm.documents import DocType
@@ -293,3 +294,85 @@ def to_dict(self):
 
 
 ENCODERS_BY_TYPE[BackLink] = lambda o: o.to_dict()
+
+
+class IndexModelField:
+    def __init__(self, index: IndexModel):
+        self.index = index
+        self.name = index.document["name"]
+
+        self.fields = tuple(sorted(self.index.document["key"]))
+        self.options = tuple(
+            sorted(
+                (k, v)
+                for k, v in self.index.document.items()
+                if k not in ["key", "v"]
+            )
+        )
+
+    def __eq__(self, other):
+        return self.fields == other.fields and self.options == other.options
+
+    def __repr__(self):
+        return f"IndexModelField({self.name}, {self.fields}, {self.options})"
+
+    @classmethod
+    def __get_validators__(cls):
+        yield cls.validate
+
+    @classmethod
+    def validate(cls, v):
+        if isinstance(v, IndexModel):
+            return IndexModelField(v)
+        else:
+            return IndexModelField(IndexModel(v))
+
+    @staticmethod
+    def list_difference(
+        left: List["IndexModelField"], right: List["IndexModelField"]
+    ):
+        result = []
+        for index in left:
+            if index not in right:
+                result.append(index)
+        return result
+
+    @staticmethod
+    def list_to_index_model(left: List["IndexModelField"]):
+        return [index.index for index in left]
+
+    @classmethod
+    def from_motor_index_information(cls, index_info: dict):
+        result = []
+        for name, details in index_info.items():
+            fields = details["key"]
+            if ("_id", 1) in fields:
+                continue
+
+            options = {k: v for k, v in details.items() if k != "key"}
+            index_model = IndexModelField(
+                IndexModel(fields, name=name, **options)
+            )
+            result.append(index_model)
+        return result
+
+    def same_fields(self, other: "IndexModelField"):
+        return self.fields == other.fields
+
+    @staticmethod
+    def find_index_with_the_same_fields(
+        indexes: List["IndexModelField"], index: "IndexModelField"
+    ):
+        for i in indexes:
+            if i.same_fields(index):
+                return i
+        return None
+
+    @staticmethod
+    def merge_indexes(
+        left: List["IndexModelField"], right: List["IndexModelField"]
+    ):
+        left_dict = {index.fields: index for index in left}
+        right_dict = {index.fields: index for index in right}
+        left_dict.update(right_dict)
+        return list(left_dict.values())
diff --git a/beanie/odm/settings/document.py b/beanie/odm/settings/document.py
@@ -1,73 +1,12 @@
 from typing import Optional, List
 
 from pydantic import Field
-from pymongo import IndexModel
 
+from beanie.odm.fields import IndexModelField
 from beanie.odm.settings.base import ItemSettings
 from beanie.odm.settings.timeseries import TimeSeriesConfig
 
 
-class IndexModelField:
-    def __init__(self, index: IndexModel):
-        self.index = index
-        self.name = index.document["name"]
-
-        self.fields = tuple(sorted(self.index.document["key"]))
-        self.options = tuple(
-            sorted(
-                (k, v)
-                for k, v in self.index.document.items()
-                if k not in ["key", "v"]
-            )
-        )
-
-    def __eq__(self, other):
-        return self.fields == other.fields and self.options == other.options
-
-    def __repr__(self):
-        return f"IndexModelField({self.name}, {self.fields}, {self.options})"
-
-    @classmethod
-    def __get_validators__(cls):
-        yield cls.validate
-
-    @classmethod
-    def validate(cls, v):
-        if isinstance(v, IndexModel):
-            return IndexModelField(v)
-        else:
-            return IndexModelField(IndexModel(v))
-
-    @staticmethod
-    def list_difference(
-        left: List["IndexModelField"], right: List["IndexModelField"]
-    ):
-        result = []
-        for index in left:
-            if index not in right:
-                result.append(index)
-        return result
-
-    @staticmethod
-    def list_to_index_model(left: List["IndexModelField"]):
-        return [index.index for index in left]
-
-    @classmethod
-    def from_motor_index_information(cls, index_info: dict):
-        result = []
-        for name, details in index_info.items():
-            fields = details["key"]
-            if ("_id", 1) in fields:
-                continue
-
-            options = {k: v for k, v in details.items() if k != "key"}
-            index_model = IndexModelField(
-                IndexModel(fields, name=name, **options)
-            )
-            result.append(index_model)
-        return result
-
-
 class DocumentSettings(ItemSettings):
     use_state_management: bool = False
     state_management_replace_objects: bool = False
@@ -77,6 +16,7 @@ class DocumentSettings(ItemSettings):
     single_root_inheritance: bool = False
 
     indexes: List[IndexModelField] = Field(default_factory=list)
+    merge_indexes: bool = False
     timeseries: Optional[TimeSeriesConfig] = None
 
     lazy_parsing: bool = False

diff --git a/beanie/odm/utils/find.py b/beanie/odm/utils/find.py
@@ -40,8 +40,7 @@ def construct_query(
             lookup_steps = [
                 {
                     "$lookup": {
-                        "from": link_info.model_class.get_motor_collection().name,
-                        # type: ignore
+                        "from": link_info.model_class.get_motor_collection().name,  # type: ignore
                         "localField": f"{link_info.lookup_field_name}.$id",
                         "foreignField": "_id",
                         "as": f"_link_{link_info.field_name}",
@@ -85,7 +84,7 @@ def construct_query(
             lookup_steps = [
                 {
                     "$lookup": {
-                        "from": link_info.model_class.get_motor_collection().name,
+                        "from": link_info.model_class.get_motor_collection().name,  # type: ignore
                         "let": {
                             "link_id": f"${link_info.lookup_field_name}.$id"
                         },
@@ -139,8 +138,7 @@ def construct_query(
             lookup_steps = [
                 {
                     "$lookup": {
-                        "from": link_info.model_class.get_motor_collection().name,
-                        # type: ignore
+                        "from": link_info.model_class.get_motor_collection().name,  # type: ignore
                         "localField": "_id",
                         "foreignField": f"{link_info.lookup_field_name}.$id",
                         "as": f"_link_{link_info.field_name}",
@@ -184,7 +182,7 @@ def construct_query(
             lookup_steps = [
                 {
                     "$lookup": {
-                        "from": link_info.model_class.get_motor_collection().name,
+                        "from": link_info.model_class.get_motor_collection().name,  # type: ignore
                         "let": {"link_id": "$_id"},
                         "as": f"_link_{link_info.field_name}",
                         "pipeline": [
@@ -241,8 +239,7 @@ def construct_query(
             queries.append(
                 {
                     "$lookup": {
-                        "from": link_info.model_class.get_motor_collection().name,
-                        # type: ignore
+                        "from": link_info.model_class.get_motor_collection().name,  # type: ignore
                         "localField": f"{link_info.lookup_field_name}.$id",
                         "foreignField": "_id",
                         "as": link_info.field_name,
@@ -261,7 +258,7 @@ def construct_query(
         else:
             lookup_step = {
                 "$lookup": {
-                    "from": link_info.model_class.get_motor_collection().name,
+                    "from": link_info.model_class.get_motor_collection().name,  # type: ignore
                     "let": {"link_id": f"${link_info.lookup_field_name}.$id"},
                     "as": link_info.field_name,
                     "pipeline": [
@@ -286,8 +283,7 @@ def construct_query(
             queries.append(
                 {
                     "$lookup": {
-                        "from": link_info.model_class.get_motor_collection().name,
-                        # type: ignore
+                        "from": link_info.model_class.get_motor_collection().name,  # type: ignore
                         "localField": "_id",
                         "foreignField": f"{link_info.lookup_field_name}.$id",
                         "as": link_info.field_name,
@@ -306,7 +302,7 @@ def construct_query(
         else:
             lookup_step = {
                 "$lookup": {
-                    "from": link_info.model_class.get_motor_collection().name,
+                    "from": link_info.model_class.get_motor_collection().name,  # type: ignore
                     "let": {"link_id": "$_id"},
                     "as": link_info.field_name,
                     "pipeline": [