Metadata - Bulk requests - Merge #141

.gitignore

@@ -97,6 +97,9 @@ metadata.json
 # logs
 *.log.*
 
+# mypy
+.mypy_cache
+
 # PTVS
 *.ptvs
 *.vs

docs/conf.py

@@ -51,6 +51,7 @@
 extensions = [
     "recommonmark",
     "sphinx.ext.autodoc",
+    "sphinx_autodoc_typehints",
     "sphinx_markdown_tables",
     "sphinx_rtd_theme",
 ]

docs/requirements_docs.txt

@@ -20,6 +20,7 @@ requests_oauthlib
 
 # required
 sphinx
+sphinx-autodoc-typehints
 sphinx-markdown-tables  # to render markdown tables
 sphinx-rtd-theme
 recommonmark

isogeo_pysdk/api/__init__.py

@@ -16,6 +16,7 @@
 from .routes_invitation import ApiInvitation  # noqa: F401
 from .routes_license import ApiLicense  # noqa: F401
 from .routes_metadata import ApiMetadata  # noqa: F401
+from .routes_metadata_bulk import ApiBulk  # noqa: F401
 from .routes_search import ApiSearch  # noqa: F401
 from .routes_service import ApiService  # noqa: F401
 from .routes_service_layers import ApiServiceLayer  # noqa: F401

isogeo_pysdk/api/routes_catalog.py

@@ -14,6 +14,7 @@
 # Standard library
 import logging
 from functools import lru_cache
+from typing import Tuple, Union
 
 # 3rd party
 from requests.exceptions import Timeout
@@ -432,27 +433,43 @@ def update(self, catalog: Catalog, caching: bool = 1) -> Catalog:
 
     # -- Routes to manage the related objects ------------------------------------------
     @ApiDecorators._check_bearer_validity
-    def associate_metadata(self, metadata: Metadata, catalog: Catalog) -> Response:
+    def associate_metadata(
+        self, metadata: Union[Metadata, Tuple[Metadata, ...]], catalog: Catalog
+    ) -> Response:
         """Associate a metadata with a catalog.
 
         If the specified catalog is already associated, the response is still 204.
 
-        :param Metadata metadata: metadata object to update
+        :param metadata: metadata object to update
         :param Catalog catalog: catalog model object to associate
 
         :Example:
 
-        >>> isogeo.catalog.associate_metadata(
-            isogeo.metadata.get(METADATA_UUID),
-            isogeo.catalog.get(WORKGROUP_UUID, CATALOG_UUID)
-            ))
-        <Response [204]>
+        .. code-block:: python
+
+            isogeo.catalog.associate_metadata(
+                isogeo.metadata.get(METADATA_UUID),
+                isogeo.catalog.get(WORKGROUP_UUID, CATALOG_UUID)
+                )
+
+            <Response [204]>
         """
         # check metadata UUID
-        if not checker.check_is_uuid(metadata._id):
+        if isinstance(metadata, Metadata) and not checker.check_is_uuid(metadata._id):
             raise ValueError(
                 "Metadata ID is not a correct UUID: {}".format(metadata._id)
             )
+        elif isinstance(metadata, (list, tuple)):
+            logger.info("Multiple association detected: using bulk instead...")
+            # prepare request
+            self.api_client.metadata.bulk.prepare(
+                metadatas=tuple([md._id for md in metadata]),
+                action="add",
+                target="catalogs",
+                models=(catalog,),
+            )
+            # send it
+            return self.api_client.metadata.bulk.send()
         else:
             pass
 

isogeo_pysdk/api/routes_metadata.py

@@ -26,6 +26,7 @@
 
 # other routes
 from .routes_event import ApiEvent
+from .routes_metadata_bulk import ApiBulk
 from .routes_condition import ApiCondition
 from .routes_conformity import ApiConformity
 from .routes_feature_attributes import ApiFeatureAttribute
@@ -70,6 +71,7 @@ def __init__(self, api_client=None):
 
         # sub routes
         self.attributes = ApiFeatureAttribute(self.api_client)
+        self.bulk = ApiBulk(self.api_client)
         self.conditions = ApiCondition(self.api_client)
         self.conformity = ApiConformity(self.api_client)
         self.events = ApiEvent(self.api_client)

isogeo_pysdk/api/routes_metadata_bulk.py

@@ -0,0 +1,195 @@
+# -*- coding: UTF-8 -*-
+#! python3  # noqa E265
+
+"""
+    Isogeo API v1 - API Route for bulk update on resources (= Metadata)
+"""
+
+# #############################################################################
+# ########## Libraries #############
+# ##################################
+
+# Standard library
+import logging
+from collections import defaultdict
+
+# submodules
+from isogeo_pysdk.checker import IsogeoChecker
+from isogeo_pysdk.decorators import ApiDecorators
+from isogeo_pysdk.enums import BulkActions, BulkTargets
+from isogeo_pysdk.models import BulkReport, BulkRequest, Metadata
+from isogeo_pysdk.utils import IsogeoUtils
+
+# #############################################################################
+# ########## Globals ###############
+# ##################################
+
+logger = logging.getLogger(__name__)
+checker = IsogeoChecker()
+utils = IsogeoUtils()
+
+
+# #############################################################################
+# ########## Classes ###############
+# ##################################
+class ApiBulk:
+    """Routes as methods of Isogeo API used to mass edition of metadatas (resources).
+
+    :Example:
+
+    .. code-block:: python
+
+        # retrieve objects to be associated
+        catalog_1 = isogeo.catalog.get(
+            workgroup_id={WORKGROUP_UUID},
+            catalog_id={CATALOG_UUID_1},
+        )
+
+        catalog_2 = isogeo.catalog.get(
+            workgroup_id={WORKGROUP_UUID},
+            catalog_id={CATALOG_UUID_2},
+        )
+
+        keyword = isogeo.keyword.get(keyword_id={KEYWORD_UUID},)
+
+        # along the script, prepare the bulk requests
+        isogeo.metadata.bulk.prepare(
+            metadatas=(
+                {METADATA_UUID_1},
+                {METADATA_UUID_2},
+            ),
+            action="add",
+            target="catalogs",
+            models=(catalog_1, catalog_2),
+        )
+
+        isogeo.metadata.bulk.prepare(
+            metadatas=(
+                {METADATA_UUID_1},
+
+            ),
+            action="add",
+            target="keywords",
+            models=(keyword,),
+        )
+
+        # send the one-shot request
+        isogeo.metadata.bulk.send()
+
+    """
+
+    BULK_DATA = []
+
+    def __init__(self, api_client=None):
+        if api_client is not None:
+            self.api_client = api_client
+
+        # store API client (Request [Oauthlib] Session) and pass it to the decorators
+        self.api_client = api_client
+        ApiDecorators.api_client = api_client
+
+        # ensure platform to request
+        (
+            self.platform,
+            self.api_url,
+            self.app_url,
+            self.csw_url,
+            self.mng_url,
+            self.oc_url,
+            self.ssl,
+        ) = utils.set_base_url(self.api_client.platform)
+
+        # initialize
+        super(ApiBulk, self).__init__()
+
+    def prepare(
+        self, metadatas: tuple, action: str, target: str, models: tuple
+    ) -> BulkRequest:
+        """Prepare requests to be sent later in one shot.
+
+        :param tuple metadatas: tuple of metadatas UUIDs or Metadatas to be updated
+        :param str action: type of action to perform on metadatas. See: `isogeo_pysdk.enums.bulk_actions`.
+        :param str target: kind of object to add/delete/update to the metadatas. See: `isogeo_pysdk.enums.bulk_targets`.
+        :param tuple models: tuple of objects to be associated with the metadatas.
+        """
+        # instanciate a new Bulk REquest object
+        prepared_request = BulkRequest()
+
+        # ensure lowercase
+        prepared_request.action = action.lower()
+        prepared_request.target = target.lower()
+
+        # check metadatas uuid
+        metadatas = list(metadatas)
+        for i in metadatas:
+            if isinstance(i, str) and not checker.check_is_uuid(i):
+                logger.error("Not a correct UUID: {}".format(i))
+                metadatas.remove(i)
+            elif isinstance(i, Metadata) and not checker.check_is_uuid(i._id):
+                print("bad md")
+                logger.error(
+                    "Metadata passed but with an incorrect UUID: {}".format(i._id)
+                )
+                metadatas.remove(i)
+            elif isinstance(i, Metadata) and checker.check_is_uuid(i._id):
+                logger.debug("Metadata passed, extracting the UUID: {}".format(i._id))
+                metadatas.append(i._id)
+                metadatas.remove(i)
+            else:
+                continue
+
+        # add it to the prepared request query
+        prepared_request.query = {"ids": metadatas}
+
+        # check passed objects
+        obj_type = models[0]
+        if not all([isinstance(obj, type(obj_type)) for obj in models]):
+            raise TypeError(
+                "Models must contain an unique type of objects. First found: {}".format(
+                    obj_type
+                )
+            )
+
+        prepared_request.model = [obj.to_dict() for obj in models]
+
+        # add it to be sent later
+        self.BULK_DATA.append(prepared_request.to_dict())
+
+        return prepared_request
+
+    @ApiDecorators._check_bearer_validity
+    def send(self) -> list:
+        """Send prepared BULK_DATA to the `POST BULK resources/`."""
+
+        # build request url
+        url_metadata_bulk = utils.get_request_base_url(route="resources")
+
+        # request
+        req_metadata_bulk = self.api_client.post(
+            url=url_metadata_bulk,
+            json=self.BULK_DATA,
+            headers=self.api_client.header,
+            proxies=self.api_client.proxies,
+            stream=True,
+            verify=self.api_client.ssl,
+            timeout=self.api_client.timeout,
+        )
+
+        # checking response
+        req_check = checker.check_api_response(req_metadata_bulk)
+        if isinstance(req_check, tuple):
+            return req_check
+
+        # empty bulk data
+        self.BULK_DATA.clear()
+
+        return [BulkReport(**req) for req in req_metadata_bulk.json()]
+
+
+# ##############################################################################
+# ##### Stand alone program ########
+# ##################################
+if __name__ == "__main__":
+    """standalone execution."""
+    sample_test = ApiBulk()
+    print(sample_test)

isogeo_pysdk/checker.py

@@ -406,7 +406,7 @@ def _check_filter_includes(self, includes: tuple, entity: str = "metadata") -> l
                                 subresource, " | ".join(ref_subresources)
                             )
                         )
-                        includes.remove(subresource)  # removing bad subresource
+                        list(includes).remove(subresource)  # removing bad subresource
                     else:
                         pass
 

isogeo_pysdk/enums/__init__.py

@@ -2,6 +2,9 @@
 #! python3  # noqa: E265
 
 from .application_types import ApplicationTypes  # noqa: F401
+from .bulk_actions import BulkActions  # noqa: F401
+from .bulk_ignore_reasons import BulkIgnoreReasons  # noqa: F401
+from .bulk_targets import BulkTargets  # noqa: F401
 from .catalog_statistics_tags import CatalogStatisticsTags  # noqa: F401
 from .contact_roles import ContactRoles  # noqa: F401
 from .contact_types import ContactTypes  # noqa: F401

isogeo_pysdk/enums/bulk_actions.py

@@ -0,0 +1,64 @@
+# -*- coding: UTF-8 -*-
+#! python3  # noqa E265
+
+"""
+    Isogeo API v1 - Enums for Resource entity accepted kinds
+
+    See: http://help.isogeo.com/api/complete/index.html#definition-application
+"""
+
+# #############################################################################
+# ########## Libraries #############
+# ##################################
+
+# standard library
+from enum import Enum, auto
+
+
+# #############################################################################
+# ########## Classes ###############
+# ##################################
+class BulkActions(Enum):
+    """Closed list of accepted Application (metadata subresource) kinds in Isogeo API.
+
+    :Example:
+
+        >>> # parse members and values
+        >>> print("{0:<30} {1:>20}".format("Enum", "Value"))
+        >>> for md_kind in BulkActions:
+        >>>     print("{0:<30} {1:>20}".format(md_kind, md_kind.value))
+        Enum                                          Value
+        BulkActions.add                                   1
+        BulkActions.delete                                2
+        BulkActions.update                                3
+
+        >>> # check if a var is an accepted value
+        >>> print("add" in BulkActions.__members__)
+        True
+        >>> print("Delete" in BulkActions.__members__)  # case sensitive
+        False
+        >>> print("truncate" in BulkActions.__members__)
+        False
+
+    See: https://docs.python.org/3/library/enum.html
+    """
+
+    add = auto()
+    delete = auto()
+    update = auto()
+
+
+# ##############################################################################
+# ##### Stand alone program ########
+# ##################################
+if __name__ == "__main__":
+    """standalone execution."""
+    print("{0:<30} {1:>20}".format("Enum", "Value"))
+    for md_kind in BulkActions:
+        print("{0:<30} {1:>20}".format(md_kind, md_kind.value))
+
+    print(len(BulkActions))
+
+    print("add" in BulkActions.__members__)
+    print("Delete" in BulkActions.__members__)
+    print("truncate" in BulkActions.__members__)