[SYNPY-1653] Adding tests and re-attach session logic

.github/workflows/build.yml

@@ -84,7 +84,7 @@ jobs:
           path: |
             ${{ steps.get-dependencies.outputs.site_packages_loc }}
             ${{ steps.get-dependencies.outputs.site_bin_dir }}
-          key: ${{ runner.os }}-${{ matrix.python }}-build-${{ env.cache-name }}-${{ hashFiles('setup.py') }}-v24
+          key: ${{ runner.os }}-${{ matrix.python }}-build-${{ env.cache-name }}-${{ hashFiles('setup.py') }}-v27
 
       - name: Install py-dependencies
         if: steps.cache-dependencies.outputs.cache-hit != 'true'

docs/reference/experimental/async/curator.md

@@ -25,7 +25,16 @@ at your own risk.
             - get_async
             - store_async
             - delete_async
-            - change_metadata_async
+            - get_acl_async
+            - get_permissions_async
+            - set_permissions_async
+            - delete_permissions_async
+            - list_acl_async
+            - bind_schema_async
+            - get_schema_async
+            - unbind_schema_async
+            - validate_schema_async
+            - get_schema_derived_keys_async
 ---
 [](){ #RecordBasedMetadataTaskProperties-reference-async }
 ::: synapseclient.models.RecordBasedMetadataTaskProperties

docs/reference/experimental/sync/curator.md

@@ -25,7 +25,16 @@ at your own risk.
             - get
             - store
             - delete
-            - change_metadata
+            - get_acl
+            - get_permissions
+            - set_permissions
+            - delete_permissions
+            - list_acl
+            - bind_schema
+            - get_schema
+            - unbind_schema
+            - validate_schema
+            - get_schema_derived_keys
 ---
 [](){ #RecordBasedMetadataTaskProperties-reference }
 ::: synapseclient.models.RecordBasedMetadataTaskProperties

setup.cfg

@@ -68,7 +68,7 @@ tests_require =
     pytest~=8.2.0
     pytest-mock>=3.0,<4.0
     pytest-socket~=0.6.0
-    pytest-asyncio>=0.25.0,<1.0
+    pytest-asyncio>=1.2.0,<2.0
     flake8>=3.7.0,<4.0
     pytest-xdist[psutil]>=2.2,<3.0.0
     pytest-rerunfailures~=12.0
@@ -81,7 +81,7 @@ dev =
     pytest~=8.2.0
     pytest-mock>=3.0,<4.0
     pytest-socket~=0.6.0
-    pytest-asyncio>=0.25.0,<1.0
+    pytest-asyncio>=1.2.0,<2.0
     flake8>=3.7.0,<4.0
     pytest-xdist[psutil]>=2.2,<3.0.0
     pytest-rerunfailures~=12.0
@@ -95,7 +95,7 @@ tests =
     pytest~=8.2.0
     pytest-mock>=3.0,<4.0
     pytest-socket~=0.6.0
-    pytest-asyncio>=0.25.0,<1.0
+    pytest-asyncio>=1.2.0,<2.0
     flake8>=3.7.0,<4.0
     pytest-xdist[psutil]>=2.2,<3.0.0
     pytest-rerunfailures~=12.0

synapseclient/api/__init__.py

@@ -19,8 +19,10 @@
 from .curation_services import (
     create_curation_task,
     delete_curation_task,
+    delete_grid_session,
     get_curation_task,
     list_curation_tasks,
+    list_grid_sessions,
     update_curation_task,
 )
 from .entity_bundle_services_v2 import (
@@ -245,8 +247,10 @@
     # curation_services
     "create_curation_task",
     "delete_curation_task",
+    "delete_grid_session",
     "get_curation_task",
     "list_curation_tasks",
+    "list_grid_sessions",
     "update_curation_task",
     # user_services
     "get_user_bundle",

synapseclient/api/curation_services.py

@@ -19,8 +19,7 @@ async def create_curation_task(
     """
     Create a CurationTask associated with a project.
 
-    POST /curation/task
-
+    https://rest-docs.synapse.org/rest/POST/curation/task.html
     Arguments:
         curation_task: The complete CurationTask object to create.
         synapse_client: If not passed in and caching was not disabled by
@@ -47,6 +46,8 @@ async def get_curation_task(
     """
     Get a CurationTask by its ID.
 
+    https://rest-docs.synapse.org/rest/GET/curation/task/taskId.html
+
     Arguments:
         task_id: The unique identifier of the task.
         synapse_client: If not passed in and caching was not disabled by
@@ -72,7 +73,7 @@ async def update_curation_task(
     """
     Update a CurationTask.
 
-    PUT /curation/task/{taskId}
+    https://rest-docs.synapse.org/rest/PUT/curation/task/taskId.html
 
     Arguments:
         task_id: The unique identifier of the task.
@@ -101,6 +102,8 @@ async def delete_curation_task(
     """
     Delete a CurationTask.
 
+    https://rest-docs.synapse.org/rest/DELETE/curation/task/taskId.html
+
     Arguments:
         task_id: The unique identifier of the task.
         synapse_client: If not passed in and caching was not disabled by
@@ -125,7 +128,7 @@ async def list_curation_tasks(
     """
     Generator to get a list of CurationTasks for a project.
 
-    POST /curation/task/list
+    https://rest-docs.synapse.org/rest/POST/curation/task/list.html
 
     Arguments:
         project_id: The synId of the project.
@@ -146,3 +149,64 @@ async def list_curation_tasks(
         "/curation/task/list", body=request_body, synapse_client=client
     ):
         yield item
+
+
+async def list_grid_sessions(
+    source_id: Optional[str] = None,
+    *,
+    synapse_client: Optional["Synapse"] = None,
+) -> AsyncGenerator[Dict[str, Any], None]:
+    """
+    Generator to get a list of active grid sessions for the user.
+
+    https://rest-docs.synapse.org/rest/POST/grid/session/list.html
+
+    Arguments:
+        source_id: Optional. When provided, only sessions with this synId will be returned.
+        synapse_client: If not passed in and caching was not disabled by
+            `Synapse.allow_client_caching(False)` this will use the last created
+            instance from the Synapse class constructor.
+
+    Yields:
+        Individual GridSession objects from each page of the response.
+    """
+    from synapseclient import Synapse
+
+    client = Synapse.get_client(synapse_client=synapse_client)
+
+    request_body = {}
+    if source_id is not None:
+        request_body["sourceId"] = source_id
+
+    async for item in rest_post_paginated_async(
+        "/grid/session/list", body=request_body, synapse_client=client
+    ):
+        yield item
+
+
+async def delete_grid_session(
+    session_id: str,
+    *,
+    synapse_client: Optional["Synapse"] = None,
+) -> None:
+    """
+    Delete a grid session.
+
+    https://rest-docs.synapse.org/rest/DELETE/grid/session/sessionId.html
+
+    Note: Only the user that created a grid session may delete it.
+
+    Arguments:
+        session_id: The unique identifier of the grid session to delete.
+        synapse_client: If not passed in and caching was not disabled by
+            `Synapse.allow_client_caching(False)` this will use the last created
+            instance from the Synapse class constructor.
+
+    Returns:
+        None
+    """
+    from synapseclient import Synapse
+
+    client = Synapse.get_client(synapse_client=synapse_client)
+
+    await client.rest_delete_async(uri=f"/grid/session/{session_id}")

synapseclient/core/constants/concrete_types.py

@@ -121,3 +121,9 @@
 GRID_RECORD_SET_EXPORT_REQUEST = (
     "org.sagebionetworks.repo.model.grid.GridRecordSetExportRequest"
 )
+LIST_GRID_SESSIONS_REQUEST = (
+    "org.sagebionetworks.repo.model.grid.ListGridSessionsRequest"
+)
+LIST_GRID_SESSIONS_RESPONSE = (
+    "org.sagebionetworks.repo.model.grid.ListGridSessionsResponse"
+)

synapseclient/core/retry.py

@@ -73,6 +73,10 @@
     "SSLZeroReturnError",
 ]
 
+NON_RETRYABLE_ERRORS = [
+    "is not a table or view",
+]
+
 DEBUG_EXCEPTION = "calling %s resulted in an Exception"
 
 
@@ -242,6 +246,7 @@ def _assign_default_values(
     retry_errors: List[str] = None,
     retry_exceptions: List[Union[Exception, str]] = None,
     verbose: bool = False,
+    non_retryable_errors: List[str] = None,
 ) -> Tuple[List[int], List[int], List[str], List[Union[Exception, str]], Logger]:
     """Assigns default values to the retry parameters."""
     if not retry_status_codes:
@@ -252,6 +257,8 @@ def _assign_default_values(
         retry_errors = []
     if not retry_exceptions:
         retry_exceptions = []
+    if not non_retryable_errors:
+        non_retryable_errors = NON_RETRYABLE_ERRORS
 
     if verbose:
         logger = logging.getLogger(DEBUG_LOGGER_NAME)
@@ -263,6 +270,7 @@ def _assign_default_values(
         retry_errors,
         retry_exceptions,
         logger,
+        non_retryable_errors,
     )
 
 
@@ -280,6 +288,7 @@ async def with_retry_time_based_async(
     retry_max_back_off: float = DEFAULT_MAX_BACK_OFF_ASYNC,
     retry_max_wait_before_failure: float = DEFAULT_MAX_WAIT_BEFORE_FAIL_ASYNC,
     read_response_content: bool = True,
+    non_retryable_errors: List[str] = None,
 ) -> Union[Exception, httpx.Response, Any, None]:
     """
     Retries the given function under certain conditions. This is created such that it
@@ -306,6 +315,8 @@ async def with_retry_time_based_async(
         retry_max_back_off: The maximum wait time.
         retry_max_wait_before_failure: The maximum wait time before failure.
         read_response_content: Whether to read the response content for HTTP requests.
+        non_retryable_errors: List of strings that if found in the response or exception
+            message will prevent a retry from occurring.
 
     Example: Using with_retry
         Using ``with_retry_time_based_async`` to consolidate inputs into a list.
@@ -321,12 +332,14 @@ async def foo(a, b, c): return [a, b, c]
         retry_errors,
         retry_exceptions,
         logger,
+        non_retry_errors,
     ) = _assign_default_values(
         retry_status_codes=retry_status_codes,
         expected_status_codes=expected_status_codes,
         retry_errors=retry_errors,
         retry_exceptions=retry_exceptions,
         verbose=verbose,
+        non_retryable_errors=non_retryable_errors,
     )
 
     # Retry until we succeed or run past the maximum wait time
@@ -356,6 +369,7 @@ async def foo(a, b, c): return [a, b, c]
             retry_status_codes=retry_status_codes,
             retry_exceptions=retry_exceptions,
             retry_errors=retry_errors,
+            non_retryable_errors=non_retry_errors,
         )
 
         # Wait then retry
@@ -408,6 +422,7 @@ def with_retry_time_based(
     retry_max_back_off: float = DEFAULT_MAX_BACK_OFF_ASYNC,
     retry_max_wait_before_failure: float = DEFAULT_MAX_WAIT_BEFORE_FAIL_ASYNC,
     read_response_content: bool = True,
+    non_retryable_errors: List[str] = None,
 ) -> Union[Exception, httpx.Response, Any, None]:
     """
     Retries the given function under certain conditions. This is created such that it
@@ -434,6 +449,8 @@ def with_retry_time_based(
         retry_max_back_off: The maximum wait time.
         retry_max_wait_before_failure: The maximum wait time before failure.
         read_response_content: Whether to read the response content for HTTP requests.
+        non_retryable_errors: List of strings that if found in the response or exception
+            message will prevent a retry from occurring.
 
     Example: Using with_retry
         Using ``with_retry_time_based`` to consolidate inputs into a list.
@@ -449,12 +466,14 @@ async def foo(a, b, c): return [a, b, c]
         retry_errors,
         retry_exceptions,
         logger,
+        non_retry_errors,
     ) = _assign_default_values(
         retry_status_codes=retry_status_codes,
         expected_status_codes=expected_status_codes,
         retry_errors=retry_errors,
         retry_exceptions=retry_exceptions,
         verbose=verbose,
+        non_retryable_errors=non_retryable_errors,
     )
 
     # Retry until we succeed or run past the maximum wait time
@@ -484,6 +503,7 @@ async def foo(a, b, c): return [a, b, c]
             retry_status_codes=retry_status_codes,
             retry_exceptions=retry_exceptions,
             retry_errors=retry_errors,
+            non_retryable_errors=non_retry_errors,
         )
 
         # Wait then retry
@@ -530,6 +550,7 @@ def _is_retryable(
     retry_status_codes: List[int],
     retry_exceptions: List[Union[Exception, str]],
     retry_errors: List[str],
+    non_retryable_errors: List[str],
 ) -> bool:
     """Determines if a request should be retried based on the response and caught
     exception.
@@ -542,20 +563,31 @@ def _is_retryable(
         retry_status_codes: The status codes that should be retried.
         retry_exceptions: The exceptions that should be retried.
         retry_errors: The errors that should be retried.
+        non_retryable_errors: The errors that should not be retried.
 
     Returns:
         True if the request should be retried, False otherwise.
     """
+    response_message = None
     # Check if we got a retry-able HTTP error
     if response is not None and hasattr(response, "status_code"):
+        # First check for non-retryable error patterns even in retry status codes
+        if response.status_code in retry_status_codes:
+            response_message = response_message or _get_message(response)
+            # Check for non-retryable error patterns that should never be retried
+            if response_message and any(
+                [pattern in response_message for pattern in non_retryable_errors]
+            ):
+                return False
+
         if (
             expected_status_codes and response.status_code not in expected_status_codes
         ) or (response.status_code in retry_status_codes):
             return True
 
         elif response.status_code not in range(200, 299):
             # For all other non 200 messages look for retryable errors in the body or reason field
-            response_message = _get_message(response)
+            response_message = response_message or _get_message(response)
             if (
                 any([msg.lower() in response_message.lower() for msg in retry_errors])
                 # special case for message throttling