[SYNPY-1416] File model finishing touches for OOP (#1060)

* File model finishing touches for OOP

.github/workflows/build.yml

@@ -180,7 +180,8 @@ jobs:
           export EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID="${{secrets.EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID}}"
           export EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY="${{secrets.EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY}}"
           if [ ${{ steps.otel-check.outputs.run_opentelemetry }} == "true" ]; then
-            export SYNAPSE_OTEL_INTEGRATION_TEST_EXPORTER="otlp"
+            # Set to 'otlp' to enable OpenTelemetry
+            export SYNAPSE_OTEL_INTEGRATION_TEST_EXPORTER="none"
           fi
 
           # use loadscope to avoid issues running tests concurrently that share scoped fixtures

docs/reference/oop/models.md

@@ -26,7 +26,15 @@ the client.
         members:
         - get
         - store
+        - copy
         - delete
+        - from_id
+        - from_path
+        - change_metadata
+::: synapseclient.models.file.FileHandle
+    options:
+      filters:
+      - "!"
 ---
 ::: synapseclient.models.Table
     options:

docs/scripts/object_orientated_programming_poc/oop_poc_file.py

@@ -1,19 +1,23 @@
-"""The purpose of this script is to demonstrate how to use the new OOP interface for files.
+"""
+Expects that ~/temp exists and is a directory.
+
+The purpose of this script is to demonstrate how to use the new OOP interface for files.
 The following actions are shown in this script:
 1. Creating a file
-2. Storing a file to a project
-3. Storing a file to a folder
-4. Getting metadata about a file
+2. Storing a file
+3. Storing a file in a sub-folder
+4. Renaming a file
 5. Downloading a file
 6. Deleting a file
+7. Copying a file
+8. Storing an activity to a file
+9. Retrieve an activity from a file
 """
 import asyncio
 import os
 
-from synapseclient.models import (
-    File,
-    Folder,
-)
+from synapseclient.models import File, Folder, Activity, UsedEntity, UsedURL
+from synapseclient.core import utils
 from datetime import date, datetime, timedelta, timezone
 import synapseclient
 
@@ -35,6 +39,15 @@ def create_random_file(
 
 
 async def store_file():
+    # Cleanup synapse for previous runs - Does not delete local files/directories:
+    script_file_folder = Folder(name="file_script_folder", parent_id=PROJECT_ID)
+    if not os.path.exists(os.path.expanduser("~/temp/myNewFolder")):
+        os.mkdir(os.path.expanduser("~/temp/myNewFolder"))
+    # Hack to get the ID as Folder does not support get by name/id yet
+    await script_file_folder.store()
+    await script_file_folder.delete()
+    await script_file_folder.store()
+
     # Creating annotations for my file ==================================================
     annotations_for_my_file = {
         "my_single_key_string": "a",
@@ -52,91 +65,150 @@ async def store_file():
         ],
     }
 
-    name_of_file = "my_file_with_random_data.txt"
+    name_of_file = "file_script_my_file_with_random_data.txt"
     path_to_file = os.path.join(os.path.expanduser("~/temp"), name_of_file)
     create_random_file(path_to_file)
 
-    # Creating and uploading a file to a project =========================================
+    # 1. Creating a file =================================================================
     file = File(
         path=path_to_file,
-        name=name_of_file,
         annotations=annotations_for_my_file,
-        parent_id=PROJECT_ID,
+        parent_id=script_file_folder.id,
         description="This is a file with random data.",
     )
 
+    # 2. Storing a file ==================================================================
     file = await file.store()
 
-    print("File created:")
-    print(file)
-
-    # Updating and storing an annotation =================================================
-    file_copy = await File(id=file.id).get()
-    file_copy.annotations["my_key_string"] = ["new", "values", "here"]
-    stored_file = await file_copy.store()
-    print("File updated:")
-    print(stored_file)
-
-    # Downloading a file =================================================================
-    downloaded_file_copy = await File(id=file.id).get(
-        download_location=os.path.expanduser("~/temp/myNewFolder")
-    )
-
-    print("Downloaded file:")
-    print(downloaded_file_copy)
-
-    # Get metadata about a file ==========================================================
-    non_downloaded_file_copy = await File(id=file.id).get(
-        download_file=False,
-    )
-
-    print("Metadata about file:")
-    print(non_downloaded_file_copy)
+    print(f"File created: ID: {file.id}, Parent ID: {file.parent_id}")
 
-    # Creating and uploading a file to a folder =========================================
-    folder = await Folder(name="my_folder", parent_id=PROJECT_ID).store()
+    name_of_file = "file_in_a_sub_folder.txt"
+    path_to_file = os.path.join(os.path.expanduser("~/temp"), name_of_file)
+    create_random_file(path_to_file)
 
-    file = File(
+    # 3. Storing a file to a sub-folder ==================================================
+    script_sub_folder = await Folder(
+        name="file_script_sub_folder", parent_id=script_file_folder.id
+    ).store()
+    file_in_a_sub_folder = File(
         path=path_to_file,
-        name=name_of_file,
         annotations=annotations_for_my_file,
-        parent_id=folder.id,
+        parent_id=script_sub_folder.id,
         description="This is a file with random data.",
     )
+    file_in_a_sub_folder = await file_in_a_sub_folder.store()
 
-    file = await file.store()
-
-    print("File created:")
-    print(file)
-
-    downloaded_file_copy = await File(id=file.id).get(
-        download_location=os.path.expanduser("~/temp/myNewFolder")
+    print(
+        f"File created in sub folder: ID: {file_in_a_sub_folder.id}, Parent ID: {file_in_a_sub_folder.parent_id}"
     )
 
-    print("Downloaded file:")
-    print(downloaded_file_copy)
+    # 4. Renaming a file =================================================================
+    name_of_file = "file_script_my_file_to_rename.txt"
+    path_to_file = os.path.join(os.path.expanduser("~/temp"), name_of_file)
+    create_random_file(path_to_file)
 
-    non_downloaded_file_copy = await File(id=file.id).get(
-        download_file=False,
+    # The name of the entity, and the name of the file is disjointed.
+    # For example, the name of the file is "file_script_my_file_to_rename.txt"
+    # and the name of the entity is "this_name_is_different"
+    file: File = await File(
+        path=path_to_file,
+        name="this_name_is_different",
+        parent_id=script_file_folder.id,
+    ).store()
+    print(f"File created with name: {file.name}")
+    print(f"The path of the file is: {file.path}")
+
+    # You can change the name of the entity without changing the name of the file.
+    file.name = "modified_name_attribute"
+    await file.store()
+    print(f"File renamed to: {file.name}")
+
+    # You can then change the name of the file that would be downloaded like:
+    await file.change_metadata(download_as="new_name_for_downloading.txt")
+    print(f"File download values changed to: {file.file_handle.file_name}")
+
+    # 5. Downloading a file ===============================================================
+    # Downloading a file to a location has a default beahvior of "keep.both"
+    downloaded_file = await File(
+        id=file.id, download_location=os.path.expanduser("~/temp/myNewFolder")
+    ).get()
+    print(f"Downloaded file: {downloaded_file.path}")
+
+    # I can also specify how collisions are handled when downloading a file.
+    # This will replace the file on disk if it already exists and is different (after).
+    path_to_file = downloaded_file.path
+    create_random_file(path_to_file)
+    print(f"Before file md5: {utils.md5_for_file(path_to_file).hexdigest()}")
+    downloaded_file = await File(
+        id=downloaded_file.id,
+        download_location=os.path.expanduser("~/temp/myNewFolder"),
+        if_collision="overwrite.local",
+    ).get()
+    print(f"After file md5: {utils.md5_for_file(path_to_file).hexdigest()}")
+
+    # This will keep the file on disk (before), and no file is downloaded
+    path_to_file = downloaded_file.path
+    create_random_file(path_to_file)
+    print(f"Before file md5: {utils.md5_for_file(path_to_file).hexdigest()}")
+    downloaded_file = await File(
+        id=downloaded_file.id,
+        download_location=os.path.expanduser("~/temp/myNewFolder"),
+        if_collision="keep.local",
+    ).get()
+    print(f"After file md5: {utils.md5_for_file(path_to_file).hexdigest()}")
+
+    # 6. Deleting a file =================================================================
+    # Suppose I have a file that I want to delete.
+    name_of_file = "file_to_delete.txt"
+    path_to_file = os.path.join(os.path.expanduser("~/temp"), name_of_file)
+    create_random_file(path_to_file)
+    file_to_delete = await File(
+        path=path_to_file, parent_id=script_file_folder.id
+    ).store()
+    await file_to_delete.delete()
+
+    # 7. Copying a file ===================================================================
+    print(
+        f"File I am going to copy: ID: {file_in_a_sub_folder.id}, Parent ID: {file_in_a_sub_folder.parent_id}"
+    )
+    new_sub_folder = await Folder(
+        name="sub_sub_folder", parent_id=script_file_folder.id
+    ).store()
+    copied_file_instance = await file_in_a_sub_folder.copy(
+        destination_id=new_sub_folder.id
+    )
+    print(
+        f"File I copied: ID: {copied_file_instance.id}, Parent ID: {copied_file_instance.parent_id}"
     )
 
-    print("Metadata about file:")
-    print(non_downloaded_file_copy)
+    # 8. Storing an activity to a file =====================================================
+    activity = Activity(
+        name="some_name",
+        description="some_description",
+        used=[
+            UsedURL(name="example", url="https://www.synapse.org/"),
+            UsedEntity(target_id="syn456", target_version_number=1),
+        ],
+        executed=[
+            UsedURL(name="example", url="https://www.synapse.org/"),
+            UsedEntity(target_id="syn789", target_version_number=1),
+        ],
+    )
 
-    # Uploading a file and then Deleting a file ==========================================
-    name_of_file = "my_file_with_random_data_to_delete.txt"
+    name_of_file = "file_with_an_activity.txt"
     path_to_file = os.path.join(os.path.expanduser("~/temp"), name_of_file)
     create_random_file(path_to_file)
-
-    file = await File(
-        path=path_to_file,
-        name=name_of_file,
-        annotations=annotations_for_my_file,
-        parent_id=PROJECT_ID,
-        description="This is a file with random data I am going to delete.",
+    file_with_activity = await File(
+        path=path_to_file, parent_id=script_file_folder.id, activity=activity
     ).store()
-
-    await file.delete()
+    print(file_with_activity.activity)
+
+    # 9. When I am retrieving that file later on I can get the activity like =============
+    # By also specifying download_file=False, I can get the activity without downloading the file.
+    new_file_with_activity_instance = await File(
+        id=file_with_activity.id, download_file=False
+    ).get(include_activity=True)
+    print(new_file_with_activity_instance.activity)
 
 
 asyncio.run(store_file())

docs/scripts/object_orientated_programming_poc/oop_poc_folder.py

@@ -1,4 +1,7 @@
-"""The purpose of this script is to demonstrate how to use the new OOP interface for folders.
+"""
+Expects that ~/temp exists and is a directory.
+
+The purpose of this script is to demonstrate how to use the new OOP interface for folders.
 The following actions are shown in this script:
 1. Creating a folder
 2. Storing a folder to a project
@@ -114,6 +117,8 @@ async def store_folder():
     }
 
     for file in folder_copy.files:
+        file.download_file = False
+        await file.get()
         file.annotations = new_annotations
 
     for folder in folder_copy.folders:

docs/scripts/object_orientated_programming_poc/oop_poc_project.py

@@ -1,4 +1,7 @@
-"""The purpose of this script is to demonstrate how to use the new OOP interface for projects.
+"""
+Expects that ~/temp exists and is a directory.
+
+The purpose of this script is to demonstrate how to use the new OOP interface for projects.
 The following actions are shown in this script:
 1. Creating a project
 2. Storing a folder to a project
@@ -116,6 +119,8 @@ async def store_project():
     }
 
     for file in project_copy.files:
+        file.download_file = False
+        await file.get()
         file.annotations = new_annotations
 
     for folder in project_copy.folders:

synapseclient/client.py

@@ -82,6 +82,7 @@
     SynapseProvenanceError,
     SynapseTimeoutError,
     SynapseUnmetAccessRestrictions,
+    SynapseMalformedEntityError,
 )
 from synapseclient.core.logging_setup import (
     DEFAULT_LOGGER_NAME,
@@ -1244,10 +1245,6 @@ def _getWithEntityBundle(
         followLink = kwargs.pop("followLink", False)
         path = kwargs.pop("path", None)
 
-        # make sure user didn't accidentlaly pass a kwarg that we don't handle
-        if kwargs:  # if there are remaining items in the kwargs
-            raise TypeError("Unexpected **kwargs: %r" % kwargs)
-
         # If Link, get target ID entity bundle
         if (
             entityBundle["entity"]["concreteType"]
@@ -1493,6 +1490,7 @@ def store(
         executed=None,
         activityName=None,
         activityDescription=None,
+        set_annotations=True,
     ):
         """
         Creates a new Entity or updates an existing Entity, uploading any files in the process.
@@ -1514,6 +1512,7 @@ def store(
                             process of adding terms-of-use or review board approval for this entity.
                             You will be contacted with regards to the specific data being restricted and the
                             requirements of access.
+            set_annotations: If True, set the annotations on the entity. If False, do not set the annotations.
 
         Returns:
             A Synapse Entity, Evaluation, or Wiki
@@ -1637,9 +1636,23 @@ def store(
             if needs_upload:
                 local_state_fh = local_state.get("_file_handle", {})
                 synapseStore = local_state.get("synapseStore", True)
+
+                # parent_id_for_upload is allowing `store` to be called on files that have
+                # already been stored to Synapse, but did not specify a parentId in the
+                # FileEntity. This is useful as it prevents the need to specify the
+                # parentId every time a file is stored to Synapse when the ID is
+                # already known.
+                parent_id_for_upload = entity.get("parentId", None)
+                if not parent_id_for_upload and bundle and bundle.get("entity", None):
+                    parent_id_for_upload = bundle["entity"]["parentId"]
+
+                if not parent_id_for_upload:
+                    raise SynapseMalformedEntityError(
+                        "Entities of type File must have a parentId."
+                    )
                 fileHandle = upload_file_handle(
                     self,
-                    entity["parentId"],
+                    parent_id_for_upload,
                     local_state["path"]
                     if (synapseStore or local_state_fh.get("externalURL") is None)
                     else local_state_fh.get("externalURL"),
@@ -1751,8 +1764,11 @@ def store(
             self._createAccessRequirementIfNone(properties)
 
         # Update annotations
-        if (not bundle and annotations) or (
-            bundle and check_annotations_changed(bundle["annotations"], annotations)
+        if set_annotations and (
+            (not bundle and annotations)
+            or (
+                bundle and check_annotations_changed(bundle["annotations"], annotations)
+            )
         ):
             annotations = self.set_annotations(
                 Annotations(properties["id"], properties["etag"], annotations)

synapseclient/core/async_utils.py

@@ -1,4 +1,5 @@
 """This utility class is to hold any utilities that are needed for async operations."""
+
 from typing import Callable, Union
 from opentelemetry import trace