Merge pull request #880 from Sage-Bionetworks/develop-blank-annotations

Update annotations unit test

pyproject.toml

@@ -55,6 +55,7 @@ PyYAML = "^5.4.1"
 rdflib = "^5.0.0"
 setuptools = "^52.0.0"
 synapseclient = "2.5.1"
+tenacity = "^8.0.1"
 toml = "^0.10.2"
 Flask = "^1.1.4"
 connexion = {extras = ["swagger-ui"], version = "^2.8.0"}

tests/conftest.py

@@ -66,31 +66,11 @@ def get_schema_explorer(path=None, *paths):
         return se
 
     @staticmethod
-    def get_version_specific_manifest_path(self, path):
+    def get_python_version(self):
         version=platform.python_version()
-        manifest_path = self.get_data_path(path)
-        temp_manifest_path = manifest_path.replace('.csv',version[0:3]+'.csv')
-        return temp_manifest_path
-
-    @staticmethod
-    def get_version_specific_syn_dataset():
-        version=platform.python_version()
-
-        synId = None
-
-        if version.startswith('3.7'):
-            synId = 'syn34999062'
-        elif version.startswith('3.8'):
-            synId = 'syn34999080'
-        elif version.startswith('3.9'):
-            synId = 'syn34999096'
-
-        if not synId:
-            raise OSError(
-                "Unsupported Version of Python"
-            )
-        else:
-            return synId
+        base_version=".".join(version.split('.')[0:2])
+
+        return base_version
 
 @pytest.fixture
 def helpers():

tests/data/mock_manifests/annotations_test_manifest.csv

@@ -1,3 +1,3 @@
-Component,CheckList,CheckRegexList,CheckRegexSingle,CheckNum,CheckFloat,CheckInt,CheckString,CheckURL,CheckMatchatLeast,CheckMatchatLeastvalues,CheckMatchExactly,CheckMatchExactlyvalues,CheckRecommended,CheckAges,CheckUnique
-MockComponent,"valid,list,values","a,c,f",a,6,99.65,7,valid,https://www.google.com/,1985,4891,23487492,24323472834,,6571,str1
-MockComponent,"valid,list,values","a,c,f",a,6,99.65,8.52,valid,https://www.google.com/,1985,4891,23487492,24323472834,,6571,str1
+Component,CheckList,CheckRegexList,CheckRegexSingle,CheckNum,CheckFloat,CheckInt,CheckString,CheckURL,CheckMatchatLeast,CheckMatchatLeastvalues,CheckMatchExactly,CheckMatchExactlyvalues,CheckRecommended,CheckAges,CheckUnique,Uuid,entityId
+MockComponent,"valid,list,values","a,c,f",a,6,99.65,7,valid,https://www.google.com/,1985,4891,23487492,24323472834,,6571,str1,0f7812cc-8a0e-4f54-b8c4-e497cb7b34d0,syn35367245
+MockComponent,"valid,list,values","a,c,f",a,6,99.65,8.52,valid,https://www.google.com/,1985,4891,23487492,24323472834,,6571,str1,da82f8e2-c7b0-428f-8f9d-677252ef5f68,syn35367246

tests/test_store.py

@@ -4,6 +4,7 @@
 import logging
 import pytest
 import time
+from tenacity import Retrying, RetryError, stop_after_attempt, wait_random_exponential
 
 import pandas as pd
 from synapseclient import EntityViewSchema
@@ -47,6 +48,9 @@ def dataset_fileview_table_tidy(dataset_fileview, dataset_fileview_table):
     yield table
 
 
+def raise_final_error(retry_state):
+    return retry_state.outcome.result()
+
 class TestBaseStorage:
     def test_init(self):
 
@@ -81,40 +85,33 @@ def test_getFileAnnotations(self, synapse_store):
         assert expected_dict == actual_dict
 
     def test_annotation_submission(self, synapse_store, helpers, config):
-        # Duplicate base file to avoid conflicts
         manifest_path = "mock_manifests/annotations_test_manifest.csv"
-        temp_manifest_path = helpers.get_version_specific_manifest_path(helpers, manifest_path)
 
         # Upload dataset annotations
         inputModelLocaiton = helpers.get_data_path(get_from_config(config.DATA, ("model", "input", "location")))
         sg = SchemaGenerator(inputModelLocaiton)
 
         try:        
-            manifest_id = synapse_store.associateMetadataWithFiles(
-                schemaGenerator=sg,
-                metadataManifestPath=temp_manifest_path,
-                datasetId=helpers.get_version_specific_syn_dataset(),
-                manifest_record_type = 'entity',
-                useSchemaLabel = True,
-                hideBlanks = True,
-                restrict_manifest = False,
-            )
-        except(SynapseHTTPError):
-            time.sleep(30)
-
-            manifest_id = synapse_store.associateMetadataWithFiles(
-                schemaGenerator=sg,
-                metadataManifestPath=temp_manifest_path,
-                datasetId=helpers.get_version_specific_syn_dataset(),
-                manifest_record_type = 'entity',
-                useSchemaLabel = True,
-                hideBlanks = True,
-                restrict_manifest = False,
-            )
-
+            for attempt in Retrying(
+                stop = stop_after_attempt(15),
+                wait = wait_random_exponential(multiplier=1,min=10,max=120),
+                retry_error_callback = raise_final_error
+                ):
+                with attempt:         
+                    manifest_id = synapse_store.associateMetadataWithFiles(
+                        schemaGenerator = sg,
+                        metadataManifestPath = helpers.get_data_path(manifest_path),
+                        datasetId = 'syn34295552',
+                        manifest_record_type = 'entity',
+                        useSchemaLabel = True,
+                        hideBlanks = True,
+                        restrict_manifest = False,
+                    )
+        except RetryError:
+            pass
 
         # Retrive annotations
-        entity_id, entity_id_spare = helpers.get_data_frame(temp_manifest_path)["entityId"][0:2]
+        entity_id, entity_id_spare = helpers.get_data_frame(manifest_path)["entityId"][0:2]
         annotations = synapse_store.getFileAnnotations(entity_id)
 
         # Check annotations of interest