From 9b6d93a06106a36237116e5519e745bd432cb9b7 Mon Sep 17 00:00:00 2001 From: Ryan Wehrle Date: Wed, 19 Apr 2023 15:33:27 -0400 Subject: [PATCH 1/8] Remove file storage from project schema; Refactor project api file storage logic --- ansys/rep/client/jms/api/project_api.py | 22 ++++++++++++---------- ansys/rep/client/jms/resource/project.py | 4 ---- ansys/rep/client/jms/schema/project.py | 3 --- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/ansys/rep/client/jms/api/project_api.py b/ansys/rep/client/jms/api/project_api.py index 298bfa7ae..9c98e14e8 100644 --- a/ansys/rep/client/jms/api/project_api.py +++ b/ansys/rep/client/jms/api/project_api.py @@ -92,10 +92,8 @@ def url(self) -> str: @property def fs_url(self) -> str: """URL of the file storage gateway""" - if self._fs_url is None or self._fs_project_id != self.project_id: - self._fs_project_id = self.project_id - project = get_project(self.client, self.jms_api_url, id=self.project_id) - self._fs_url = get_fs_url(project) + if self._fs_url is None: + self._fs_url = get_fs_url(self) return self._fs_url @property @@ -600,15 +598,19 @@ def sync_jobs(project_api: ProjectApi, jobs: List[Job]): r = project_api.client.session.put(f"{url}", data=json_data) -@cached(cache=TTLCache(1024, 60), key=lambda project: project.id) -def get_fs_url(project: Project): - if project.file_storages == missing: - raise REPError(f"The project object has no file storages information.") - rest_gateways = [fs for fs in project.file_storages if fs["obj_type"] == "RestGateway"] +def get_fs_url(project_api: ProjectApi): + + response = project_api.client.session.get(f"{project_api.jms_api_url}/storage", verify=False) + file_storages = response.json()["backends"] + + if not file_storages: + raise REPError(f"No file storage information.") + + rest_gateways = [fs for fs in file_storages if fs["obj_type"] == "RestGateway"] rest_gateways.sort(key=lambda fs: fs["priority"]) if not rest_gateways: - raise REPError(f"Project {project.name} (id={project.id}) has no Rest Gateway defined.") + raise REPError(f"No Rest Gateway defined.") for d in rest_gateways: url = d["url"] diff --git a/ansys/rep/client/jms/resource/project.py b/ansys/rep/client/jms/resource/project.py index a216cbb46..b6c704391 100644 --- a/ansys/rep/client/jms/resource/project.py +++ b/ansys/rep/client/jms/resource/project.py @@ -20,8 +20,6 @@ class Project(Object): The date and time the project was created. modification_time : datetime, optional The date and time the project was last modified. - file_storages : list - List of file storages defined for the project. statistics : dict Optional dictionary containing various project statistics. @@ -38,7 +36,6 @@ def __init__(self, priority=missing, creation_time=missing, modification_time=missing, - file_storages=missing, statistics=missing ): self.id = id @@ -47,7 +44,6 @@ def __init__(self, self.priority = priority self.creation_time = creation_time self.modification_time = modification_time - self.file_storages = file_storages self.statistics = statistics self.obj_type = self.__class__.__name__ diff --git a/ansys/rep/client/jms/schema/project.py b/ansys/rep/client/jms/schema/project.py index c6ae0400e..2e781aee9 100644 --- a/ansys/rep/client/jms/schema/project.py +++ b/ansys/rep/client/jms/schema/project.py @@ -31,9 +31,6 @@ class Meta(BaseSchema.Meta): description="The date and time the project was last modified.", ) - file_storages = fields.List( - fields.Dict(), description="List of file storages defined for the project." - ) statistics = fields.Dict( load_only=True, description="Optional dictionary containing various project statistics." ) From c801c8f542ae8a22c7d7a1d8f5d4fb9813611325 Mon Sep 17 00:00:00 2001 From: Ryan Wehrle Date: Wed, 19 Apr 2023 16:16:07 -0400 Subject: [PATCH 2/8] Precommit modified files --- ansys/rep/client/jms/api/project_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansys/rep/client/jms/api/project_api.py b/ansys/rep/client/jms/api/project_api.py index 9c98e14e8..ef024d580 100644 --- a/ansys/rep/client/jms/api/project_api.py +++ b/ansys/rep/client/jms/api/project_api.py @@ -599,7 +599,7 @@ def sync_jobs(project_api: ProjectApi, jobs: List[Job]): def get_fs_url(project_api: ProjectApi): - + response = project_api.client.session.get(f"{project_api.jms_api_url}/storage", verify=False) file_storages = response.json()["backends"] From 5f58c5fd94a92710fb64eab86d5034597137664c Mon Sep 17 00:00:00 2001 From: Ryan Wehrle Date: Fri, 21 Apr 2023 08:59:01 -0400 Subject: [PATCH 3/8] Move get_fs_url to jms api; Expose fs_url on jms api; Create method to retrieve storages --- ansys/rep/client/jms/api/jms_api.py | 46 +++++++++++++++++++++++++ ansys/rep/client/jms/api/project_api.py | 28 +-------------- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/ansys/rep/client/jms/api/jms_api.py b/ansys/rep/client/jms/api/jms_api.py index 7ebb95b31..0952be391 100644 --- a/ansys/rep/client/jms/api/jms_api.py +++ b/ansys/rep/client/jms/api/jms_api.py @@ -4,6 +4,7 @@ import time from typing import List, Union import uuid +import requests from ansys.rep.client.client import Client from ansys.rep.client.exceptions import REPError @@ -46,12 +47,20 @@ class JmsApi(object): def __init__(self, client: Client): self.client = client + self._fs_url = None @property def url(self) -> str: """Returns the API url""" return f"{self.client.rep_url}/jms/api/v1" + @property + def fs_url(self) -> str: + """URL of the file storage gateway""" + if self._fs_url is None: + self._fs_url = get_fs_url(self.client, self.url) + return self._fs_url + def get_api_info(self): """Return info like version, build date etc of the JMS API the client is connected to""" r = self.client.session.get(self.url) @@ -233,6 +242,11 @@ def get_operation(self, id, as_object=True) -> Operation: def _monitor_operation(self, operation_id: str, interval: float = 1.0): return _monitor_operation(self, operation_id, interval) + ################################################################ + # Storages + def get_storage(self): + """Return a list of storages""" + return get_storages(self.client, self.url) def get_projects(client, api_url, as_objects=True, **query_params) -> List[Project]: @@ -382,3 +396,35 @@ def restore_project(jms_api, archive_path): r = jms_api.client.session.put(f"{jms_api.client.rep_url}/fs/api/v1/remove/{bucket}") return get_project(jms_api.client, jms_api.url, project_id) + +def get_storages(client, api_url): + """ + Returns list of storages + """ + url = f"{api_url}/storage" + r = client.session.get(url) + return r.json()["backends"] + +def get_fs_url(client, api_url): + + file_storages = get_storages(client, api_url) + + if not file_storages: + raise REPError(f"No file storage information.") + + rest_gateways = [fs for fs in file_storages if fs["obj_type"] == "RestGateway"] + rest_gateways.sort(key=lambda fs: fs["priority"]) + + if not rest_gateways: + raise REPError(f"No Rest Gateway defined.") + + for d in rest_gateways: + url = d["url"] + try: + r = requests.get(url, verify=False, timeout=2) + except Exception as ex: + log.debug(ex) + continue + if r.status_code == 200: + return url + return None \ No newline at end of file diff --git a/ansys/rep/client/jms/api/project_api.py b/ansys/rep/client/jms/api/project_api.py index ef024d580..f3e2cb254 100644 --- a/ansys/rep/client/jms/api/project_api.py +++ b/ansys/rep/client/jms/api/project_api.py @@ -93,7 +93,7 @@ def url(self) -> str: def fs_url(self) -> str: """URL of the file storage gateway""" if self._fs_url is None: - self._fs_url = get_fs_url(self) + self._fs_url = JmsApi(self.client).fs_url return self._fs_url @property @@ -598,32 +598,6 @@ def sync_jobs(project_api: ProjectApi, jobs: List[Job]): r = project_api.client.session.put(f"{url}", data=json_data) -def get_fs_url(project_api: ProjectApi): - - response = project_api.client.session.get(f"{project_api.jms_api_url}/storage", verify=False) - file_storages = response.json()["backends"] - - if not file_storages: - raise REPError(f"No file storage information.") - - rest_gateways = [fs for fs in file_storages if fs["obj_type"] == "RestGateway"] - rest_gateways.sort(key=lambda fs: fs["priority"]) - - if not rest_gateways: - raise REPError(f"No Rest Gateway defined.") - - for d in rest_gateways: - url = d["url"] - try: - r = requests.get(url, verify=False, timeout=2) - except Exception as ex: - log.debug(ex) - continue - if r.status_code == 200: - return url - return None - - def _fs_copy_file( session: requests.Session, fs_url: str, From 974dca8905d238ccc189c750e6316fcc28b2efb8 Mon Sep 17 00:00:00 2001 From: Ryan Wehrle Date: Fri, 21 Apr 2023 09:02:48 -0400 Subject: [PATCH 4/8] Precommit modified files --- ansys/rep/client/jms/api/jms_api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ansys/rep/client/jms/api/jms_api.py b/ansys/rep/client/jms/api/jms_api.py index 0952be391..7bf2f1437 100644 --- a/ansys/rep/client/jms/api/jms_api.py +++ b/ansys/rep/client/jms/api/jms_api.py @@ -4,6 +4,7 @@ import time from typing import List, Union import uuid + import requests from ansys.rep.client.client import Client @@ -242,6 +243,7 @@ def get_operation(self, id, as_object=True) -> Operation: def _monitor_operation(self, operation_id: str, interval: float = 1.0): return _monitor_operation(self, operation_id, interval) + ################################################################ # Storages def get_storage(self): @@ -397,6 +399,7 @@ def restore_project(jms_api, archive_path): return get_project(jms_api.client, jms_api.url, project_id) + def get_storages(client, api_url): """ Returns list of storages @@ -405,6 +408,7 @@ def get_storages(client, api_url): r = client.session.get(url) return r.json()["backends"] + def get_fs_url(client, api_url): file_storages = get_storages(client, api_url) @@ -427,4 +431,4 @@ def get_fs_url(client, api_url): continue if r.status_code == 200: return url - return None \ No newline at end of file + return None From 39cd34e417ea8ea300f074d750986a56a9385cce Mon Sep 17 00:00:00 2001 From: Ryan Wehrle Date: Fri, 21 Apr 2023 09:32:13 -0400 Subject: [PATCH 5/8] Remove unused imports --- ansys/rep/client/jms/api/project_api.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ansys/rep/client/jms/api/project_api.py b/ansys/rep/client/jms/api/project_api.py index f3e2cb254..09c35ab38 100644 --- a/ansys/rep/client/jms/api/project_api.py +++ b/ansys/rep/client/jms/api/project_api.py @@ -4,8 +4,6 @@ from pathlib import Path from typing import Callable, List, Type -from cachetools import TTLCache, cached -from marshmallow.utils import missing import requests from ansys.rep.client.client import Client @@ -21,14 +19,13 @@ ParameterDefinition, ParameterMapping, Permission, - Project, Task, TaskDefinition, ) from ansys.rep.client.jms.schema.job import JobSchema from .base import create_objects, delete_objects, get_objects, update_objects -from .jms_api import JmsApi, _monitor_operation, get_project +from .jms_api import JmsApi, _monitor_operation log = logging.getLogger(__name__) From 0afeccf46920179c883970d118ee9aeaf526d04a Mon Sep 17 00:00:00 2001 From: Ryan Wehrle Date: Fri, 21 Apr 2023 10:12:43 -0400 Subject: [PATCH 6/8] Fix project mock data and test --- tests/jms/test_projects.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/tests/jms/test_projects.py b/tests/jms/test_projects.py index 08ba16d87..5de87b2bf 100644 --- a/tests/jms/test_projects.py +++ b/tests/jms/test_projects.py @@ -44,34 +44,6 @@ def test_project_deserialization(self): }, "num_jobs": 56, }, - "file_storages": [ - { - "obj_type": "RestGateway", - "name": "dc_fs_gateway", - "url": "https://212.126.163.153:443/dcs/fs/api", - "use_default_url": False, - "priority": 20, - "reference": "file_system_storage", - }, - { - "obj_type": "FileSystemStorage", - "name": "file_system_storage", - "cache": False, - "persistent": True, - "priority": 10, - "storage_directory": "/media/dcp_data/ansft_gateway/default_fs_storage/", - "owner_uuid": "0ea21dc4-37da-46e2-85e2-4f4a78dcdf0a", - }, - { - "obj_type": "FileSystemStorage", - "name": "shared_file_system_storage", - "cache": False, - "persistent": True, - "priority": 5, - "storage_directory": "/media/ansys/tmp_storage/", - "owner_uuid": "0ea21dc4-37da-46e2-85e2-4f4a78dcdf0a", - }, - ], } project = ProjectSchema().load(project_dict) @@ -89,11 +61,6 @@ def test_project_deserialization(self): project_dict["statistics"]["eval_status"]["failed"], ) - self.assertEqual(len(project.file_storages), 3) - self.assertEqual(project.file_storages[0]["name"], "dc_fs_gateway") - self.assertEqual(project.file_storages[0]["reference"], "file_system_storage") - self.assertEqual(project.file_storages[2]["storage_directory"], "/media/ansys/tmp_storage/") - def test_project_serialization(self): project = Project(name="new_project") @@ -104,7 +71,6 @@ def test_project_serialization(self): serialized_project = ProjectSchema().dump(project) self.assertTrue("name" in serialized_project.keys()) - self.assertFalse("file_storages" in serialized_project.keys()) self.assertEqual(serialized_project["name"], "new_project") def test_project_integration(self): From ea3b5c12b571474a3a781c50dbc86367effcab3c Mon Sep 17 00:00:00 2001 From: Federico Negri Date: Wed, 10 May 2023 11:32:56 +0200 Subject: [PATCH 7/8] Bump version 0.2.dev0 --- .github/workflows/ci_cd.yml | 6 +++--- ansys/rep/client/__version__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 5359ebed8..aeaa9dc2b 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -173,6 +173,6 @@ jobs: uses: softprops/action-gh-release@v1 with: generate_release_notes: true - files: | - dist/*.whl - build/sphinx/html \ No newline at end of file + # files: | + # dist/*.whl + # build/sphinx/html \ No newline at end of file diff --git a/ansys/rep/client/__version__.py b/ansys/rep/client/__version__.py index bf3f80463..b9c7f7b58 100644 --- a/ansys/rep/client/__version__.py +++ b/ansys/rep/client/__version__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.dev0" +__version__ = "0.2.dev0" __external_version__ = "2023 R2" __version_no_dots__ = "232" __company__ = "ANSYS, Inc." From 5133487eecaf86fb26bf561f3801c74fbe76bee2 Mon Sep 17 00:00:00 2001 From: Federico Negri Date: Wed, 10 May 2023 11:59:57 +0200 Subject: [PATCH 8/8] Add storage test --- tests/jms/test_jms_api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/jms/test_jms_api.py b/tests/jms/test_jms_api.py index c23ae38fb..3151b2e87 100644 --- a/tests/jms/test_jms_api.py +++ b/tests/jms/test_jms_api.py @@ -120,6 +120,16 @@ def test_fields_query_parameter(self): # Delete project JmsApi(client1).delete_project(project) + def test_storage_configuration(self): + + client = self.client() + jms_api = JmsApi(client) + storages = jms_api.get_storage() + for storage in storages: + self.assertTrue("name" in storage) + self.assertTrue("priority" in storage) + self.assertTrue("obj_type" in storage) + if __name__ == "__main__": unittest.main()