From 02490505cb690412ead8d57dfd90058e1f9e44f8 Mon Sep 17 00:00:00 2001 From: Federico Negri Date: Wed, 22 Mar 2023 08:29:58 +0100 Subject: [PATCH 1/3] Working --- ansys/rep/client/client.py | 6 ++ ansys/rep/client/common/base_resource.py | 2 +- ansys/rep/client/jms/api/base.py | 4 -- ansys/rep/client/jms/api/project_api.py | 2 - doc/source/quickstart.rst | 75 +++++++++++++++--------- tests/jms/test_tasks.py | 10 +--- 6 files changed, 55 insertions(+), 44 deletions(-) diff --git a/ansys/rep/client/client.py b/ansys/rep/client/client.py index e5ac9ad08..637446c29 100644 --- a/ansys/rep/client/client.py +++ b/ansys/rep/client/client.py @@ -37,6 +37,8 @@ class Client(object): Refresh Token access_token : str, optional Access Token + always_request_all_fields: bool, optional + Set (True by default) Examples -------- @@ -72,6 +74,7 @@ def __init__( access_token: str = None, refresh_token: str = None, auth_url: str = None, + always_request_all_fields=True, ): self.rep_url = rep_url @@ -103,6 +106,9 @@ def __init__( self.refresh_token = tokens["refresh_token"] self.session = create_session(self.access_token) + if always_request_all_fields: + self.session.params = {"fields": "all"} + # register hook to handle expiring of the refresh token self.session.hooks["response"] = [self._auto_refresh_token, raise_for_status] self._unauthorized_num_retry = 0 diff --git a/ansys/rep/client/common/base_resource.py b/ansys/rep/client/common/base_resource.py index a214e679b..92be29464 100644 --- a/ansys/rep/client/common/base_resource.py +++ b/ansys/rep/client/common/base_resource.py @@ -67,7 +67,7 @@ def __str__(self): for attr_name, field_obj in schema.fields.items(): value = missing try: - value = field_obj.serialize(attr_name, self, accessor=schema.get_attribute) + value = field_obj.get_value(self, attr_name, accessor=schema.get_attribute) except: pass if value is missing: diff --git a/ansys/rep/client/jms/api/base.py b/ansys/rep/client/jms/api/base.py index 1982708bf..4e5c0b544 100644 --- a/ansys/rep/client/jms/api/base.py +++ b/ansys/rep/client/jms/api/base.py @@ -16,7 +16,6 @@ def get_objects( rest_name = obj_type.Meta.rest_name url = f"{url}/{rest_name}" - query_params.setdefault("fields", "all") r = session.get(url, params=query_params) if query_params.get("count"): @@ -36,7 +35,6 @@ def get_object( rest_name = obj_type.Meta.rest_name url = f"{url}/{rest_name}/{id}" - query_params.setdefault("fields", "all") r = session.get(url, params=query_params) data = r.json()[rest_name] @@ -68,7 +66,6 @@ def create_objects( rest_name = obj_type.Meta.rest_name url = f"{url}/{rest_name}" - query_params.setdefault("fields", "all") schema = obj_type.Meta.schema(many=True) serialized_data = schema.dump(objects) json_data = json.dumps({rest_name: serialized_data}) @@ -100,7 +97,6 @@ def update_objects( rest_name = obj_type.Meta.rest_name url = f"{url}/{rest_name}" - query_params.setdefault("fields", "all") schema = obj_type.Meta.schema(many=True) serialized_data = schema.dump(objects) json_data = json.dumps({rest_name: serialized_data}) diff --git a/ansys/rep/client/jms/api/project_api.py b/ansys/rep/client/jms/api/project_api.py index 365664230..0ccb98281 100644 --- a/ansys/rep/client/jms/api/project_api.py +++ b/ansys/rep/client/jms/api/project_api.py @@ -549,8 +549,6 @@ def copy_jobs(project_api: ProjectApi, jobs: List[Job], as_objects=True, **query url = f"{project_api.url}/jobs" - query_params.setdefault("fields", "all") - json_data = json.dumps({"source_ids": [obj.id for obj in jobs]}) r = project_api.client.session.post(f"{url}", data=json_data, params=query_params) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index da8e2fb8f..3cea25d28 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -67,7 +67,7 @@ Query parameters Most ``get`` functions support filtering by query parameters. .. code-block:: python - + project = jms_api.get_project_by_name(name="Mapdl Motorbike Frame") project_api = ProjectApi(client, project.id) @@ -77,34 +77,9 @@ Most ``get`` functions support filtering by query parameters. # Get id and parameter values for all evaluated jobs jobs = project_api.get_jobs(fields=["id", "values"], eval_status="evaluated") - # Get name and elapsed time of max 5 evaluated jobs - jobs = project_api.get_jobs(fields=["name", "elapsed_time"], - eval_status="evaluated", limit=5) - for job in jobs: - print(job) - # { - # "id": "02qoqedl8QCjkuLcqCi10Q", - # "name": "Job.0", - # "priority": 0, - # "elapsed_time": 35.275044 - # } - # { - # "id": "02qoqedlDMO1LrSGoHQqnT", - # "name": "Job.1", - # "priority": 0, - # "elapsed_time": 34.840801 - # } - # ... - - # Get all jobs sorted by fitness value in ascending order - jobs = project_api.get_jobs(sort="fitness") - # Get all jobs sorted by fitness value in descending order - jobs = project_api.get_jobs(sort="-fitness") - - # Get all jobs sorted by the parameters tube1 and weight - jobs = project_api.get_jobs(sort=["values.tube1", "values.weight"]) - print([(job.values["tube1"], job.values["weight"]) for job in jobs]) +Operators +^^^^^^^^^ In general, query parameters support the following operators: ``lt`` (less than), ``le`` (less or equal), ``=`` (equal), ``ne`` (not equal), ``ge`` (greater or equal), ``gt`` (greater than), ``in`` (value found in list) and @@ -126,6 +101,50 @@ In general, query parameters support the following operators: ``lt`` (less than) query_params = {"fitness.lt": 1.8} jobs = project_api.get_jobs(**query_params) + +Fields +^^^^^^ + +When you query a resource, it returns a set of fields by default. You can specify which fields +you want returned by using the ``fields`` query parameter (this returns only the fields you specify, +and the ID of the resource, which is always returned). Moreover, you can request all fields to be returned by specifying ``fields="all"``. + +.. code-block:: python + + project = jms_api.get_project_by_name(name="Mapdl Motorbike Frame") + project_api = ProjectApi(client, project.id) + + # Get all jobs with all fields + jobs = project_api.get_jobs() + + # Get id and parameter values for all evaluated jobs + jobs = project_api.get_jobs(fields=["id", "values"], eval_status="evaluated") + +Sorting +^^^^^^^ + +.. code-block:: python + + # Get all jobs sorted by fitness value in ascending order + jobs = project_api.get_jobs(sort="fitness") + + # Get all jobs sorted by fitness value in descending order + jobs = project_api.get_jobs(sort="-fitness") + + # Get all jobs sorted by the parameters tube1 and weight + jobs = project_api.get_jobs(sort=["values.tube1", "values.weight"]) + print([(job.values["tube1"], job.values["weight"]) for job in jobs]) + +Pagination +^^^^^^^^^^ + +.. code-block:: python + + # Get name and elapsed time of max 5 evaluated jobs + jobs = project_api.get_jobs(fields=["name", "elapsed_time"], + eval_status="evaluated", limit=5) + + Objects vs dictionaries ----------------------------------- diff --git a/tests/jms/test_tasks.py b/tests/jms/test_tasks.py index 4d16c0691..386d85c9a 100644 --- a/tests/jms/test_tasks.py +++ b/tests/jms/test_tasks.py @@ -97,18 +97,10 @@ def test_task_deserialization(self): def test_task_integration(self): - # This test assumes that the project mapdl_motorbike_frame - # already exists on the REP server. - # In case, you can create such project running the script - # examples/mapdl_motorbike_frame/project_setup.py - client = self.client() proj_name = "Mapdl Motorbike Frame" - jms_api = JmsApi(client) - project = jms_api.get_project_by_name(name=proj_name) - if not project: - project = create_project(client, proj_name, num_jobs=5, use_exec_script=False) + project = create_project(client, proj_name, num_jobs=5, use_exec_script=False) project_api = ProjectApi(client, project.id) tasks = project_api.get_tasks(limit=5) From aef0f44de03d2bb6f73015a4957e09b9321f609e Mon Sep 17 00:00:00 2001 From: Federico Negri Date: Wed, 22 Mar 2023 21:53:12 +0100 Subject: [PATCH 2/3] Update doc, adjust retry --- ansys/rep/client/client.py | 24 ++++++----- ansys/rep/client/common/base_resource.py | 2 +- ansys/rep/client/connection.py | 2 +- doc/source/quickstart.rst | 41 +++++++++++-------- examples/lsdyna_cylinder_plate/lsdyna_job.py | 4 +- .../mapdl_tyre_performance/project_setup.py | 5 ++- 6 files changed, 45 insertions(+), 33 deletions(-) diff --git a/ansys/rep/client/client.py b/ansys/rep/client/client.py index 637446c29..a0ec8fa03 100644 --- a/ansys/rep/client/client.py +++ b/ansys/rep/client/client.py @@ -37,8 +37,10 @@ class Client(object): Refresh Token access_token : str, optional Access Token - always_request_all_fields: bool, optional - Set (True by default) + all_fields: bool, optional + If True, the query parameter ``fields="all"`` is applied by default + to all requests, so that all available fields are returned for + the requested resources. Examples -------- @@ -46,17 +48,17 @@ class Client(object): >>> from ansys.rep.client import Client >>> # Create client object and connect to REP with username & password >>> cl = Client( - rep_url="https://localhost:8443/rep", username="repadmin", password="repadmin" - ) + ... rep_url="https://localhost:8443/rep", username="repuser", password="repuser" + ... ) >>> # Extract refresh token to eventually store it >>> refresh_token = cl.refresh_token >>> # Alternative: Create client object and connect to REP with refresh token >>> cl = Client( - rep_url="https://localhost:8443/rep", - username="repadmin", - refresh_token=refresh_token, - grant_type="refresh_token" - ) + ... rep_url="https://localhost:8443/rep", + ... username="repuser", + ... refresh_token=refresh_token, + ... grant_type="refresh_token" + >>> ) """ @@ -74,7 +76,7 @@ def __init__( access_token: str = None, refresh_token: str = None, auth_url: str = None, - always_request_all_fields=True, + all_fields=True, ): self.rep_url = rep_url @@ -106,7 +108,7 @@ def __init__( self.refresh_token = tokens["refresh_token"] self.session = create_session(self.access_token) - if always_request_all_fields: + if all_fields: self.session.params = {"fields": "all"} # register hook to handle expiring of the refresh token diff --git a/ansys/rep/client/common/base_resource.py b/ansys/rep/client/common/base_resource.py index 92be29464..a214e679b 100644 --- a/ansys/rep/client/common/base_resource.py +++ b/ansys/rep/client/common/base_resource.py @@ -67,7 +67,7 @@ def __str__(self): for attr_name, field_obj in schema.fields.items(): value = missing try: - value = field_obj.get_value(self, attr_name, accessor=schema.get_attribute) + value = field_obj.serialize(attr_name, self, accessor=schema.get_attribute) except: pass if value is missing: diff --git a/ansys/rep/client/connection.py b/ansys/rep/client/connection.py index 69371b218..62957582b 100644 --- a/ansys/rep/client/connection.py +++ b/ansys/rep/client/connection.py @@ -38,7 +38,7 @@ def create_session(access_token: str = None) -> requests.Session: if access_token: session.headers.update({"Authorization": "Bearer %s" % access_token}) - retries = Retry(total=10, backoff_factor=1, status_forcelist=[503]) + retries = Retry(total=5, backoff_factor=0.5, status_forcelist=[502, 503, 504]) session.mount("http://", HTTPAdapter(max_retries=retries)) session.mount("https://", HTTPAdapter(max_retries=retries)) return session diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 3cea25d28..1c0c13b33 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -62,24 +62,26 @@ of a tubular steel trellis motorbike-frame. See :ref:`example_mapdl_motorbike_frame` for a detailed description of this example. Query parameters ------------------------------------ +---------------- Most ``get`` functions support filtering by query parameters. +Properties +^^^^^^^^^^ + +You can query resources by the value of their properties. + .. code-block:: python project = jms_api.get_project_by_name(name="Mapdl Motorbike Frame") project_api = ProjectApi(client, project.id) - # Get all jobs with all fields + # Get all jobs jobs = project_api.get_jobs() - # Get id and parameter values for all evaluated jobs - jobs = project_api.get_jobs(fields=["id", "values"], eval_status="evaluated") - + # Get all evaluated jobs + jobs = project_api.get_jobs(eval_status="evaluated") -Operators -^^^^^^^^^ In general, query parameters support the following operators: ``lt`` (less than), ``le`` (less or equal), ``=`` (equal), ``ne`` (not equal), ``ge`` (greater or equal), ``gt`` (greater than), ``in`` (value found in list) and @@ -105,17 +107,15 @@ In general, query parameters support the following operators: ``lt`` (less than) Fields ^^^^^^ -When you query a resource, it returns a set of fields by default. You can specify which fields +When you query a resource, the REST API returns a set of fields by default. You can specify which fields you want returned by using the ``fields`` query parameter (this returns only the fields you specify, -and the ID of the resource, which is always returned). Moreover, you can request all fields to be returned by specifying ``fields="all"``. +and the ID of the resource, which is always returned). +Moreover, you can request all fields to be returned by specifying ``fields="all"``. .. code-block:: python - project = jms_api.get_project_by_name(name="Mapdl Motorbike Frame") - project_api = ProjectApi(client, project.id) - # Get all jobs with all fields - jobs = project_api.get_jobs() + jobs = project_api.get_jobs(fields="all") # Get id and parameter values for all evaluated jobs jobs = project_api.get_jobs(fields=["id", "values"], eval_status="evaluated") @@ -123,6 +123,9 @@ and the ID of the resource, which is always returned). Moreover, you can request Sorting ^^^^^^^ +You can sort resource collections by their properties. +Prefixing with ``-`` (minus) denotes descending order. + .. code-block:: python # Get all jobs sorted by fitness value in ascending order @@ -138,11 +141,17 @@ Sorting Pagination ^^^^^^^^^^ +You can use the ``offset`` and ``limit`` query parameters to paginate items in a collection. + .. code-block:: python - # Get name and elapsed time of max 5 evaluated jobs - jobs = project_api.get_jobs(fields=["name", "elapsed_time"], - eval_status="evaluated", limit=5) + # Get name and elapsed time of max 5 evaluated jobs, sorted by creation time + jobs = project_api.get_jobs(fields=["name", "elapsed_time"], sort="-creation_time", + eval_status="evaluated", limit=5) + + # Query the next 10 jobs + jobs = project_api.get_jobs(fields=["name", "elapsed_time"], sort="-creation_time", + eval_status="evaluated", limit=10, offset=5) Objects vs dictionaries diff --git a/examples/lsdyna_cylinder_plate/lsdyna_job.py b/examples/lsdyna_cylinder_plate/lsdyna_job.py index ce22b805c..fdc8ea885 100644 --- a/examples/lsdyna_cylinder_plate/lsdyna_job.py +++ b/examples/lsdyna_cylinder_plate/lsdyna_job.py @@ -190,8 +190,8 @@ def submit_job() -> REPJob: max_execution_time=3600.0, resource_requirements=ResourceRequirements( cpu_core_usage=6, - memory=6000, - disk_space=4000, + memory=6000 * 1024 * 1024, + disk_space=4000 * 1024 * 1024, ), execution_level=0, num_trials=1, diff --git a/examples/mapdl_tyre_performance/project_setup.py b/examples/mapdl_tyre_performance/project_setup.py index 09bcc502a..01e89cb61 100644 --- a/examples/mapdl_tyre_performance/project_setup.py +++ b/examples/mapdl_tyre_performance/project_setup.py @@ -181,8 +181,9 @@ def main(client: Client, name: str, num_jobs: int, version: str) -> Project: execution_command="%executable% -b -i %file:mac% -o file.out -np %resource:num_cores%", resource_requirements=ResourceRequirements( cpu_core_usage=4, - memory=4000, - disk_space=500, + memory=4000 * 1024 * 1024, + disk_space=500 * 1024 * 1024, + distributed=True, ), max_execution_time=1800.0, execution_level=0, From 3e404020e90c2c0599948a7acce2325726ca363c Mon Sep 17 00:00:00 2001 From: Federico Negri Date: Wed, 22 Mar 2023 22:19:38 +0100 Subject: [PATCH 3/3] Add test --- tests/jms/test_jms_api.py | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/jms/test_jms_api.py b/tests/jms/test_jms_api.py index a092693f7..c23ae38fb 100644 --- a/tests/jms/test_jms_api.py +++ b/tests/jms/test_jms_api.py @@ -9,7 +9,9 @@ import unittest from examples.mapdl_motorbike_frame.project_setup import create_project +from marshmallow.utils import missing +from ansys.rep.client import Client from ansys.rep.client.jms import JmsApi, ProjectApi from ansys.rep.client.jms.resource import Job, Project from tests.rep_test import REPTestCase @@ -75,6 +77,49 @@ def test_jms_api(self): # Delete Jobs again project_api.delete_jobs(created_jobs) + def test_fields_query_parameter(self): + + client1 = Client(self.rep_url, self.username, self.password, all_fields=False) + + proj_name = "test_fields_query_parameter" + project = create_project(client1, proj_name, num_jobs=2, use_exec_script=False) + + project_api1 = ProjectApi(client1, project.id) + + jobs = project_api1.get_jobs() + self.assertEqual(len(jobs), 2) + self.assertNotEqual(jobs[0].id, missing) + self.assertNotEqual(jobs[0].eval_status, missing) + self.assertEqual(jobs[0].values, missing) + self.assertEqual(jobs[0].host_ids, missing) + + jobs = project_api1.get_jobs(fields=["id", "eval_status", "host_ids"]) + self.assertEqual(len(jobs), 2) + self.assertNotEqual(jobs[0].id, missing) + self.assertNotEqual(jobs[0].eval_status, missing) + self.assertNotEqual(jobs[0].host_ids, missing) + self.assertEqual(jobs[0].values, missing) + + client2 = Client(self.rep_url, self.username, self.password, all_fields=True) + project_api2 = ProjectApi(client2, project.id) + + jobs = project_api2.get_jobs() + self.assertEqual(len(jobs), 2) + self.assertNotEqual(jobs[0].id, missing) + self.assertNotEqual(jobs[0].eval_status, missing) + self.assertNotEqual(jobs[0].values, missing) + self.assertNotEqual(jobs[0].host_ids, missing) + + jobs = project_api2.get_jobs(fields=["id", "eval_status"]) + self.assertEqual(len(jobs), 2) + self.assertNotEqual(jobs[0].id, missing) + self.assertNotEqual(jobs[0].eval_status, missing) + self.assertEqual(jobs[0].host_ids, missing) + self.assertEqual(jobs[0].values, missing) + + # Delete project + JmsApi(client1).delete_project(project) + if __name__ == "__main__": unittest.main()