diff --git a/oscar_python/_providers/_s3.py b/oscar_python/_providers/_s3.py index 176ceb3..b5a4a32 100644 --- a/oscar_python/_providers/_s3.py +++ b/oscar_python/_providers/_s3.py @@ -59,7 +59,7 @@ def upload_file(self, local_path, remote_path): bucket_name = remote_path.split('/')[0] file_key = remote_path.split('/', 1)[1] file_name = local_path.split('/')[-1] - print("Uploading to bucket '{0}' with key '{1}'".format(bucket_name,file_key)) + print("Uploading to bucket '{0}' with key '{1}'".format(bucket_name, file_key)) with open(local_path, 'rb') as data: try: self.client.upload_fileobj(data, bucket_name, file_key + "/" + file_name) diff --git a/oscar_python/_utils.py b/oscar_python/_utils.py index d159963..945d8c4 100644 --- a/oscar_python/_utils.py +++ b/oscar_python/_utils.py @@ -13,6 +13,7 @@ # limitations under the License. import base64 +import json import os import requests import liboidcagent as agent @@ -32,14 +33,16 @@ def make_request(c, path, method, **kwargs): url = c.endpoint+path if method in ["post", "put"]: - if "token" in kwargs.keys() and kwargs["token"]: + if "token" in kwargs.keys() and kwargs["token"]: headers = get_headers_with_token(kwargs["token"]) + req_kwargs = {"headers": headers, "verify": c.ssl, "timeout": timeout} if "data" in kwargs.keys() and kwargs["data"]: - result = requests.request(method, url, headers=headers, verify=c.ssl, data=kwargs["data"], timeout=timeout) + req_kwargs["data"] = kwargs["data"] + result = requests.request(method, url, **req_kwargs) else: result = requests.request(method, url, headers=headers, verify=c.ssl, timeout=timeout) - if "handle" in kwargs.keys() and kwargs["handle"] == False: + if "handle" in kwargs.keys() and kwargs["handle"] is False: return result result.raise_for_status() @@ -94,7 +97,7 @@ def decode_b64(b64_str, file_out): except ValueError: print('Error decoding output: Invalid base64 string.') except OSError: - print('Error decoding output: Failed to write decoded data to file.') + print('Error decoding output: Failed to write decoded data to file.') def encode_input(data): @@ -111,6 +114,21 @@ def encode_input(data): return base64.b64encode(message_bytes) +def load_config(config_path, cluster_name="default"): + with open(config_path) as f: + config = json.load(f) + cluster = config["clusters"][cluster_name] + opts = { + "cluster_id": cluster_name, + "endpoint": cluster["endpoint"], + "ssl": cluster.get("ssl", True), + } + for key in ("user", "password", "shortname", "oidc_token", "refresh_token"): + if key in cluster: + opts[key] = cluster[key] + return opts + + def decode_output(output, file_path): if isBase64(output): decode_b64(output, file_path) diff --git a/oscar_python/client.py b/oscar_python/client.py index 834ee45..8c74e8d 100644 --- a/oscar_python/client.py +++ b/oscar_python/client.py @@ -27,7 +27,12 @@ _SVC_PATH = "/system/services" _LOGS_PATH = "/system/logs" _RUN_PATH = "/run" -_STATUS_PATH="/system/status" +_STATUS_PATH = "/system/status" +_HEALTH_PATH = "/health" +_VOLUMES_PATH = "/system/volumes" +_BUCKETS_PATH = "/system/buckets" +_METRICS_PATH = "/system/metrics" +_QUOTAS_USER_PATH = "/system/quotas/user" # _JOB_PATH = "/job" @@ -101,13 +106,13 @@ def get_access_token(self): """ Creates a generic storage client to interact with the storage providers defined on a specific service of the refered OSCAR cluster """ def create_storage_client(self, svc=None): - if svc != None: + if svc is not None: return Storage( client_obj=self, svc_name=svc) else: return Storage( client_obj=self) - + """ Function to get cluster info """ def get_cluster_info(self): return utils.make_request(self, _INFO_PATH, _GET) @@ -137,17 +142,18 @@ def _check_fdl_definition(self, fdl_path): raise Exception("FDL clusterID does not match current clusterID: {0}".format(err)) try: if os.path.isabs(svc["script"]): - script_path = svc["script"] + script_path = svc["script"] else: fdl_directory = os.path.dirname(fdl_path) script_path = os.path.join(fdl_directory, svc['script']) with open(script_path) as s: svc["script"] = s.read() - except IOError as e: + except IOError: raise Exception("Couldn't read script") # cpu parameter has to be string on the request - if type(svc["cpu"]) is int or type(svc["cpu"]) is float: svc["cpu"] = str(svc["cpu"]) + if type(svc["cpu"]) is int or type(svc["cpu"]) is float: + svc["cpu"] = str(svc["cpu"]) except ValueError as err: print(err) @@ -198,6 +204,8 @@ def remove_service(self, name): return utils.make_request(self, _SVC_PATH+"/"+name, _DELETE) def _get_token(self, svc): + if self._AUTH_TYPE != 'basicauth': + return self.get_access_token() service = utils.make_request(self, _SVC_PATH+"/"+svc, _GET) service = json.loads(service.text) return service["token"] @@ -224,3 +232,111 @@ def remove_job(self, svc, job): """ Remove all service jobs """ def remove_all_jobs(self, svc): return utils.make_request(self, _LOGS_PATH+"/"+svc, _DELETE) + + """ Check cluster health """ + def health_check(self): + return utils.make_request(self, _HEALTH_PATH, _GET) + + """ Get deployment status of a service """ + def get_deployment_status(self, name): + return utils.make_request(self, _SVC_PATH + "/" + name + "/deployment", _GET) + + """ Get deployment logs of a service """ + def get_deployment_logs(self, name): + return utils.make_request(self, _SVC_PATH + "/" + name + "/deployment/logs", _GET) + + """ List all managed volumes """ + def list_volumes(self): + return utils.make_request(self, _VOLUMES_PATH, _GET) + + """ Create a new managed volume """ + def create_volume(self, name, size): + data = json.dumps({"name": name, "size": size}) + return utils.make_request(self, _VOLUMES_PATH, _POST, data=data) + + """ Get a specific managed volume """ + def get_volume(self, name): + return utils.make_request(self, _VOLUMES_PATH + "/" + name, _GET) + + """ Delete a managed volume """ + def delete_volume(self, name): + return utils.make_request(self, _VOLUMES_PATH + "/" + name, _DELETE) + + """ Create a bucket """ + def create_bucket(self, name, visibility="private", allowed_users=None): + data = json.dumps({ + "bucket_name": name, + "visibility": visibility, + "allowed_users": allowed_users or [] + }) + return utils.make_request(self, _BUCKETS_PATH, _POST, data=data) + + """ Update a bucket """ + def update_bucket(self, name, visibility, allowed_users=None): + data = json.dumps({ + "bucket_name": name, + "visibility": visibility, + "allowed_users": allowed_users or [] + }) + return utils.make_request(self, _BUCKETS_PATH, _PUT, data=data) + + """ List all buckets """ + def list_buckets(self): + return utils.make_request(self, _BUCKETS_PATH, _GET) + + """ Get a specific bucket """ + def get_bucket(self, name): + return utils.make_request(self, _BUCKETS_PATH + "/" + name, _GET) + + """ Delete a bucket """ + def delete_bucket(self, name): + return utils.make_request(self, _BUCKETS_PATH + "/" + name, _DELETE) + + """ Get a presigned URL for a bucket file """ + def presign_bucket(self, name, object_key, operation="download", expires=0, content_type="", extra_headers=None): + path = _BUCKETS_PATH + "/" + name + "/presign" + data = json.dumps({ + "object_key": object_key, + "operation": operation, + "expires": expires, + "content_type": content_type, + "extra_headers": extra_headers or {}, + }) + return utils.make_request(self, path, _POST, data=data) + + """ Get system logs (admin only) """ + def get_system_logs(self, timestamps=False, previous=False): + path = _LOGS_PATH + params = [] + if timestamps: + params.append("timestamps=true") + if previous: + params.append("previous=true") + if params: + path += "?" + "&".join(params) + return utils.make_request(self, path, _GET) + + """ Get metrics summary """ + def get_metrics_summary(self): + return utils.make_request(self, _METRICS_PATH, _GET) + + """ Get metrics breakdown """ + def get_metrics_breakdown(self, group_by="service"): + return utils.make_request(self, _METRICS_PATH + "/breakdown?group_by=" + group_by, _GET) + + """ Get metrics for a specific service """ + def get_service_metrics(self, service_name): + return utils.make_request(self, _METRICS_PATH + "/" + service_name, _GET) + + """ Get own quota """ + def get_own_quota(self): + return utils.make_request(self, _QUOTAS_USER_PATH, _GET) + + """ Get quota for a specific user """ + def get_user_quota(self, user_id): + return utils.make_request(self, _QUOTAS_USER_PATH + "/" + user_id, _GET) + + """ Update quota for a user """ + def update_user_quota(self, user_id, cpu, memory): + data = json.dumps({"cpu": cpu, "memory": memory}) + return utils.make_request(self, _QUOTAS_USER_PATH + "/" + user_id, _PUT, data=data) diff --git a/oscar_python/local_test.py b/oscar_python/local_test.py index 0cad50a..6c0d77d 100644 --- a/oscar_python/local_test.py +++ b/oscar_python/local_test.py @@ -1,6 +1,6 @@ from client import Client -client = Client("oscar-gpu-cluster","https://focused-boyd8.im.grycap.net", "oscar", "oscar123", True) +client = Client("oscar-gpu-cluster", "https://focused-boyd8.im.grycap.net", "oscar", "oscar123", True) res = client.remove_service("cowsay") if res: diff --git a/oscar_python/storage.py b/oscar_python/storage.py index 4584ec0..e0653f2 100644 --- a/oscar_python/storage.py +++ b/oscar_python/storage.py @@ -31,10 +31,10 @@ # TODO check returns from functions class Storage: - def __init__(self, client_obj, svc_name = None) -> None: + def __init__(self, client_obj, svc_name=None) -> None: self.client_obj = client_obj self.storage_providers = {} - if svc_name != None: + if svc_name is not None: self.svc_name = svc_name self._store_provider_from_service() self._store_default_minio_provider() @@ -46,13 +46,12 @@ def _store_provider_from_service(self): """ Function to store the user credentials for the default MinIO provider """ def _store_default_minio_provider(self): - config = utils.make_request(self.client_obj, _CONFIG_PATH + "/" , _GET) + config = utils.make_request(self.client_obj, _CONFIG_PATH + "/", _GET) if _MINIO in self.storage_providers: self.storage_providers[_MINIO]["default"] = json.loads(config.text)["minio_provider"] else: default = {"default": json.loads(config.text)["minio_provider"]} self.storage_providers[_MINIO] = default - """ Function to retreive credentials of a specific storage provider """ def _get_provider_creds(self, provider, provider_name): diff --git a/tests/test_client.py b/tests/test_client.py index 6d38181..16ba450 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -107,7 +107,12 @@ def test_create_service_from_dict(options): def test_create_service_from_file(options): client = Client(options) - service_definition = "functions:\n oscar:\n - test_cluster:\n name: test_service\n script: test_script\n cpu: 1" + service_definition = ( + "functions:\n oscar:\n - test_cluster:\n" + " name: test_service\n" + " script: test_script\n" + " cpu: 1" + ) service_file = "path/to/service.yaml" with patch('os.path.isfile', return_value=True), \ patch('builtins.open', mock_open(read_data=service_definition)), \ @@ -136,3 +141,184 @@ def test_remove_service(options): with patch('oscar_python._utils.make_request') as mock_request: client.remove_service("test_service") mock_request.assert_called_once_with(client, "/system/services/test_service", "delete") + + +def test_health_check(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.health_check() + mock_request.assert_called_once_with(client, "/health", "get") + + +def test_get_deployment_status(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_deployment_status("test_service") + mock_request.assert_called_once_with(client, "/system/services/test_service/deployment", "get") + + +def test_get_deployment_logs(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_deployment_logs("test_service") + mock_request.assert_called_once_with(client, "/system/services/test_service/deployment/logs", "get") + + +def test_list_volumes(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.list_volumes() + mock_request.assert_called_once_with(client, "/system/volumes", "get") + + +def test_create_volume(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.create_volume("test_vol", "1Gi") + mock_request.assert_called_once_with(client, "/system/volumes", "post", + data=json.dumps({"name": "test_vol", "size": "1Gi"})) + + +def test_get_volume(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_volume("test_vol") + mock_request.assert_called_once_with(client, "/system/volumes/test_vol", "get") + + +def test_delete_volume(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.delete_volume("test_vol") + mock_request.assert_called_once_with(client, "/system/volumes/test_vol", "delete") + + +def test_list_buckets(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.list_buckets() + mock_request.assert_called_once_with(client, "/system/buckets", "get") + + +def test_get_bucket(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_bucket("test_bucket") + mock_request.assert_called_once_with(client, "/system/buckets/test_bucket", "get") + + +def test_create_bucket(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.create_bucket("test_bucket") + mock_request.assert_called_once_with( + client, "/system/buckets", "post", + data=json.dumps({"bucket_name": "test_bucket", + "visibility": "private", + "allowed_users": []})) + + +def test_create_bucket_with_visibility(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.create_bucket("test_bucket", visibility="public", allowed_users=["user1"]) + mock_request.assert_called_once_with( + client, "/system/buckets", "post", + data=json.dumps({"bucket_name": "test_bucket", + "visibility": "public", + "allowed_users": ["user1"]})) + + +def test_update_bucket(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.update_bucket("test_bucket", "public", ["user1"]) + mock_request.assert_called_once_with( + client, "/system/buckets", "put", + data=json.dumps({"bucket_name": "test_bucket", + "visibility": "public", + "allowed_users": ["user1"]})) + + +def test_delete_bucket(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.delete_bucket("test_bucket") + mock_request.assert_called_once_with(client, "/system/buckets/test_bucket", "delete") + + +def test_presign_bucket(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.presign_bucket("test_bucket", "file.txt", operation="upload", expires=3600) + mock_request.assert_called_once_with( + client, "/system/buckets/test_bucket/presign", "post", + data=json.dumps({"object_key": "file.txt", + "operation": "upload", + "expires": 3600, + "content_type": "", + "extra_headers": {}})) + + +def test_get_system_logs(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_system_logs() + mock_request.assert_called_once_with(client, "/system/logs", "get") + + +def test_get_system_logs_with_flags(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_system_logs(timestamps=True, previous=True) + mock_request.assert_called_once_with(client, "/system/logs?timestamps=true&previous=true", "get") + + +def test_get_metrics_summary(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_metrics_summary() + mock_request.assert_called_once_with(client, "/system/metrics", "get") + + +def test_get_metrics_breakdown(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_metrics_breakdown("user") + mock_request.assert_called_once_with(client, "/system/metrics/breakdown?group_by=user", "get") + + +def test_get_metrics_breakdown_default(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_metrics_breakdown() + mock_request.assert_called_once_with(client, "/system/metrics/breakdown?group_by=service", "get") + + +def test_get_service_metrics(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_service_metrics("test_service") + mock_request.assert_called_once_with(client, "/system/metrics/test_service", "get") + + +def test_get_own_quota(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_own_quota() + mock_request.assert_called_once_with(client, "/system/quotas/user", "get") + + +def test_get_user_quota(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_user_quota("test_user") + mock_request.assert_called_once_with(client, "/system/quotas/user/test_user", "get") + + +def test_update_user_quota(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.update_user_quota("test_user", "2", "4Gi") + mock_request.assert_called_once_with(client, "/system/quotas/user/test_user", "put", + data=json.dumps({"cpu": "2", "memory": "4Gi"})) diff --git a/tests/test_default_client.py b/tests/test_default_client.py index 0ec71bc..74428e1 100644 --- a/tests/test_default_client.py +++ b/tests/test_default_client.py @@ -25,7 +25,9 @@ def test_run_service_with_input_and_token(mock_decode_output, mock_encode_input, response = client.run_service("test_service", input="test_input", token="test_token", output="output_file", timeout=30) mock_encode_input.assert_called_once_with("test_input") - mock_make_request.assert_called_once_with(client, _RUN_PATH+"/test_service", _POST, data="encoded_input", token="test_token", timeout=30) + mock_make_request.assert_called_once_with( + client, _RUN_PATH+"/test_service", _POST, + data="encoded_input", token="test_token", timeout=30) mock_decode_output.assert_called_once_with("response_text", "output_file") assert response == mock_response @@ -41,7 +43,9 @@ def test_run_service_with_input_no_token(mock_encode_input, mock_make_request, c response = client.run_service("test_service", input="test_input") mock_encode_input.assert_called_once_with("test_input") - mock_make_request.assert_called_once_with(client, _RUN_PATH+"/test_service", _POST, data="encoded_input", token="test_token", timeout=None) + mock_make_request.assert_called_once_with( + client, _RUN_PATH+"/test_service", _POST, + data="encoded_input", token="test_token", timeout=None) assert response == mock_response diff --git a/tests/test_storage.py b/tests/test_storage.py index 0f3318d..2d42e33 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -1,3 +1,4 @@ +import json import pytest from unittest.mock import MagicMock, patch from oscar_python.storage import Storage @@ -11,7 +12,10 @@ def mock_client_obj(): @pytest.fixture def storage(mock_client_obj): mock_response = MagicMock() - mock_response.text = '{"minio_provider": {"access_key": "key","secret_key": "secret", "endpoint": "http://test.endpoint", "region": "us-east-1", "verify": false}}' + mock_response.text = json.dumps({ + "minio_provider": {"access_key": "key", "secret_key": "secret", + "endpoint": "http://test.endpoint", + "region": "us-east-1", "verify": False}}) with patch('oscar_python._utils.make_request', return_value=mock_response): return Storage(mock_client_obj) diff --git a/tests/test_utils.py b/tests/test_utils.py index b636fef..dcc66dc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,6 @@ import base64 +import json +import pytest from unittest.mock import patch, MagicMock, mock_open import oscar_python._utils as utils @@ -74,7 +76,72 @@ class MockClient: mock_request.return_value.raise_for_status = MagicMock() response = utils.make_request(c, "/test", "post", data="test_data", token="test_token") assert response.status_code == 200 - mock_request.assert_called_once_with("post", "http://test.com/test", headers={"Authorization": "Bearer test_token"}, verify=True, data="test_data", timeout=60) + mock_request.assert_called_once_with( + "post", "http://test.com/test", + headers={"Authorization": "Bearer test_token"}, + verify=True, data="test_data", timeout=60) + + +def test_load_config(tmp_path): + config_file = tmp_path / "config.json" + config_file.write_text(json.dumps({ + "clusters": { + "default": { + "endpoint": "http://test.cluster", + "user": "test_user", + "password": "test_pass", + "ssl": False + } + } + })) + opts = utils.load_config(str(config_file)) + assert opts["cluster_id"] == "default" + assert opts["endpoint"] == "http://test.cluster" + assert opts["user"] == "test_user" + assert opts["password"] == "test_pass" + assert opts["ssl"] is False + + +def test_load_config_with_oidc(tmp_path): + config_file = tmp_path / "config.json" + config_file.write_text(json.dumps({ + "clusters": { + "oidc_cluster": { + "endpoint": "http://oidc.cluster", + "oidc_token": "test_token", + "ssl": True + } + } + })) + opts = utils.load_config(str(config_file), "oidc_cluster") + assert opts["cluster_id"] == "oidc_cluster" + assert opts["endpoint"] == "http://oidc.cluster" + assert "user" not in opts + assert opts["oidc_token"] == "test_token" + assert opts["ssl"] is True + + +def test_load_config_missing_cluster(tmp_path): + config_file = tmp_path / "config.json" + config_file.write_text(json.dumps({"clusters": {}})) + with pytest.raises(KeyError): + utils.load_config(str(config_file), "nonexistent") + + +def test_load_config_minimal(tmp_path): + config_file = tmp_path / "config.json" + config_file.write_text(json.dumps({ + "clusters": { + "minimal": { + "endpoint": "http://minimal.cluster" + } + } + })) + opts = utils.load_config(str(config_file), "minimal") + assert opts["endpoint"] == "http://minimal.cluster" + assert opts["ssl"] is True + for key in ("user", "password", "shortname", "oidc_token", "refresh_token"): + assert key not in opts def test_make_request_get(): diff --git a/tests/test_webdav.py b/tests/test_webdav.py index ba321ae..50d8526 100644 --- a/tests/test_webdav.py +++ b/tests/test_webdav.py @@ -34,7 +34,7 @@ def test_webdav_download_file(webdav): webdav.client = MagicMock(["download_sync"]) webdav.client.download_sync.return_value = None - with patch("builtins.open", mock_open()) as mock_file: + with patch("builtins.open", mock_open()): webdav.download_file('local_path', 'remote_path/file.txt') webdav.client.download_sync.assert_called_with('remote_path/file.txt', 'local_path/file.txt')