From 10681764e370e51c21894f4a72696629d0506fe6 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Sun, 19 Apr 2026 19:14:51 +0200 Subject: [PATCH 1/3] feat: added file bundle task when downloading file archive --- obiba_opal/core.py | 48 +++++++++++++++++++++++++++++++++++++++----- obiba_opal/file.py | 42 ++++++++++++++++++++++++++++++++++++-- obiba_opal/system.py | 22 +++++++++++--------- pyproject.toml | 7 ++++++- uv.lock | 2 +- 5 files changed, 102 insertions(+), 19 deletions(-) diff --git a/obiba_opal/core.py b/obiba_opal/core.py index 7d4f4fe..9442078 100755 --- a/obiba_opal/core.py +++ b/obiba_opal/core.py @@ -25,6 +25,8 @@ def __init__(self, server=None): self.base_url = self.__ensure_entry("Opal address", server) self.id = None self.rid = None + self.profile = None + self.version = None def __del__(self): self.close() @@ -149,15 +151,15 @@ def token(self, token): :param token - token key """ tk = self.__ensure_entry("Token", token, True) - return self.header("X-Opal-Auth", tk) + self.header("X-Opal-Auth", tk) + self.init() + return self def init_otp(self): """ Checks if an OTP is needed and if yes, prompts the user for the security code """ - request = self.new_request() - profile_url = "/system/subject-profile/_current" - response = request.accept_json().get().resource(profile_url).send() + response = self.init() if response.code == 401: auth_header = response.get_header("WWW-Authenticate") if auth_header: @@ -166,7 +168,40 @@ def init_otp(self): val = input("Enter 6-digits code: ") # validate code and get the opalsid cookie for further requests request = self.new_request() - request.header(otp_header, val).accept_json().get().resource(profile_url).send() + self.init(request.header(otp_header, val)) + + def init(self, request=None): + """ + Initializes the client profile and version information by calling the system profile endpoint + + :return: the response of the profile endpoint + """ + _request = self.new_request() if request is None else request + profile_url = "/system/subject-profile/_current" + response = _request.accept_json().get().resource(profile_url).send() + if response.code == 200: + self.profile = response.from_json() + self.version = response.get_version() + return response + + def compare_version(self, version): + """ + Compares the Opal version with the provided version and raises an exception if the Opal version is lower than the provided version + + :param version - the version to compare with + :return: -1 if the Opal version is lower than the provided version, 0 if they are equal, 1 if the Opal version is higher than the provided version + """ + if self.version is None: + raise Exception("Opal version is not initialized") + clean_version = self.version.split("-")[0] # Remove any suffix like "-SNAPSHOT" + opal_version = tuple(map(int, clean_version.split("."))) + required_version = tuple(map(int, version.split("."))) + if opal_version < required_version: + return -1 + elif opal_version > required_version: + return 1 + else: + return 0 def verify(self, value): """ @@ -500,6 +535,9 @@ def get_header(self, header: str) -> str | None: def get_location(self): return self.get_header("Location") + def get_version(self): + return self.get_header("X-Opal-Version") + def extract_cookie_value(self, name: str) -> str | None: if "set-cookie" in self.response.headers: if isinstance(self.response.headers["set-cookie"], str): diff --git a/obiba_opal/file.py b/obiba_opal/file.py index 15c708e..acf8118 100755 --- a/obiba_opal/file.py +++ b/obiba_opal/file.py @@ -2,10 +2,14 @@ Opal file management. """ +import json + import obiba_opal.core as core import sys import os +from obiba_opal.system import TaskService + class FileService: """ @@ -90,7 +94,11 @@ def download_file(self, path: str, fd: int | os.PathLike, download_password: str :param fd: The file descriptor or path to the destination file :param download_password: The password to use to encrypt the downloaded zip archive + :raises Exception: If the file is not readable """ + info = self.file_info(path) + if not info["readable"]: + raise Exception(f"File {path} is not readable") request = self.client.new_request() request.fail_on_error() @@ -100,8 +108,30 @@ def download_file(self, path: str, fd: int | os.PathLike, download_password: str file = FileService.OpalFile(path) fp = os.fdopen(fd, "wb") if isinstance(fd, int) else fd - request.get().resource(file.get_ws()).accept("*/*").header("X-File-Key", download_password).send(fp) - fp.flush() + if self.client.compare_version("5.7.0") < 0: + # File download before Opal 5.7.0 + request.get().resource(file.get_ws()).accept("*/*").header("X-File-Key", download_password).send(fp) + fp.flush() + else: + # File download with Opal 5.7.0 or later + if download_password or info["type"] == "FOLDER": + # File to be bundled as zip archive + options = file.make_bundle_options(download_password) + response = request.post().resource("/shell/commands/_file-bundle").accept_json().content_type_json().content(json.dumps(options)).send() + task = response.from_json() + task_service = TaskService(self.client) + status = task_service.wait_task(task["id"], True) + if status == "SUCCEEDED": + # Task succeeded, downloading file + request.get().resource(f"/shell/command/{task['id']}/_result").accept("*/*").send(fp) + fp.flush() + request.delete().resource(f"/shell/command/{task['id']}").send() + else: + raise Exception(f"File bundle task failed with status: {status}") + else: + # File to be downloaded as is + request.get().resource(file.get_ws()).accept("*/*").header("X-File-Key", download_password).send(fp) + fp.flush() def upload_file(self, upload: str, path: str): """ @@ -167,3 +197,11 @@ def get_meta_ws(self): def get_ws(self): return f"/files{self.path}" + + def make_bundle_options(self, download_password): + options = { + "paths": [self.path] + } + if download_password: + options["password"] = download_password + return options diff --git a/obiba_opal/system.py b/obiba_opal/system.py index c95bae5..9ebe375 100644 --- a/obiba_opal/system.py +++ b/obiba_opal/system.py @@ -452,18 +452,20 @@ def cancel_task(self, id: str | int): request.content("CANCELED") request.put().resource(f"/shell/command/{id}/status").send() - def wait_task(self, id: str | int): + def wait_task(self, id: str | int, silently: bool = False): + """ Wait for the task to complete or being canceled, and return its status. """ task = self.get_task(id) - while task["status"] not in ["SUCCEEDED", "CANCELED", "FAILED"]: - if "progress" in task: - progress = task["progress"] - if "message" in progress: - sys.stdout.write("\r\033[K" + str(progress["percent"]) + "% " + progress["message"]) + while task["status"] not in ["SUCCEEDED", "CANCELED", "FAILED", "CANCEL_PENDING"]: + if not silently: + if "progress" in task: + progress = task["progress"] + if "message" in progress: + sys.stdout.write("\r\033[K" + str(progress["percent"]) + "% " + progress["message"]) + else: + sys.stdout.write("\r\033[K" + str(progress["percent"]) + "%") else: - sys.stdout.write("\r\033[K" + str(progress["percent"]) + "%") - else: - sys.stdout.write(".") - sys.stdout.flush() + sys.stdout.write(".") + sys.stdout.flush() time.sleep(1) task = self.get_task(id) return task["status"] diff --git a/pyproject.toml b/pyproject.toml index 3b242f1..7f6a202 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "obiba-opal" -version = "6.0.2" +version = "6.1.0" description = "OBiBa/Opal python client." authors = [ {name = "Yannick Marcon", email = "yannick.marcon@obiba.org"} @@ -52,3 +52,8 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["obiba_opal"] + +[tool.pytest.ini_options] +markers = [ + "integration: marks tests that require a running Opal instance", +] diff --git a/uv.lock b/uv.lock index cdb26b5..57cd145 100644 --- a/uv.lock +++ b/uv.lock @@ -174,7 +174,7 @@ wheels = [ [[package]] name = "obiba-opal" -version = "6.0.2" +version = "6.1.0" source = { editable = "." } dependencies = [ { name = "requests" }, From f2788021c37b5191df9b29bf28b3d37523ed5f97 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Sun, 19 Apr 2026 19:30:52 +0200 Subject: [PATCH 2/3] fix: do not reuse request Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- obiba_opal/file.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/obiba_opal/file.py b/obiba_opal/file.py index acf8118..e77c572 100755 --- a/obiba_opal/file.py +++ b/obiba_opal/file.py @@ -117,15 +117,28 @@ def download_file(self, path: str, fd: int | os.PathLike, download_password: str if download_password or info["type"] == "FOLDER": # File to be bundled as zip archive options = file.make_bundle_options(download_password) - response = request.post().resource("/shell/commands/_file-bundle").accept_json().content_type_json().content(json.dumps(options)).send() + bundle_request = self.client.new_request() + bundle_request.fail_on_error() + if self.verbose: + bundle_request.verbose() + response = bundle_request.post().resource("/shell/commands/_file-bundle").accept_json().content_type_json().content(json.dumps(options)).send() task = response.from_json() task_service = TaskService(self.client) status = task_service.wait_task(task["id"], True) if status == "SUCCEEDED": # Task succeeded, downloading file - request.get().resource(f"/shell/command/{task['id']}/_result").accept("*/*").send(fp) + download_request = self.client.new_request() + download_request.fail_on_error() + if self.verbose: + download_request.verbose() + download_request.get().resource(f"/shell/command/{task['id']}/_result").accept("*/*").send(fp) fp.flush() - request.delete().resource(f"/shell/command/{task['id']}").send() + + delete_request = self.client.new_request() + delete_request.fail_on_error() + if self.verbose: + delete_request.verbose() + delete_request.delete().resource(f"/shell/command/{task['id']}").send() else: raise Exception(f"File bundle task failed with status: {status}") else: From 9fe8e511d3cc34726e7d67c4ba3938000ac8684f Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Sun, 19 Apr 2026 19:41:36 +0200 Subject: [PATCH 3/3] fix: code review --- obiba_opal/core.py | 8 +++++++- obiba_opal/file.py | 34 ++++++++++++++++------------------ 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/obiba_opal/core.py b/obiba_opal/core.py index 9442078..4c70bed 100755 --- a/obiba_opal/core.py +++ b/obiba_opal/core.py @@ -69,13 +69,14 @@ def buildWithCertificate(cls, server, cert, key, no_ssl_verify: bool = False): :param key - private key (must be named as 'privatekey.pem') :param no_ssl_verify - if True, the SSL certificate is not verified (not recommended) + :return: the client instance """ client = cls(server) if client.base_url.startswith("https:"): client.session.verify = not no_ssl_verify client.session.cert = (cert, key) - + client.init() return client @classmethod @@ -88,6 +89,7 @@ def buildWithAuthentication(cls, server, user, password, no_ssl_verify: bool = F :param password - user password :param no_ssl_verify - if True, the SSL certificate is not verified (not recommended) + :return: the client instance """ client = cls(server) if client.base_url.startswith("https:"): @@ -116,6 +118,7 @@ def buildWithToken(cls, server, token, no_ssl_verify: bool = False): :param token - token key :param no_ssl_verify - if True, the SSL certificate is not verified (not recommended) + :return: the client instance """ client = cls(server) if client.base_url.startswith("https:"): @@ -136,6 +139,7 @@ def credentials(self, user, password): :param user - username :param password - user password + :return: the client instance """ u = self.__ensure_entry("User name", user) p = self.__ensure_entry("Password", password, True) @@ -149,6 +153,7 @@ def token(self, token): Creates the authorization header and attempts to input the required token :param token - token key + :return: the client instance """ tk = self.__ensure_entry("Token", token, True) self.header("X-Opal-Auth", tk) @@ -190,6 +195,7 @@ def compare_version(self, version): :param version - the version to compare with :return: -1 if the Opal version is lower than the provided version, 0 if they are equal, 1 if the Opal version is higher than the provided version + :raises Exception: if the Opal version is not initialized """ if self.version is None: raise Exception("Opal version is not initialized") diff --git a/obiba_opal/file.py b/obiba_opal/file.py index e77c572..5269463 100755 --- a/obiba_opal/file.py +++ b/obiba_opal/file.py @@ -3,11 +3,10 @@ """ import json - -import obiba_opal.core as core import sys import os +import obiba_opal.core as core from obiba_opal.system import TaskService @@ -117,30 +116,29 @@ def download_file(self, path: str, fd: int | os.PathLike, download_password: str if download_password or info["type"] == "FOLDER": # File to be bundled as zip archive options = file.make_bundle_options(download_password) - bundle_request = self.client.new_request() - bundle_request.fail_on_error() + bundle_request = self.client.new_request().fail_on_error() if self.verbose: bundle_request.verbose() response = bundle_request.post().resource("/shell/commands/_file-bundle").accept_json().content_type_json().content(json.dumps(options)).send() task = response.from_json() task_service = TaskService(self.client) - status = task_service.wait_task(task["id"], True) - if status == "SUCCEEDED": - # Task succeeded, downloading file - download_request = self.client.new_request() - download_request.fail_on_error() - if self.verbose: - download_request.verbose() - download_request.get().resource(f"/shell/command/{task['id']}/_result").accept("*/*").send(fp) - fp.flush() - - delete_request = self.client.new_request() - delete_request.fail_on_error() + try: + status = task_service.wait_task(task["id"], True) + if status == "SUCCEEDED": + # Task succeeded, downloading file + download_request = self.client.new_request().fail_on_error() + if self.verbose: + download_request.verbose() + download_request.get().resource(f"/shell/command/{task['id']}/_result").accept("*/*").send(fp) + fp.flush() + else: + raise Exception(f"File bundle task failed with status: {status}") + finally: + # Ensure task is deleted in any case to avoid leaving pending tasks on the server + delete_request = self.client.new_request().fail_on_error() if self.verbose: delete_request.verbose() delete_request.delete().resource(f"/shell/command/{task['id']}").send() - else: - raise Exception(f"File bundle task failed with status: {status}") else: # File to be downloaded as is request.get().resource(file.get_ws()).accept("*/*").header("X-File-Key", download_password).send(fp)