From 38779ed97cdb3c1e5d41504b57acb9eeb6f8f25a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 16 Nov 2025 11:33:29 +0100 Subject: [PATCH 1/9] expose base_data_dir in self instead of extra method "deposit..." --- qiita_client/testing.py | 65 +++++++---------------------------------- 1 file changed, 11 insertions(+), 54 deletions(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index ef19893..4ca6bac 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -7,8 +7,7 @@ # ----------------------------------------------------------------------------- from unittest import TestCase -from os import environ, sep -from os.path import join, isabs +from os import environ from time import sleep from qiita_client import QiitaClient @@ -46,6 +45,16 @@ def setUpClass(cls): cls.qclient._plugincoupling = environ.get( 'QIITA_PLUGINCOUPLING', BaseQiitaPlugin._DEFAULT_PLUGIN_COUPLINGS) + # use artifact 1 info to determine BASA_DATA_DIR, as we know that the + # filepath ends with ....raw_data/1_s_G1_L001_sequences.fastq.gz, thus + # BASE_DATA_DIR must be the prefix, e.g. /qiita_data/ + # This might break IF file + # qiita-spots/qiita/qiita_db/support_files/populate_test_db.sql + # changes. + ainfo = cls.qclient.get('/qiita_db/artifacts/1/') + cls.base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ + :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] + # Give enough time for the plugins to register sleep(5) @@ -83,55 +92,3 @@ def _wait_for_running_job(self, job_id): break return status - - def deposite_in_qiita_basedir(self, fps, update_fp_only=False): - """Pushs a file to qiita main AND adapts given filepath accordingly. - - A helper function to fix file paths in tests such that they point to - the expected BASE_DATA_DIR. This becomes necessary when uncoupling the - plugin filesystem as some methods now actually fetches expected files - from BASE_DATA_DIR. This will fail for protocols other than filesystem - IF files are created locally by the plugin test. - - Parameters - ---------- - fps : str or [str] - Filepath or list of filepaths to file(s) that shall be part of - BASE_DATA_DIR, but currently points to some tmp file for testing. - update_fp_only : bool - Some tests operate on filepaths only - files do not actually need - to exist. Thus, we don't need to tranfer a file. - - Returns - ------- - The potentially modified filepaths. - """ - def _stripRoot(fp): - # chop off leading / for join to work properly when prepending - # the BASE_DATA_DIR - if isabs(fp): - return fp[len(sep):] - return fp - - # use artifact 1 info to determine BASA_DATA_DIR, as we know that the - # filepath ends with ....raw_data/1_s_G1_L001_sequences.fastq.gz, thus - # BASE_DATA_DIR must be the prefix, e.g. /qiita_data/ - # This might break IF file - # qiita-spots/qiita/qiita_db/support_files/populate_test_db.sql - # changes. - ainfo = self.qclient.get('/qiita_db/artifacts/1/') - base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ - :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] - if isinstance(fps, str): - if not update_fp_only: - self.qclient.push_file_to_central(fps) - return join(base_data_dir, _stripRoot(fps)) - elif isinstance(fps, list): - for fp in fps: - if not update_fp_only: - self.qclient.push_file_to_central(fp) - return [join(base_data_dir, _stripRoot(fp)) for fp in fps] - else: - raise ValueError( - "deposite_in_qiita_basedir is not implemented for type %s" - % type(fps)) From 01e18ddfed876c26f763a3bd68a19745a36d6bfb Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 16 Nov 2025 11:44:00 +0100 Subject: [PATCH 2/9] add attribute to fake client --- qiita_client/tests/test_plugin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index e6d28ae..1f73d7d 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -75,6 +75,9 @@ def func(a, b, c, d): def test__push_artifacts_files_to_central(self): class fakeClient(): + def __init__(): + self._plugincoupling = 'null protocol' + def push_file_to_central(self, filepath): return 'pushed:%s' % filepath From 267fe7da9bc7d59c6ff968b8542661eee71e96a9 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 16 Nov 2025 11:49:37 +0100 Subject: [PATCH 3/9] fix signature --- qiita_client/tests/test_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index 1f73d7d..f3010c8 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -75,7 +75,7 @@ def func(a, b, c, d): def test__push_artifacts_files_to_central(self): class fakeClient(): - def __init__(): + def __init__(self): self._plugincoupling = 'null protocol' def push_file_to_central(self, filepath): From 6e5ed9129a16b44cc0a9c4937d9d4a6c4e718263 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 17 Nov 2025 07:50:33 +0100 Subject: [PATCH 4/9] added option to turn off automatic file fetching --- qiita_client/qiita_client.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 24cfe80..bf6141f 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -447,7 +447,7 @@ def _fetch_artifact_files(self, ainfo): else: return ainfo - def get(self, url, rettype='json', **kwargs): + def get(self, url, rettype='json', no_file_fetching=False, **kwargs): """Execute a get request against the Qiita server Parameters @@ -457,6 +457,12 @@ def get(self, url, rettype='json', **kwargs): rettype : string The return type of the function, either "json" (default) or "object" for the response object itself + no_file_fetching : bool + If plugin is coupled through none "filesystem" protocols, artifact + files will automatically fetched from Qiita central when requesting + via "/qiita_db/prep_template/" or "/qiita_db/artifacts/". For + testing, you can turn off this behaviour. Handy if e.g. files not + yet exists in Qiita central. kwargs : dict The request kwargs @@ -469,7 +475,8 @@ def get(self, url, rettype='json', **kwargs): result = self._request_retry( self._session.get, url, rettype=rettype, **kwargs) - if self._plugincoupling != 'filesystem': + if (self._plugincoupling != 'filesystem') and \ + (no_file_fetching is False): # intercept get requests from plugins that request metadata or # artifact files and ensure they get transferred from Qiita # central, when not using "filesystem" From a6889082277573d4f79173f372e86bb962460110 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 17 Nov 2025 07:51:56 +0100 Subject: [PATCH 5/9] use a more simple request to determine base_data_dir --- qiita_client/testing.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index 4ca6bac..e4ff381 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -45,15 +45,15 @@ def setUpClass(cls): cls.qclient._plugincoupling = environ.get( 'QIITA_PLUGINCOUPLING', BaseQiitaPlugin._DEFAULT_PLUGIN_COUPLINGS) - # use artifact 1 info to determine BASA_DATA_DIR, as we know that the - # filepath ends with ....raw_data/1_s_G1_L001_sequences.fastq.gz, thus - # BASE_DATA_DIR must be the prefix, e.g. /qiita_data/ + # Determine BASE_DATA_DIR of qiita central, without having direct + # access to qiita's settings file. This is done by requesting + # information about prep 1, which should be in the test database. # This might break IF file # qiita-spots/qiita/qiita_db/support_files/populate_test_db.sql # changes. - ainfo = cls.qclient.get('/qiita_db/artifacts/1/') - cls.base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ - :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] + prep_info = cls.qclient.get('/qiita_db/prep_template/1/', + no_file_fetching=True) + cls.base_data_dir = prep_info['prep-file'].split('templates/')[0] # Give enough time for the plugins to register sleep(5) From 8508a8072eb2f1ecaed10d050d7f235ba888ac06 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 17 Nov 2025 16:07:52 +0100 Subject: [PATCH 6/9] push files when patching artifact summaries --- qiita_client/qiita_client.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index bf6141f..08abd1c 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -12,7 +12,7 @@ import requests import threading import pandas as pd -from json import dumps +from json import dumps, loads from random import randint import fnmatch from io import BytesIO @@ -389,7 +389,8 @@ def _request_retry(self, req, url, rettype='json', **kwargs): elif r.status_code in (500, 405): raise RuntimeError( "Request '%s %s' did not succeed. Status code: %d. " - "Message: %s" % (req.__name__, url, r.status_code, r.text)) + "Message: %s%s" % (req.__name__, url, r.status_code, + r.text, r.reason)) elif 0 <= (r.status_code - 200) < 100: try: if rettype is None or rettype == 'json': @@ -574,6 +575,26 @@ def patch(self, url, op, path, value=None, from_p=None, **kwargs): # we made sure that data is correctly formatted here kwargs['data'] = data + # similar to above get() injection mechanism, we are here pushing files + # to Qiita central, when patching artifact summaries + if (self._plugincoupling != 'filesystem') and \ + (path == '/html_summary/') and (op == 'add'): + if re.search(r"/qiita_db/artifacts/\d+/?$", url): + if value is not None: + logger.debug('QiitaClient::patch: push summary files to' \ + 'central: %s' % value) + try: + # values might be an json encoded dictioary with + # multiple filepaths... + dictValues = loads(value) + for ftype in ['html', 'dir']: + if (ftype in dictValues.keys()) and \ + (dictValues[ftype] is not None): + self.push_file_to_central(dictValues[ftype]) + except TypeError as e: + # or just a single string, i.e. filepath + self.push_file_to_central(value) + return self._request_retry( self._session.patch, url, rettype='json', **kwargs) From 3f38b1306eeba491e6e900436edb28e2f9724dcf Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 17 Nov 2025 16:31:21 +0100 Subject: [PATCH 7/9] codestyle --- qiita_client/qiita_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 08abd1c..696f310 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -581,7 +581,7 @@ def patch(self, url, op, path, value=None, from_p=None, **kwargs): (path == '/html_summary/') and (op == 'add'): if re.search(r"/qiita_db/artifacts/\d+/?$", url): if value is not None: - logger.debug('QiitaClient::patch: push summary files to' \ + logger.debug('QiitaClient::patch: push summary files to' 'central: %s' % value) try: # values might be an json encoded dictioary with @@ -591,7 +591,7 @@ def patch(self, url, op, path, value=None, from_p=None, **kwargs): if (ftype in dictValues.keys()) and \ (dictValues[ftype] is not None): self.push_file_to_central(dictValues[ftype]) - except TypeError as e: + except TypeError: # or just a single string, i.e. filepath self.push_file_to_central(value) From cdbf4e484e729e6699465df458bf7c7113fd806e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 17 Nov 2025 16:54:34 +0100 Subject: [PATCH 8/9] more elaborate test for plain string instead of json dict --- qiita_client/qiita_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 696f310..8ae3320 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -12,7 +12,7 @@ import requests import threading import pandas as pd -from json import dumps, loads +from json import dumps, loads, JSONDecodeError from random import randint import fnmatch from io import BytesIO @@ -581,7 +581,7 @@ def patch(self, url, op, path, value=None, from_p=None, **kwargs): (path == '/html_summary/') and (op == 'add'): if re.search(r"/qiita_db/artifacts/\d+/?$", url): if value is not None: - logger.debug('QiitaClient::patch: push summary files to' + logger.debug('QiitaClient::patch: push summary files to ' 'central: %s' % value) try: # values might be an json encoded dictioary with @@ -591,7 +591,7 @@ def patch(self, url, op, path, value=None, from_p=None, **kwargs): if (ftype in dictValues.keys()) and \ (dictValues[ftype] is not None): self.push_file_to_central(dictValues[ftype]) - except TypeError: + except (TypeError, JSONDecodeError): # or just a single string, i.e. filepath self.push_file_to_central(value) From 2321dbe973df0f17f4151c69ce73fa83afa017b6 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 17 Nov 2025 21:20:05 +0100 Subject: [PATCH 9/9] handle missing py27 JSONDecodeError --- qiita_client/qiita_client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 8ae3320..0ea317a 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -12,7 +12,13 @@ import requests import threading import pandas as pd -from json import dumps, loads, JSONDecodeError +from json import dumps, loads +try: + from json import JSONDecodeError +except ImportError: + # dirty hack to cope with the fact that python 2.7 does not have + # JSONDecodeError, but is needed for qp-target-gene plugin + JSONDecodeError = ValueError from random import randint import fnmatch from io import BytesIO