From bbcab5f27234f6f50e6764de35ec817684e0a7d2 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 17 Sep 2025 09:59:09 +0200 Subject: [PATCH 01/78] change plugin coupling protocol from "filesystem" to other values by environment variable - as config files are not parsed in most test cases --- qiita_client/testing.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index 41b183c..9925c42 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -11,6 +11,7 @@ from time import sleep from qiita_client import QiitaClient +from plugin import BaseQiitaPlugin import logging @@ -36,6 +37,14 @@ def setUpClass(cls): logger.debug( 'PluginTestCase.setUpClass() token %s' % cls.qclient._token) cls.qclient.post('/apitest/reload_plugins/') + + # When testing, we access plugin functions often directly. Plugin + # configuration files are not parsed in these cases. To be able to + # change the plugin coupling protocol, we resort to the environment + # variable here. + cls.qclient._plugincoupling = environ.get( + 'QIITA_PLUGINCOUPLING', BaseQiitaPlugin._DEFAULT_PLUGIN_COUPLINGS) + # Give enough time for the plugins to register sleep(5) From 21736bc0db740f611e956c183d3c3d6c982597f8 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 17 Sep 2025 10:06:42 +0200 Subject: [PATCH 02/78] fix import --- qiita_client/testing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index 9925c42..5e06c37 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -10,8 +10,7 @@ from os import environ from time import sleep -from qiita_client import QiitaClient -from plugin import BaseQiitaPlugin +from qiita_client import QiitaClient, BaseQiitaPlugin import logging From a52d3878e2751258d00128456c9a554e038dc0ac Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 17 Sep 2025 10:13:09 +0200 Subject: [PATCH 03/78] fix import --- qiita_client/testing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index 5e06c37..534ee0b 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -10,7 +10,8 @@ from os import environ from time import sleep -from qiita_client import QiitaClient, BaseQiitaPlugin +from qiita_client import QiitaClient +from qiita_client.plugin import BaseQiitaPlugin import logging From b707937a972b62395dd451f50d60123be8df4643 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 17 Sep 2025 12:56:08 +0200 Subject: [PATCH 04/78] relative import to fix py2 import issue --- qiita_client/testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index 534ee0b..fa8b8e5 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -11,7 +11,7 @@ from time import sleep from qiita_client import QiitaClient -from qiita_client.plugin import BaseQiitaPlugin +from .plugin import BaseQiitaPlugin import logging From 878df24af1b39a495ce8cb79afb93eeb11fdaa08 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 17 Sep 2025 13:01:56 +0200 Subject: [PATCH 05/78] python2 compatible makedirs --- qiita_client/qiita_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 4b34613..d77f9e4 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -784,7 +784,8 @@ def fetch_file_from_central(self, filepath, prefix=None): if self._plugincoupling == 'filesystem': if (prefix is not None) and (prefix != ""): # create necessary directory locally - os.makedirs(os.path.dirname(target_filepath), exist_ok=True) + if not os.path.exists(os.path.dirname(target_filepath)): + os.makedirs(os.path.dirname(target_filepath)) shutil.copyfile(filepath, target_filepath) @@ -803,7 +804,8 @@ def fetch_file_from_central(self, filepath, prefix=None): rettype='content') # create necessary directory locally - os.makedirs(os.path.dirname(target_filepath), exist_ok=True) + if not os.path.exists(os.path.dirname(target_filepath)): + os.makedirs(os.path.dirname(target_filepath)) # write retrieved file content with open(target_filepath, 'wb') as f: From 7dcaa1e99310129a6ad4d70034dd8f4d16d56d3d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 19 Sep 2025 12:06:22 +0200 Subject: [PATCH 06/78] add ability to fetch all files of a given directory (ensure compatibility with old py27 for tests) --- qiita_client/qiita_client.py | 22 ++++++++-- qiita_client/tests/test_qiita_client.py | 56 +++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index d77f9e4..c395c68 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -14,6 +14,7 @@ import pandas as pd from json import dumps from random import randint +import fnmatch try: from itertools import zip_longest @@ -842,16 +843,29 @@ def push_file_to_central(self, filepath): return filepath elif self._plugincoupling == 'https': - logger.debug('Submitting file %s to qiita server.' % filepath) + logger.debug('Submitting %s %s to qiita server.' % ( + 'directory' if os.path.isdir(filepath) else 'file', filepath)) # target path, i.e. without filename dirpath = os.path.dirname(filepath) if dirpath == "": dirpath = "/" - self.post( - '/cloud/push_file_to_central/', - files={dirpath: open(filepath, 'rb')}) + if os.path.isdir(filepath): + # Pushing all files of a directory, not only a single file. + # Cannot use "glob" as it lacks the "recursive" parameter in + # py27. This is used e.g. in qp-target-gene + for root, dirnames, filenames in os.walk(filepath): + for filename in fnmatch.filter(filenames, "*"): + fp = os.path.join(root, filename) + self.post('/cloud/push_file_to_central/', + files={os.path.join( + dirpath, + os.path.dirname(fp)): open(fp, 'rb')}) + else: + self.post( + '/cloud/push_file_to_central/', + files={dirpath: open(filepath, 'rb')}) return filepath diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 15e4f77..3ebbd2c 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -8,11 +8,12 @@ from unittest import TestCase, main import filecmp -from os import remove, close -from os.path import basename, exists, expanduser, join +from os import remove, close, makedirs +from os.path import basename, exists, expanduser, join, isdir from tempfile import mkstemp from json import dumps import pandas as pd +from shutil import rmtree from qiita_client.qiita_client import (QiitaClient, _format_payload, ArtifactInfo) @@ -110,7 +111,10 @@ def setUp(self): def tearDown(self): for fp in self.clean_up_files: if exists(fp): - remove(fp) + if isdir(fp): + rmtree(fp) + else: + remove(fp) def test_init(self): obs = QiitaClient(URL, @@ -184,7 +188,7 @@ def test_patch(self): with open(fp, 'w') as f: f.write('\n') self.clean_up_files.append(fp) - obs = self.tester.patch(f'/qiita_db/artifacts/{artifact_id}/', 'add', + obs = self.tester.patch('/qiita_db/artifacts/%s/' % artifact_id, 'add', '/html_summary/', value=fp) self.assertIsNone(obs) @@ -440,6 +444,50 @@ def test_push_file_to_central(self): fp_obs = self.tester.push_file_to_central(fp_source) self.assertEqual(fp_source, fp_obs) + def _create_test_dir(self, prefix=None): + """Creates a test directory with files and subdirs.""" + # prefix + # |- testdir/ + # |---- fileA.txt + # |---- subdirA_l1/ + # |-------- fileB.fna + # |-------- subdirC_l2/ + # |------------ fileC.log + # |------------ fileD.seq + # |---- subdirB_l1/ + # |-------- fileE.sff + if (prefix is not None) and (prefix != ""): + prefix = join(prefix, 'testdir') + else: + prefix = 'testdir' + + for dir in [join(prefix, 'subdirA_l1', 'subdirC_l2'), + join(prefix, 'subdirB_l1')]: + if not exists(dir): + makedirs(dir) + for file, cont in [(join(prefix, 'fileA.txt'), 'contentA'), + (join(prefix, 'subdirA_l1', + 'fileB.fna'), 'this is B'), + (join(prefix, 'subdirA_l1', 'subdirC_l2', + 'fileC.log'), 'call me c'), + (join(prefix, 'subdirA_l1', 'subdirC_l2', + 'fileD.seq'), 'I d'), + (join(prefix, 'subdirB_l1', 'fileE.sff'), 'oh e')]: + with open(file, "w") as f: + f.write(cont + "\n") + self.clean_up_files.append(prefix) + + return prefix + + def test_push_file_to_central_dir(self): + self.tester._plugincoupling = 'https' + + fp_source = self._create_test_dir('/tmp/test_push_dir/') + fp_obs = self.tester.push_file_to_central(fp_source) + self.assertEqual(fp_source, fp_obs) + # As we don't necessarily know the QIITA_BASE_DIR, we cannot fetch one + # of the files to double check for it's content + if __name__ == '__main__': main() From 296a3e3641f7a2e55a5f5cec1f77bb45dfc88df1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 26 Sep 2025 09:10:12 +0200 Subject: [PATCH 07/78] adding a delete_file_from_central method, which does NOT delete when in https mode --- qiita_client/qiita_client.py | 41 ++++++++++++++++++++++--- qiita_client/tests/test_qiita_client.py | 39 ++++++++++++++++++++++- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index c395c68..82635c8 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -820,20 +820,23 @@ def fetch_file_from_central(self, filepath, prefix=None): "configuration is NOT defined.") % self._plugincoupling) def push_file_to_central(self, filepath): - """Pushs filecontent to Qiita's central BASE_DATA_DIR directory. + """Pushs file- or directory content to Qiita's central BASE_DATA_DIR + directory. By default, plugin and Qiita's central BASE_DATA_DIR filesystems are identical. In this case, no files are touched and the filepath is directly returned. If however, plugincoupling is set to 'https', the content of the file - is sent via https POST to Qiita's master/worker, which has to receive - and store in an appropriate location. + (or content of recursively all files in the given directory) is sent + via https POST to Qiita's master/worker, which has to receive and store + in an appropriate location. Parameters ---------- filepath : str - The filepath of the files whos content shall be send to Qiita's - central BASE_DATA_DIR + The filepath of the file(s) whos content shall be send to Qiita's + central BASE_DATA_DIR. + Can be a path to a directory as well. Returns ------- @@ -872,3 +875,31 @@ def push_file_to_central(self, filepath): raise ValueError( ("File communication protocol '%s' as defined in plugins " "configuration is NOT defined.") % self._plugincoupling) + + def delete_file_from_central(self, filepath): + """Deletes a file in Qiita's central BASE_DATA_DIR directory. + + I currently (2025-09-25) assess this operation to be too dangerous for + protocols other than "filesystem", i.e. on "https" the files are NOT + deleted. + However, this might change in the future and since I don't want to + touch every plugin's code again, I am adding this function here already + and use it in according plugin code locations, e.g. function _gzip_file + in qtp-sequencing. + + Parameters + ---------- + filepath : str + The filepath of the file that shall be deletes in Qiita's + central BASE_DATA_DIR + + Returns + ------- + The given filepath - to be transparent in plugin code. + """ + if self._plugincoupling == 'filesystem': + os.remove(filepath) + elif self._plugincoupling == 'https': + pass + + return filepath diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 3ebbd2c..8cc2c53 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -9,7 +9,7 @@ from unittest import TestCase, main import filecmp from os import remove, close, makedirs -from os.path import basename, exists, expanduser, join, isdir +from os.path import basename, exists, expanduser, join, isdir, dirname from tempfile import mkstemp from json import dumps import pandas as pd @@ -488,6 +488,43 @@ def test_push_file_to_central_dir(self): # As we don't necessarily know the QIITA_BASE_DIR, we cannot fetch one # of the files to double check for it's content + def test_delete_file_from_central(self): + # obtain current filepaths to infer QIITA_BASE_DIR + ainfo = self.tester.get("/qiita_db/artifacts/%s/" % 1) + cwd = dirname(ainfo['files']['raw_forward_seqs'][0]['filepath']) + + for protocol in ['filesystem', 'https']: + self.qclient._plugincoupling = protocol + + # deposit a test file + fp_test = join(cwd, 'deletme_%s.txt' % protocol) + makedirs(cwd, exist_ok=True) + with open(fp_test, 'w') as f: + f.write('This is a testfile content\n') + self.clean_up_files.append(fp_test) + self.qclient.push_file_to_central(fp_test) + + # sanity check that test file has been deposited correctly + fp_obs = self.qclient.fetch_file_from_central(fp_test) + self.assertTrue(exists(fp_obs)) + + # delete file and test if it is gone + fp_deleted = self.qclient.delete_file_from_central(fp_test) + if protocol == 'filesystem': + # all three fp should point to the same filepath + self.assertFalse(exists(fp_obs)) + self.assertFalse(exists(fp_test)) + self.assertFalse(exists(fp_deleted)) + elif protocol == 'https': + # as of 2025-09-26, I don't allow deletion of qiita main files + # through API endpoints. Thus, the file is NOT deleted! + # local version of the file + self.assertTrue(exists(fp_test)) + # qiita main filepath + self.assertTrue(exists(fp_obs)) + # qiita main filepath, returned by delete_file_from_central + self.assertTrue(exists(fp_deleted)) + if __name__ == '__main__': main() From 41fa3516b81ca6bcb7b542aa3882e120d88040a0 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 26 Sep 2025 09:24:41 +0200 Subject: [PATCH 08/78] avoid error when file already present --- qiita_client/tests/test_qiita_client.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 8cc2c53..393cd2e 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -18,7 +18,7 @@ from qiita_client.qiita_client import (QiitaClient, _format_payload, ArtifactInfo) from qiita_client.testing import PluginTestCase, URL -from qiita_client.exceptions import BadRequestError +from qiita_client.exceptions import BadRequestError, ForbiddenError CLIENT_ID = '19ndkO3oMKsoChjVVWluF7QkxHRfYhTKSFbAVt8IhK7gZgDaO4' BAD_CLIENT_ID = 'NOT_A_CLIENT_ID' @@ -497,12 +497,17 @@ def test_delete_file_from_central(self): self.qclient._plugincoupling = protocol # deposit a test file - fp_test = join(cwd, 'deletme_%s.txt' % protocol) + fp_test = join(cwd, 'deleteme_%s.txt' % protocol) makedirs(cwd, exist_ok=True) with open(fp_test, 'w') as f: f.write('This is a testfile content\n') self.clean_up_files.append(fp_test) - self.qclient.push_file_to_central(fp_test) + try: + self.qclient.push_file_to_central(fp_test) + except ForbiddenError as e: + # previous test instances might not have cleaned this file + # properly if ran with https protocol + pass # sanity check that test file has been deposited correctly fp_obs = self.qclient.fetch_file_from_central(fp_test) From 07efd6419520ba44359a0ae85387f80bdccd93ff Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 26 Sep 2025 09:27:21 +0200 Subject: [PATCH 09/78] remove unused variable name --- qiita_client/tests/test_qiita_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 393cd2e..b432278 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -504,7 +504,7 @@ def test_delete_file_from_central(self): self.clean_up_files.append(fp_test) try: self.qclient.push_file_to_central(fp_test) - except ForbiddenError as e: + except ForbiddenError: # previous test instances might not have cleaned this file # properly if ran with https protocol pass From 5ac2380df00e81c3ecaa3723ec575b82e45af4dd Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 26 Sep 2025 09:51:37 +0200 Subject: [PATCH 10/78] more clean test file removal --- qiita_client/tests/test_qiita_client.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index b432278..0bdfdf2 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -502,12 +502,7 @@ def test_delete_file_from_central(self): with open(fp_test, 'w') as f: f.write('This is a testfile content\n') self.clean_up_files.append(fp_test) - try: - self.qclient.push_file_to_central(fp_test) - except ForbiddenError: - # previous test instances might not have cleaned this file - # properly if ran with https protocol - pass + self.qclient.push_file_to_central(fp_test) # sanity check that test file has been deposited correctly fp_obs = self.qclient.fetch_file_from_central(fp_test) @@ -530,6 +525,12 @@ def test_delete_file_from_central(self): # qiita main filepath, returned by delete_file_from_central self.assertTrue(exists(fp_deleted)) + # clean up test file directly after tests have passed + if exists(fp_test): + remove(fp_test) + if exists(fp_obs): + remove(fp_obs) + if __name__ == '__main__': main() From 216b22c7b39abec796678dfc897ed50bb3727dc5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 26 Sep 2025 10:06:50 +0200 Subject: [PATCH 11/78] account for having direct access to qiita main in tests --- qiita_client/tests/test_qiita_client.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 0bdfdf2..4768ad8 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -502,9 +502,10 @@ def test_delete_file_from_central(self): with open(fp_test, 'w') as f: f.write('This is a testfile content\n') self.clean_up_files.append(fp_test) - self.qclient.push_file_to_central(fp_test) # sanity check that test file has been deposited correctly + # no push required, as in this test local and remote QIITA_BASE_DIR + # is identical fp_obs = self.qclient.fetch_file_from_central(fp_test) self.assertTrue(exists(fp_obs)) @@ -525,12 +526,6 @@ def test_delete_file_from_central(self): # qiita main filepath, returned by delete_file_from_central self.assertTrue(exists(fp_deleted)) - # clean up test file directly after tests have passed - if exists(fp_test): - remove(fp_test) - if exists(fp_obs): - remove(fp_obs) - if __name__ == '__main__': main() From 4dfb5928affe36b1e904f198fa3789e6e3afd709 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 26 Sep 2025 10:08:03 +0200 Subject: [PATCH 12/78] codestyle --- qiita_client/tests/test_qiita_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 4768ad8..1d8e66c 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -18,7 +18,7 @@ from qiita_client.qiita_client import (QiitaClient, _format_payload, ArtifactInfo) from qiita_client.testing import PluginTestCase, URL -from qiita_client.exceptions import BadRequestError, ForbiddenError +from qiita_client.exceptions import BadRequestError CLIENT_ID = '19ndkO3oMKsoChjVVWluF7QkxHRfYhTKSFbAVt8IhK7gZgDaO4' BAD_CLIENT_ID = 'NOT_A_CLIENT_ID' From 464c6c655d05a5fc815173fb25a16e5236fa9e52 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 3 Nov 2025 15:33:07 +0100 Subject: [PATCH 13/78] be more verbose in debug mode --- qiita_client/qiita_client.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 82635c8..0d2feb9 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -790,6 +790,10 @@ def fetch_file_from_central(self, filepath, prefix=None): shutil.copyfile(filepath, target_filepath) + logger.debug( + 'Fetching file "%s" via protocol=%s from Qiita main.' % ( + filepath, self._plugincoupling)) + return target_filepath elif self._plugincoupling == 'https': @@ -812,6 +816,10 @@ def fetch_file_from_central(self, filepath, prefix=None): with open(target_filepath, 'wb') as f: f.write(content) + logger.debug( + 'Fetching file "%s" via protocol=%s from Qiita main.' % ( + filepath, self._plugincoupling)) + return target_filepath else: @@ -820,7 +828,7 @@ def fetch_file_from_central(self, filepath, prefix=None): "configuration is NOT defined.") % self._plugincoupling) def push_file_to_central(self, filepath): - """Pushs file- or directory content to Qiita's central BASE_DATA_DIR + """Pushs file- or directory content to Qiita's central BASE_DATA_DIR directory. By default, plugin and Qiita's central BASE_DATA_DIR filesystems are From b12d74003e5bcd6d5c88c40fb48acb4118407753 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 3 Nov 2025 15:36:00 +0100 Subject: [PATCH 14/78] generally provide a _fix_plugincoupling_filepath function for plugin tests --- qiita_client/testing.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index fa8b8e5..9bc3331 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -7,7 +7,8 @@ # ----------------------------------------------------------------------------- from unittest import TestCase -from os import environ +from os import environ, sep +from os.path import isabs, join, exists from time import sleep from qiita_client import QiitaClient @@ -82,3 +83,30 @@ def _wait_for_running_job(self, job_id): break return status + + def _fix_plugincoupling_filepath(self, fp, BASE_DATA_DIR="/qiita_data/"): + # In some plugin tests, example files are generated temporarily on + # local tmp directories. This is fine for plugincoupling == filesystem + # but will cause "File not found errors" when communication through + # other protocols as the file actually has never been pushed to Qiita + # main. This helper function shall fix this by copying the according + # files over to Qiita main WHEN not using "filesystem" and fix the + # according file paths, i.e. we need to prepend the BASE_DATA_DIR to + # the filename, which should be '/qiita_data/' e.g. here + # https://github.com/jlab/qiita-keycloak + if not exists(fp): + raise ValueError("Filepath does not exist!") + + if self.qclient._plugincoupling == 'filesystem': + return fp + else: + processed_fp = fp + # chop off leading / for join to work properly when prepending + # the BASE_DATA_DIR + if isabs(processed_fp): + processed_fp = processed_fp[len(sep):] + processed_fp = join(BASE_DATA_DIR, processed_fp) + # ensure file is transferred to qiita main + self.qclient.push_file_to_central(fp) + # return the filepath prepended with qiita main base_data_dir + return processed_fp From 816c4f39c5a64610ce78e4c0534e1de25f86c6da Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 3 Nov 2025 15:56:40 +0100 Subject: [PATCH 15/78] unified logging --- qiita_client/qiita_client.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 0d2feb9..c0c49c7 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -774,6 +774,10 @@ def fetch_file_from_central(self, filepath, prefix=None): str : the filepath of the requested file within the local file system """ target_filepath = filepath + logger.debug( + 'Fetching file "%s" via protocol=%s from Qiita main.' % ( + filepath, self._plugincoupling)) + if (prefix is not None) and (prefix != ""): # strip off root if filepath.startswith(os.path.abspath(os.sep)): @@ -790,10 +794,6 @@ def fetch_file_from_central(self, filepath, prefix=None): shutil.copyfile(filepath, target_filepath) - logger.debug( - 'Fetching file "%s" via protocol=%s from Qiita main.' % ( - filepath, self._plugincoupling)) - return target_filepath elif self._plugincoupling == 'https': @@ -801,8 +801,6 @@ def fetch_file_from_central(self, filepath, prefix=None): if filepath.startswith(os.path.abspath(os.sep)): filepath = filepath[len(os.path.abspath(os.sep)):] - logger.debug('Requesting file %s from qiita server.' % filepath) - # actual call to Qiita central to obtain file content content = self.get( '/cloud/fetch_file_from_central/' + filepath, @@ -816,10 +814,6 @@ def fetch_file_from_central(self, filepath, prefix=None): with open(target_filepath, 'wb') as f: f.write(content) - logger.debug( - 'Fetching file "%s" via protocol=%s from Qiita main.' % ( - filepath, self._plugincoupling)) - return target_filepath else: From d42b53496b6704b91c2c1b905ebc9ad205095fd6 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 3 Nov 2025 16:19:16 +0100 Subject: [PATCH 16/78] a more versatile function --- qiita_client/testing.py | 65 ++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index 9bc3331..8afbe5e 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -8,7 +8,7 @@ from unittest import TestCase from os import environ, sep -from os.path import isabs, join, exists +from os.path import join, relpath from time import sleep from qiita_client import QiitaClient @@ -84,29 +84,46 @@ def _wait_for_running_job(self, job_id): return status - def _fix_plugincoupling_filepath(self, fp, BASE_DATA_DIR="/qiita_data/"): - # In some plugin tests, example files are generated temporarily on - # local tmp directories. This is fine for plugincoupling == filesystem - # but will cause "File not found errors" when communication through - # other protocols as the file actually has never been pushed to Qiita - # main. This helper function shall fix this by copying the according - # files over to Qiita main WHEN not using "filesystem" and fix the - # according file paths, i.e. we need to prepend the BASE_DATA_DIR to - # the filename, which should be '/qiita_data/' e.g. here - # https://github.com/jlab/qiita-keycloak - if not exists(fp): - raise ValueError("Filepath does not exist!") + def deposite_in_qiita_basedir(self, fps, update_fp_only=False): + """Pushs a file to qiita main AND adapts given filepath accordingly. + A helper function to fix file paths in tests such that they point to the + expected BASE_DATA_DIR. This becomes necessary when uncoupling the + plugin filesystem as some methods now actually fetches expected files + from BASE_DATA_DIR. This will fail for protocols other than filesystem + IF files are created locally by the plugin test. + + Parameters + ---------- + fps : str or [str] + Filepath or list of filepaths to file(s) that shall be part of + BASE_DATA_DIR, but currently points to some tmp file for testing. + update_fp_only : bool + Some tests operate on filepaths only - files do not actually need to + exist. Thus, we don't need to tranfer a file. + """ if self.qclient._plugincoupling == 'filesystem': - return fp + return fps + + # use artifact 1 info to determine BASA_DATA_DIR, as we know that the + # filepath ends with ....raw_data/1_s_G1_L001_sequences.fastq.gz, thus + # BASE_DATA_DIR must be the prefix, e.g. /qiita_data/ + # This might break IF file + # qiita-spots/qiita/qiita_db/support_files/populate_test_db.sql + # changes. + ainfo = self.qclient.get('/qiita_db/artifacts/1/') + base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ + :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] + if isinstance(fps, str): + if not update_fp_only: + self.qclient.push_file_to_central(fps) + return join(base_data_dir, relpath(fps, sep)) + elif isinstance(fps, list): + for fp in fps: + if not update_fp_only: + self.qclient.push_file_to_central(fp) + return [join(base_data_dir, relpath(fp, sep)) for fp in fps] else: - processed_fp = fp - # chop off leading / for join to work properly when prepending - # the BASE_DATA_DIR - if isabs(processed_fp): - processed_fp = processed_fp[len(sep):] - processed_fp = join(BASE_DATA_DIR, processed_fp) - # ensure file is transferred to qiita main - self.qclient.push_file_to_central(fp) - # return the filepath prepended with qiita main base_data_dir - return processed_fp + raise ValueError( + "_deposite_in_qiita_basedir is not implemented for type %s" + % type(fps)) From 10780a27729f36d40602a7f564f691c1ebdf4aa9 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 3 Nov 2025 16:35:05 +0100 Subject: [PATCH 17/78] codestyle --- qiita_client/testing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index 8afbe5e..ea26717 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -87,8 +87,8 @@ def _wait_for_running_job(self, job_id): def deposite_in_qiita_basedir(self, fps, update_fp_only=False): """Pushs a file to qiita main AND adapts given filepath accordingly. - A helper function to fix file paths in tests such that they point to the - expected BASE_DATA_DIR. This becomes necessary when uncoupling the + A helper function to fix file paths in tests such that they point to + the expected BASE_DATA_DIR. This becomes necessary when uncoupling the plugin filesystem as some methods now actually fetches expected files from BASE_DATA_DIR. This will fail for protocols other than filesystem IF files are created locally by the plugin test. @@ -99,8 +99,8 @@ def deposite_in_qiita_basedir(self, fps, update_fp_only=False): Filepath or list of filepaths to file(s) that shall be part of BASE_DATA_DIR, but currently points to some tmp file for testing. update_fp_only : bool - Some tests operate on filepaths only - files do not actually need to - exist. Thus, we don't need to tranfer a file. + Some tests operate on filepaths only - files do not actually need + to exist. Thus, we don't need to tranfer a file. """ if self.qclient._plugincoupling == 'filesystem': return fps From bcbe53d9ac8685cae582dabe6f1eb525caf8621a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 3 Nov 2025 17:26:31 +0100 Subject: [PATCH 18/78] avoid computation from / and instead just trim away --- qiita_client/testing.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index ea26717..67ca68e 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -8,7 +8,7 @@ from unittest import TestCase from os import environ, sep -from os.path import join, relpath +from os.path import join, isabs from time import sleep from qiita_client import QiitaClient @@ -101,7 +101,18 @@ def deposite_in_qiita_basedir(self, fps, update_fp_only=False): update_fp_only : bool Some tests operate on filepaths only - files do not actually need to exist. Thus, we don't need to tranfer a file. + + Returns + ------- + The potentially modified filepaths. """ + def _stripRoot(fp): + # chop off leading / for join to work properly when prepending + # the BASE_DATA_DIR + if isabs(fp): + return fp[len(sep):] + return fp + if self.qclient._plugincoupling == 'filesystem': return fps @@ -117,13 +128,13 @@ def deposite_in_qiita_basedir(self, fps, update_fp_only=False): if isinstance(fps, str): if not update_fp_only: self.qclient.push_file_to_central(fps) - return join(base_data_dir, relpath(fps, sep)) + return join(base_data_dir, _stripRoot(fps)) elif isinstance(fps, list): for fp in fps: if not update_fp_only: self.qclient.push_file_to_central(fp) - return [join(base_data_dir, relpath(fp, sep)) for fp in fps] + return [join(base_data_dir, _stripRoot(fp)) for fp in fps] else: raise ValueError( - "_deposite_in_qiita_basedir is not implemented for type %s" + "deposite_in_qiita_basedir is not implemented for type %s" % type(fps)) From 4d6ab0df5e5a22e519537492dbeae7577187d61d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 13:05:37 +0100 Subject: [PATCH 19/78] fetching of whole directories --- qiita_client/qiita_client.py | 63 ++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index c0c49c7..1c6e5fc 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -15,6 +15,9 @@ from json import dumps from random import randint import fnmatch +from io import BytesIO +from zipfile import ZipFile + try: from itertools import zip_longest @@ -269,7 +272,7 @@ def _request_oauth2(self, req, rettype, *args, **kwargs): The request to execute rettype : string The return type of the function, either "json" or - if e.g. files are transferred "content" + "object" for the response object itself args : tuple The request args kwargs : dict @@ -328,7 +331,7 @@ def _request_retry(self, req, url, rettype='json', **kwargs): The request to execute rettype : string The return type of the function, either "json" (default) or - if e.g. files are transferred "content" + "object" for the response object itself url : str The url to access in the server kwargs : dict @@ -336,7 +339,7 @@ def _request_retry(self, req, url, rettype='json', **kwargs): Returns ------- - dict or None or plain content IF rettype='content' + dict or None or response object IF rettype='object' The JSON information in the request response, if any Raises @@ -391,13 +394,13 @@ def _request_retry(self, req, url, rettype='json', **kwargs): if rettype is None or rettype == 'json': return r.json() else: - if rettype == 'content': - return r.content + if rettype == 'object': + return r else: raise ValueError( ("return type rettype='%s' cannot be " "understand. Choose from 'json' (default) " - "or 'content!") % rettype) + "or 'object!") % rettype) except ValueError: return None stime = randint(MIN_TIME_SLEEP, MAX_TIME_SLEEP) @@ -418,7 +421,7 @@ def get(self, url, rettype='json', **kwargs): The url to access in the server rettype : string The return type of the function, either "json" (default) or - if e.g. files are transferred "content" + "object" for the response object itself kwargs : dict The request kwargs @@ -746,8 +749,8 @@ def _process_files_per_sample_fastq(self, files, prep_info, return sample_names, prep_info def fetch_file_from_central(self, filepath, prefix=None): - """Moves content of a file from Qiita's central BASE_DATA_DIR to a - local plugin file-system. + """Moves content of a file or directory from Qiita's central + BASE_DATA_DIR to a local plugin file-system. By default, this is exactly the same location, i.e. the return filepath is identical to the requested one and nothing is moved / @@ -760,22 +763,24 @@ def fetch_file_from_central(self, filepath, prefix=None): ---------- filepath : str The filepath in Qiita's central BASE_DATA_DIR to the requested - file content + file or directory content prefix : str Primarily for testing: prefix the target filepath with this filepath prefix to - a) in 'filesystem' mode: create an actual file copy (for testing) + a) in 'filesystem' mode: create an actual file/directiry copy + (for testing) If prefix=None, nothing will be copied/moved - b) in 'https' mode: flexibility to locate files differently in - plugin local file system. + b) in 'https' mode: flexibility to locate files/directories + differently in plugin local file system. Returns ------- - str : the filepath of the requested file within the local file system + str : the filepath of the requested file or directory within the local + file system """ target_filepath = filepath logger.debug( - 'Fetching file "%s" via protocol=%s from Qiita main.' % ( + 'Fetching file/directory "%s" via protocol=%s from Qiita main.' % ( filepath, self._plugincoupling)) if (prefix is not None) and (prefix != ""): @@ -792,7 +797,10 @@ def fetch_file_from_central(self, filepath, prefix=None): if not os.path.exists(os.path.dirname(target_filepath)): os.makedirs(os.path.dirname(target_filepath)) - shutil.copyfile(filepath, target_filepath) + if os.path.isdir(filepath): + shutil.copytree(filepath, target_filepath) + else: + shutil.copyfile(filepath, target_filepath) return target_filepath @@ -802,17 +810,24 @@ def fetch_file_from_central(self, filepath, prefix=None): filepath = filepath[len(os.path.abspath(os.sep)):] # actual call to Qiita central to obtain file content - content = self.get( + response = self.get( '/cloud/fetch_file_from_central/' + filepath, - rettype='content') + rettype='object') + + # check if requested filepath is a single file OR a whole directory + if 'Is-Qiita-Directory' in response.headers.keys(): + with ZipFile(BytesIO(response.content)) as zf: + zf.extractall(path=target_filepath) + else: + content = response.content - # create necessary directory locally - if not os.path.exists(os.path.dirname(target_filepath)): - os.makedirs(os.path.dirname(target_filepath)) + # create necessary directory locally + if not os.path.exists(os.path.dirname(target_filepath)): + os.makedirs(os.path.dirname(target_filepath)) - # write retrieved file content - with open(target_filepath, 'wb') as f: - f.write(content) + # write retrieved file content + with open(target_filepath, 'wb') as f: + f.write(content) return target_filepath From dae77651b03cb9c03b78290f35be71ce2aca2289 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 15:08:07 +0100 Subject: [PATCH 20/78] adding a test for directory fetching --- qiita_client/tests/test_qiita_client.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 1d8e66c..64d7c53 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -14,6 +14,7 @@ from json import dumps import pandas as pd from shutil import rmtree +from collections import namedtuple from qiita_client.qiita_client import (QiitaClient, _format_payload, ArtifactInfo) @@ -526,6 +527,25 @@ def test_delete_file_from_central(self): # qiita main filepath, returned by delete_file_from_central self.assertTrue(exists(fp_deleted)) + def test_fetch_directory(self): + # creating a test directory + fp_test = join('/job', '2_test_folder', 'source') + self._create_test_dir(prefix=fp_test) + + # transmitting test directory into qiita main + self.tester._plugincoupling = 'https' + fakeTest = namedtuple("fakeTest", "qclient") + fakeTest.qclient = self.tester + fp_main = PluginTestCase.deposite_in_qiita_basedir(fakeTest, fp_test) + + # fetch test directory from qiita main, this time storing it at + # QIITA_BASE_DIR + fp_obs = self.tester.fetch_file_from_central(dirname(fp_main)) + # test a file of the freshly transferred directory from main has + # expected file content + with open(join(fp_obs, 'source', 'testdir', 'fileA.txt'), 'r') as f: + self.assertIn('contentA', '\n'.join(f.readlines())) + if __name__ == '__main__': main() From 296e138a86b05c906764804292f113bc1af25ad1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 15:18:18 +0100 Subject: [PATCH 21/78] using a local path --- qiita_client/tests/test_qiita_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 64d7c53..94d6c5f 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -529,7 +529,7 @@ def test_delete_file_from_central(self): def test_fetch_directory(self): # creating a test directory - fp_test = join('/job', '2_test_folder', 'source') + fp_test = join('./job', '2_test_folder', 'source') self._create_test_dir(prefix=fp_test) # transmitting test directory into qiita main From 821d5a1b99e89a51565b2052b29f3b5ceb4ae030 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 15:39:58 +0100 Subject: [PATCH 22/78] debug --- qiita_client/tests/test_qiita_client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 94d6c5f..73b0812 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -546,6 +546,10 @@ def test_fetch_directory(self): with open(join(fp_obs, 'source', 'testdir', 'fileA.txt'), 'r') as f: self.assertIn('contentA', '\n'.join(f.readlines())) + import sys + print(">>>>>>>>>>>>", fp_test, fp_main, fp_obs, file=sys.stderr) + print(">>>>>>>>>>>>", fp_test, fp_main, fp_obs) + if __name__ == '__main__': main() From bf327a7da4a43100909a1c84c42ae21be215e628 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 15:48:34 +0100 Subject: [PATCH 23/78] more debugging --- qiita_client/tests/test_qiita_client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 73b0812..ac17b3d 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -528,8 +528,10 @@ def test_delete_file_from_central(self): self.assertTrue(exists(fp_deleted)) def test_fetch_directory(self): + import sys # creating a test directory fp_test = join('./job', '2_test_folder', 'source') + print(">>>>>A>>>>>>>", fp_test, file=sys.stderr) self._create_test_dir(prefix=fp_test) # transmitting test directory into qiita main @@ -537,16 +539,17 @@ def test_fetch_directory(self): fakeTest = namedtuple("fakeTest", "qclient") fakeTest.qclient = self.tester fp_main = PluginTestCase.deposite_in_qiita_basedir(fakeTest, fp_test) - + print(">>>>>B>>>>>>>", fp_main, file=sys.stderr) + # fetch test directory from qiita main, this time storing it at # QIITA_BASE_DIR + print(">>>>>C>>>>>>>", dirname(fp_main), file=sys.stderr) fp_obs = self.tester.fetch_file_from_central(dirname(fp_main)) # test a file of the freshly transferred directory from main has # expected file content with open(join(fp_obs, 'source', 'testdir', 'fileA.txt'), 'r') as f: self.assertIn('contentA', '\n'.join(f.readlines())) - import sys print(">>>>>>>>>>>>", fp_test, fp_main, fp_obs, file=sys.stderr) print(">>>>>>>>>>>>", fp_test, fp_main, fp_obs) From 74a8dfff9e3dbe24d3435899e8780c1b4ab1da8d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 16:12:28 +0100 Subject: [PATCH 24/78] more debug --- qiita_client/tests/test_qiita_client.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index ac17b3d..8caa65a 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -544,6 +544,17 @@ def test_fetch_directory(self): # fetch test directory from qiita main, this time storing it at # QIITA_BASE_DIR print(">>>>>C>>>>>>>", dirname(fp_main), file=sys.stderr) + import os + with open(os.environ['NGINX_FILE_NEW'], 'r') as f: + for l in f.readlines(): + print(">>>>>>>>>>>> NGINX >>> %s" % l, file=sys.stderr) + with open(os.environ['QIITA_CONFIG_FP'], 'r') as f: + for l in f.readlines(): + print(">>>>>>>>>>>> Q-Conf >>> %s" % l, file=sys.stderr) + from glob import glob + for f in glob('%s/**/*' % fp_main, recursive=True): + print(">>>>>>>>>>>> files >>> %s" % f, file=sys.stderr) + fp_obs = self.tester.fetch_file_from_central(dirname(fp_main)) # test a file of the freshly transferred directory from main has # expected file content @@ -556,3 +567,9 @@ def test_fetch_directory(self): if __name__ == '__main__': main() + + +>>>>>>>>>>>> ./job/2_test_folder/source /qiita_data/./job/2_test_folder/source /qiita_data/./job/2_test_folder +2025-11-06T14:51:59.6108533Z >>>>>A>>>>>>> ./job/2_test_folder/source +2025-11-06T14:51:59.6655140Z >>>>>B>>>>>>> /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/./job/2_test_folder/source +2025-11-06T14:51:59.6656458Z >>>>>C>>>>>>> /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/./job/2_test_folder From 138d6a329073a15edc137206b0c198b1495df352 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 16:15:35 +0100 Subject: [PATCH 25/78] remove infos --- qiita_client/tests/test_qiita_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 8caa65a..fa57e00 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -569,7 +569,7 @@ def test_fetch_directory(self): main() ->>>>>>>>>>>> ./job/2_test_folder/source /qiita_data/./job/2_test_folder/source /qiita_data/./job/2_test_folder -2025-11-06T14:51:59.6108533Z >>>>>A>>>>>>> ./job/2_test_folder/source -2025-11-06T14:51:59.6655140Z >>>>>B>>>>>>> /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/./job/2_test_folder/source -2025-11-06T14:51:59.6656458Z >>>>>C>>>>>>> /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/./job/2_test_folder +# >>>>>>>>>>>> ./job/2_test_folder/source /qiita_data/./job/2_test_folder/source /qiita_data/./job/2_test_folder +# 2025-11-06T14:51:59.6108533Z >>>>>A>>>>>>> ./job/2_test_folder/source +# 2025-11-06T14:51:59.6655140Z >>>>>B>>>>>>> /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/./job/2_test_folder/source +# 2025-11-06T14:51:59.6656458Z >>>>>C>>>>>>> /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/./job/2_test_folder From 3c086c86b7027fe07d93fdd80b26b7563c5bfa52 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 16:51:23 +0100 Subject: [PATCH 26/78] explicit file names --- qiita_client/tests/test_qiita_client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index fa57e00..a6865a8 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -545,14 +545,15 @@ def test_fetch_directory(self): # QIITA_BASE_DIR print(">>>>>C>>>>>>>", dirname(fp_main), file=sys.stderr) import os - with open(os.environ['NGINX_FILE_NEW'], 'r') as f: + + with open("/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_pet/nginx_example_local.conf", 'r') as f: for l in f.readlines(): print(">>>>>>>>>>>> NGINX >>> %s" % l, file=sys.stderr) - with open(os.environ['QIITA_CONFIG_FP'], 'r') as f: + with open('/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_core/support_files/config_test_local.cfg', 'r') as f: for l in f.readlines(): print(">>>>>>>>>>>> Q-Conf >>> %s" % l, file=sys.stderr) from glob import glob - for f in glob('%s/**/*' % fp_main, recursive=True): + for f in glob('/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/**/*', recursive=True): print(">>>>>>>>>>>> files >>> %s" % f, file=sys.stderr) fp_obs = self.tester.fetch_file_from_central(dirname(fp_main)) From 9b54be336c890891f854b61007589b26e9f79028 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 20:36:54 +0100 Subject: [PATCH 27/78] avoid access to PluginTestCase --- qiita_client/tests/test_qiita_client.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index a6865a8..bd79013 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -535,10 +535,22 @@ def test_fetch_directory(self): self._create_test_dir(prefix=fp_test) # transmitting test directory into qiita main + # self.tester._plugincoupling = 'https' + # fakeTest = namedtuple("fakeTest", "qclient") + # fakeTest.qclient = self.tester + # fp_main = PluginTestCase.deposite_in_qiita_basedir(fakeTest, fp_test) self.tester._plugincoupling = 'https' - fakeTest = namedtuple("fakeTest", "qclient") - fakeTest.qclient = self.tester - fp_main = PluginTestCase.deposite_in_qiita_basedir(fakeTest, fp_test) + self.tester.push_file_to_central(fp_test) + # a bit hacky, but should work as long as test database does not change + ainfo = self.qclient.get('/qiita_db/artifacts/1/') + base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ + :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] + fp_main = join(base_data_dir, fp_test) + + # fakeTest = namedtuple("fakeTest", "qclient") + # fakeTest.qclient = self.tester + # fp_main = PluginTestCase.deposite_in_qiita_basedir(fakeTest, fp_test) + print(">>>>>B>>>>>>>", fp_main, file=sys.stderr) # fetch test directory from qiita main, this time storing it at @@ -556,6 +568,7 @@ def test_fetch_directory(self): for f in glob('/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/**/*', recursive=True): print(">>>>>>>>>>>> files >>> %s" % f, file=sys.stderr) + fp_obs = self.tester.fetch_file_from_central(dirname(fp_main)) # test a file of the freshly transferred directory from main has # expected file content From bfe86f8387f00745f09ffbbd396b1ada96402587 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 20:43:13 +0100 Subject: [PATCH 28/78] test with prefix --- qiita_client/tests/test_qiita_client.py | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index bd79013..cd8ab97 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -558,22 +558,22 @@ def test_fetch_directory(self): print(">>>>>C>>>>>>>", dirname(fp_main), file=sys.stderr) import os - with open("/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_pet/nginx_example_local.conf", 'r') as f: - for l in f.readlines(): - print(">>>>>>>>>>>> NGINX >>> %s" % l, file=sys.stderr) - with open('/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_core/support_files/config_test_local.cfg', 'r') as f: - for l in f.readlines(): - print(">>>>>>>>>>>> Q-Conf >>> %s" % l, file=sys.stderr) - from glob import glob - for f in glob('/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/**/*', recursive=True): - print(">>>>>>>>>>>> files >>> %s" % f, file=sys.stderr) - - - fp_obs = self.tester.fetch_file_from_central(dirname(fp_main)) - # test a file of the freshly transferred directory from main has - # expected file content - with open(join(fp_obs, 'source', 'testdir', 'fileA.txt'), 'r') as f: - self.assertIn('contentA', '\n'.join(f.readlines())) + # with open("/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_pet/nginx_example_local.conf", 'r') as f: + # for l in f.readlines(): + # print(">>>>>>>>>>>> NGINX >>> %s" % l, file=sys.stderr) + # with open('/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_core/support_files/config_test_local.cfg', 'r') as f: + # for l in f.readlines(): + # print(">>>>>>>>>>>> Q-Conf >>> %s" % l, file=sys.stderr) + # from glob import glob + # for f in glob('/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/**/*', recursive=True): + # print(">>>>>>>>>>>> files >>> %s" % f, file=sys.stderr) + + prefix = join(expanduser("~"), 'karl') + fp_obs = self.tester.fetch_file_from_central(dirname(fp_main), prefix=prefix) + # # test a file of the freshly transferred directory from main has + # # expected file content + # with open(join(fp_obs, 'source', 'testdir', 'fileA.txt'), 'r') as f: + # self.assertIn('contentA', '\n'.join(f.readlines())) print(">>>>>>>>>>>>", fp_test, fp_main, fp_obs, file=sys.stderr) print(">>>>>>>>>>>>", fp_test, fp_main, fp_obs) From f70bd867c10b64012b6aac13e07fba34a0d9a6bc Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 20:51:22 +0100 Subject: [PATCH 29/78] remove ./ --- qiita_client/tests/test_qiita_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index cd8ab97..0020603 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -14,7 +14,7 @@ from json import dumps import pandas as pd from shutil import rmtree -from collections import namedtuple +from pathlib import Path from qiita_client.qiita_client import (QiitaClient, _format_payload, ArtifactInfo) @@ -545,7 +545,7 @@ def test_fetch_directory(self): ainfo = self.qclient.get('/qiita_db/artifacts/1/') base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] - fp_main = join(base_data_dir, fp_test) + fp_main = join(base_data_dir, join(*Path(fp_test).parts)) # fakeTest = namedtuple("fakeTest", "qclient") # fakeTest.qclient = self.tester From d3709d929bfb283e8f27dd6229cb4f26a92e6d6a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 23:01:46 +0100 Subject: [PATCH 30/78] debug --- qiita_client/tests/test_qiita_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 0020603..23f02ce 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -569,6 +569,7 @@ def test_fetch_directory(self): # print(">>>>>>>>>>>> files >>> %s" % f, file=sys.stderr) prefix = join(expanduser("~"), 'karl') + print(">>>>>server url>>>>>>>", self.tester._server_url, file=sys.stderr) fp_obs = self.tester.fetch_file_from_central(dirname(fp_main), prefix=prefix) # # test a file of the freshly transferred directory from main has # # expected file content From 405f52ecef7a352d36213f8a49f77a89fddd8b25 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 23:05:21 +0100 Subject: [PATCH 31/78] check nginx --- .github/workflows/qiita-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index d3c6558..97917c4 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -106,6 +106,7 @@ jobs: export NGINX_FILE_NEW=`pwd`/qiita-dev/qiita_pet/nginx_example_local.conf sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE} > ${NGINX_FILE_NEW} sed -i "s#/Users/username/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE_NEW} + nginx -V >&2 nginx -c ${NGINX_FILE_NEW} echo "3. Setting up qiita" From 09a43ca65a435f7c9076678c23aa9c2b43e66065 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Nov 2025 23:11:33 +0100 Subject: [PATCH 32/78] manually switch to tornado --- qiita_client/tests/test_qiita_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 23f02ce..6a1ad58 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -569,6 +569,7 @@ def test_fetch_directory(self): # print(">>>>>>>>>>>> files >>> %s" % f, file=sys.stderr) prefix = join(expanduser("~"), 'karl') + self.tester._server_url.replace('8383', '21174') print(">>>>>server url>>>>>>>", self.tester._server_url, file=sys.stderr) fp_obs = self.tester.fetch_file_from_central(dirname(fp_main), prefix=prefix) # # test a file of the freshly transferred directory from main has From 038db3447e5ae2b1cda4d6dc257a8344d7cbdcac Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 08:30:56 +0100 Subject: [PATCH 33/78] check nginx logs --- .github/workflows/qiita-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 97917c4..ceead33 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -104,7 +104,9 @@ jobs: mkdir -p ${CONDA_PREFIX}/var/run/nginx/ export NGINX_FILE=`pwd`/qiita-dev/qiita_pet/nginx_example.conf export NGINX_FILE_NEW=`pwd`/qiita-dev/qiita_pet/nginx_example_local.conf - sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE} > ${NGINX_FILE_NEW} + echo "error_log /home/runner/logs/nginx_error.log debug;" > ${NGINX_FILE_NEW} + echo "access_log /home/runner/logs/nginx_access.log debug;" >> ${NGINX_FILE_NEW} + sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE} >> ${NGINX_FILE_NEW} sed -i "s#/Users/username/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE_NEW} nginx -V >&2 nginx -c ${NGINX_FILE_NEW} @@ -128,7 +130,9 @@ jobs: export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg export QIITA_CLIENT_DEBUG_LEVEL=DEBUG - nosetests --with-doctest --with-coverage --cover-package=qiita_client + nosetests --with-doctest --with-coverage --cover-package=qiita_client; true + cat /home/runner/logs/nginx_error.log + cat /home/runner/logs/nginx_access.log - uses: codecov/codecov-action@v3 with: From a2919ccc5111bcad212d5b3daf198787cb179ac8 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 08:37:03 +0100 Subject: [PATCH 34/78] pull correct branch --- .github/workflows/qiita-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index ceead33..831d428 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -52,7 +52,8 @@ jobs: # we need to download qiita directly so we have "easy" access to # all config files - wget https://github.com/biocore/qiita/archive/dev.zip + # wget https://github.com/biocore/qiita/archive/dev.zip + wget https://github.com/jlab/qiita/archive/refs/heads/tornado_FetchFileFromCentralHandler.zip -O dev.zip unzip dev.zip # pull out the port so we can modify the configuration file easily @@ -105,7 +106,6 @@ jobs: export NGINX_FILE=`pwd`/qiita-dev/qiita_pet/nginx_example.conf export NGINX_FILE_NEW=`pwd`/qiita-dev/qiita_pet/nginx_example_local.conf echo "error_log /home/runner/logs/nginx_error.log debug;" > ${NGINX_FILE_NEW} - echo "access_log /home/runner/logs/nginx_access.log debug;" >> ${NGINX_FILE_NEW} sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE} >> ${NGINX_FILE_NEW} sed -i "s#/Users/username/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE_NEW} nginx -V >&2 From c48fe15de560bff9079a3f25a930a76e3fc599c3 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 08:40:24 +0100 Subject: [PATCH 35/78] proper renaming --- .github/workflows/qiita-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 831d428..ada3e28 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -53,8 +53,10 @@ jobs: # we need to download qiita directly so we have "easy" access to # all config files # wget https://github.com/biocore/qiita/archive/dev.zip - wget https://github.com/jlab/qiita/archive/refs/heads/tornado_FetchFileFromCentralHandler.zip -O dev.zip - unzip dev.zip + # unzip dev.zip + wget https://github.com/jlab/qiita/archive/refs/heads/tornado_FetchFileFromCentralHandler.zip + unzip tornado_FetchFileFromCentralHandler.zip + mv qiita-tornado_FetchFileFromCentralHandler qiita-dev # pull out the port so we can modify the configuration file easily pgport=${{ job.services.postgres.ports[5432] }} From c3163fbcb03fa63da9688e03be5dc2056ac33e29 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 08:44:11 +0100 Subject: [PATCH 36/78] create dir first --- .github/workflows/qiita-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index ada3e28..4655b65 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -107,6 +107,7 @@ jobs: mkdir -p ${CONDA_PREFIX}/var/run/nginx/ export NGINX_FILE=`pwd`/qiita-dev/qiita_pet/nginx_example.conf export NGINX_FILE_NEW=`pwd`/qiita-dev/qiita_pet/nginx_example_local.conf + mkdir -p /home/runner/logs/ echo "error_log /home/runner/logs/nginx_error.log debug;" > ${NGINX_FILE_NEW} sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE} >> ${NGINX_FILE_NEW} sed -i "s#/Users/username/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE_NEW} From d36f4b8d394401a425b22d6dc96f044b9759e4e2 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 08:54:43 +0100 Subject: [PATCH 37/78] clean up test --- qiita_client/tests/test_qiita_client.py | 47 ++++--------------------- 1 file changed, 6 insertions(+), 41 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 6a1ad58..55afa9a 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -528,17 +528,11 @@ def test_delete_file_from_central(self): self.assertTrue(exists(fp_deleted)) def test_fetch_directory(self): - import sys # creating a test directory fp_test = join('./job', '2_test_folder', 'source') - print(">>>>>A>>>>>>>", fp_test, file=sys.stderr) self._create_test_dir(prefix=fp_test) # transmitting test directory into qiita main - # self.tester._plugincoupling = 'https' - # fakeTest = namedtuple("fakeTest", "qclient") - # fakeTest.qclient = self.tester - # fp_main = PluginTestCase.deposite_in_qiita_basedir(fakeTest, fp_test) self.tester._plugincoupling = 'https' self.tester.push_file_to_central(fp_test) # a bit hacky, but should work as long as test database does not change @@ -547,45 +541,16 @@ def test_fetch_directory(self): :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] fp_main = join(base_data_dir, join(*Path(fp_test).parts)) - # fakeTest = namedtuple("fakeTest", "qclient") - # fakeTest.qclient = self.tester - # fp_main = PluginTestCase.deposite_in_qiita_basedir(fakeTest, fp_test) - - print(">>>>>B>>>>>>>", fp_main, file=sys.stderr) - # fetch test directory from qiita main, this time storing it at # QIITA_BASE_DIR - print(">>>>>C>>>>>>>", dirname(fp_main), file=sys.stderr) - import os - - # with open("/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_pet/nginx_example_local.conf", 'r') as f: - # for l in f.readlines(): - # print(">>>>>>>>>>>> NGINX >>> %s" % l, file=sys.stderr) - # with open('/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_core/support_files/config_test_local.cfg', 'r') as f: - # for l in f.readlines(): - # print(">>>>>>>>>>>> Q-Conf >>> %s" % l, file=sys.stderr) - # from glob import glob - # for f in glob('/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/**/*', recursive=True): - # print(">>>>>>>>>>>> files >>> %s" % f, file=sys.stderr) - prefix = join(expanduser("~"), 'karl') - self.tester._server_url.replace('8383', '21174') - print(">>>>>server url>>>>>>>", self.tester._server_url, file=sys.stderr) - fp_obs = self.tester.fetch_file_from_central(dirname(fp_main), prefix=prefix) - # # test a file of the freshly transferred directory from main has - # # expected file content - # with open(join(fp_obs, 'source', 'testdir', 'fileA.txt'), 'r') as f: - # self.assertIn('contentA', '\n'.join(f.readlines())) - - print(">>>>>>>>>>>>", fp_test, fp_main, fp_obs, file=sys.stderr) - print(">>>>>>>>>>>>", fp_test, fp_main, fp_obs) + fp_obs = self.tester.fetch_file_from_central( + dirname(fp_main), prefix=prefix) + # test a file of the freshly transferred directory from main has + # expected file content + with open(join(fp_obs, 'source', 'testdir', 'fileA.txt'), 'r') as f: + self.assertIn('contentA', '\n'.join(f.readlines())) if __name__ == '__main__': main() - - -# >>>>>>>>>>>> ./job/2_test_folder/source /qiita_data/./job/2_test_folder/source /qiita_data/./job/2_test_folder -# 2025-11-06T14:51:59.6108533Z >>>>>A>>>>>>> ./job/2_test_folder/source -# 2025-11-06T14:51:59.6655140Z >>>>>B>>>>>>> /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/./job/2_test_folder/source -# 2025-11-06T14:51:59.6656458Z >>>>>C>>>>>>> /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/./job/2_test_folder From 4005690371dbe8f7838537816008ff64733d02b4 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 09:00:43 +0100 Subject: [PATCH 38/78] debug --- qiita_client/tests/test_qiita_client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 55afa9a..ac390fb 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -546,6 +546,10 @@ def test_fetch_directory(self): prefix = join(expanduser("~"), 'karl') fp_obs = self.tester.fetch_file_from_central( dirname(fp_main), prefix=prefix) + + import sys + print(">>>>>>>>>>>>", fp_test, fp_main, fp_obs, file=sys.stderr) + # test a file of the freshly transferred directory from main has # expected file content with open(join(fp_obs, 'source', 'testdir', 'fileA.txt'), 'r') as f: From 34ff8b099636a1d02600d06808d3c34039523342 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 09:09:15 +0100 Subject: [PATCH 39/78] more debug --- qiita_client/tests/test_qiita_client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index ac390fb..33f20b7 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -543,12 +543,18 @@ def test_fetch_directory(self): # fetch test directory from qiita main, this time storing it at # QIITA_BASE_DIR - prefix = join(expanduser("~"), 'karl') + prefix = join(expanduser("~"), 'localFetch') fp_obs = self.tester.fetch_file_from_central( dirname(fp_main), prefix=prefix) import sys print(">>>>>>>>>>>>", fp_test, fp_main, fp_obs, file=sys.stderr) + from glob import glob + for fp in glob("%s/**/*" % fp_obs, recursive=True): + print(">>>>>>>>glob>>>> %s" % fp) + # >>>>>>>>>>>> ./job/2_test_folder/source /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/source /home/runner/karl/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder + # /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder + # No such file or directory: '/home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/source/testdir/fileA.txt' # test a file of the freshly transferred directory from main has # expected file content From 94064535b11dc04af438777dba7a6a31810576ff Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 09:15:36 +0100 Subject: [PATCH 40/78] adapt fp --- qiita_client/tests/test_qiita_client.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 33f20b7..c19e772 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -556,9 +556,26 @@ def test_fetch_directory(self): # /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder # No such file or directory: '/home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/source/testdir/fileA.txt' +# 2025-11-07T08:12:39.6827567Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job +# 2025-11-07T08:12:39.6828500Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder +# 2025-11-07T08:12:39.6829468Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job +# 2025-11-07T08:12:39.6830521Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder +# 2025-11-07T08:12:39.6831641Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source +# 2025-11-07T08:12:39.6832764Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir +# 2025-11-07T08:12:39.6833951Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/fileA.txt +# 2025-11-07T08:12:39.6835367Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirB_l1 +# 2025-11-07T08:12:39.6836581Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirA_l1 +# 2025-11-07T08:12:39.6837856Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirB_l1/fileE.sff +# 2025-11-07T08:12:39.6839142Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirA_l1/fileB.fna +# 2025-11-07T08:12:39.6840407Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirA_l1/subdirC_l2 +# 2025-11-07T08:12:39.6841923Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirA_l1/subdirC_l2/fileD.seq +# 2025-11-07T08:12:39.6843318Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirA_l1/subdirC_l2/fileC.log + +# No such file or directory: '/home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/source/testdir/fileA.txt' + # test a file of the freshly transferred directory from main has # expected file content - with open(join(fp_obs, 'source', 'testdir', 'fileA.txt'), 'r') as f: + with open(join(fp_obs, 'job/2_test_folder/job/2_test_folder/', 'source', 'testdir', 'fileA.txt'), 'r') as f: self.assertIn('contentA', '\n'.join(f.readlines())) From 5d968a8a8b346141393f1f49a3745942ac31a900 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 09:20:41 +0100 Subject: [PATCH 41/78] clean up --- qiita_client/tests/test_qiita_client.py | 29 ++----------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index c19e772..09db5b6 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -546,36 +546,11 @@ def test_fetch_directory(self): prefix = join(expanduser("~"), 'localFetch') fp_obs = self.tester.fetch_file_from_central( dirname(fp_main), prefix=prefix) - - import sys - print(">>>>>>>>>>>>", fp_test, fp_main, fp_obs, file=sys.stderr) - from glob import glob - for fp in glob("%s/**/*" % fp_obs, recursive=True): - print(">>>>>>>>glob>>>> %s" % fp) - # >>>>>>>>>>>> ./job/2_test_folder/source /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/source /home/runner/karl/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder - # /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder - # No such file or directory: '/home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/source/testdir/fileA.txt' - -# 2025-11-07T08:12:39.6827567Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job -# 2025-11-07T08:12:39.6828500Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder -# 2025-11-07T08:12:39.6829468Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job -# 2025-11-07T08:12:39.6830521Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder -# 2025-11-07T08:12:39.6831641Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source -# 2025-11-07T08:12:39.6832764Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir -# 2025-11-07T08:12:39.6833951Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/fileA.txt -# 2025-11-07T08:12:39.6835367Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirB_l1 -# 2025-11-07T08:12:39.6836581Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirA_l1 -# 2025-11-07T08:12:39.6837856Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirB_l1/fileE.sff -# 2025-11-07T08:12:39.6839142Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirA_l1/fileB.fna -# 2025-11-07T08:12:39.6840407Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirA_l1/subdirC_l2 -# 2025-11-07T08:12:39.6841923Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirA_l1/subdirC_l2/fileD.seq -# 2025-11-07T08:12:39.6843318Z >>>>>>>>glob>>>> /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/job/2_test_folder/source/testdir/subdirA_l1/subdirC_l2/fileC.log - -# No such file or directory: '/home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/source/testdir/fileA.txt' # test a file of the freshly transferred directory from main has # expected file content - with open(join(fp_obs, 'job/2_test_folder/job/2_test_folder/', 'source', 'testdir', 'fileA.txt'), 'r') as f: + with open(join(fp_obs, 'job/2_test_folder/job/2_test_folder/', + 'source', 'testdir', 'fileA.txt'), 'r') as f: self.assertIn('contentA', '\n'.join(f.readlines())) From d226ab58618dd5bdb92f9212069d513f4b09c20a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 09:25:15 +0100 Subject: [PATCH 42/78] clean up workflow file --- .github/workflows/qiita-ci.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 4655b65..6d58a2c 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -107,11 +107,8 @@ jobs: mkdir -p ${CONDA_PREFIX}/var/run/nginx/ export NGINX_FILE=`pwd`/qiita-dev/qiita_pet/nginx_example.conf export NGINX_FILE_NEW=`pwd`/qiita-dev/qiita_pet/nginx_example_local.conf - mkdir -p /home/runner/logs/ - echo "error_log /home/runner/logs/nginx_error.log debug;" > ${NGINX_FILE_NEW} - sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE} >> ${NGINX_FILE_NEW} + sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE} > ${NGINX_FILE_NEW} sed -i "s#/Users/username/qiita#${PWD}/qiita-dev#g" ${NGINX_FILE_NEW} - nginx -V >&2 nginx -c ${NGINX_FILE_NEW} echo "3. Setting up qiita" @@ -133,9 +130,7 @@ jobs: export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg export QIITA_CLIENT_DEBUG_LEVEL=DEBUG - nosetests --with-doctest --with-coverage --cover-package=qiita_client; true - cat /home/runner/logs/nginx_error.log - cat /home/runner/logs/nginx_access.log + nosetests --with-doctest --with-coverage --cover-package=qiita_client - uses: codecov/codecov-action@v3 with: From 18dbaa820d324cb6d34aa5245aeead489a1bc40a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 09:59:03 +0100 Subject: [PATCH 43/78] prepare to clean up filepath mess --- qiita_client/qiita_client.py | 3 +++ qiita_client/tests/test_qiita_client.py | 32 +++++++++++++------------ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 1c6e5fc..9128d4e 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -817,6 +817,9 @@ def fetch_file_from_central(self, filepath, prefix=None): # check if requested filepath is a single file OR a whole directory if 'Is-Qiita-Directory' in response.headers.keys(): with ZipFile(BytesIO(response.content)) as zf: + import sys + print("ÖÖÖÖÖ in client:\n%s" % zf.filelist, file=sys.stderr) + zf.extractall(path=target_filepath) else: content = response.content diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 09db5b6..28bc69e 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -14,7 +14,6 @@ from json import dumps import pandas as pd from shutil import rmtree -from pathlib import Path from qiita_client.qiita_client import (QiitaClient, _format_payload, ArtifactInfo) @@ -528,29 +527,32 @@ def test_delete_file_from_central(self): self.assertTrue(exists(fp_deleted)) def test_fetch_directory(self): - # creating a test directory - fp_test = join('./job', '2_test_folder', 'source') - self._create_test_dir(prefix=fp_test) - - # transmitting test directory into qiita main - self.tester._plugincoupling = 'https' - self.tester.push_file_to_central(fp_test) # a bit hacky, but should work as long as test database does not change ainfo = self.qclient.get('/qiita_db/artifacts/1/') base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] - fp_main = join(base_data_dir, join(*Path(fp_test).parts)) - # fetch test directory from qiita main, this time storing it at - # QIITA_BASE_DIR + print("@@@@@@@@@@@STEFAN", base_data_dir) + + # creating a LOCAL test directory within base_data_dir as the DB entry + # but no files exist. "job" is the according mountpoint + fp_test = join(base_data_dir, 'job', '2_test_folder') + self._create_test_dir(prefix=fp_test) + + # transmitting test directory to qiita main (remote) + self.tester._plugincoupling = 'https' + self.tester.push_file_to_central(fp_test) + # fp_main = join(base_data_dir, join(*Path(fp_test).parts)) + + # fetch test directory from qiita main to a different location + # (=prefix) than it was generated prefix = join(expanduser("~"), 'localFetch') - fp_obs = self.tester.fetch_file_from_central( - dirname(fp_main), prefix=prefix) + fp_obs = self.tester.fetch_file_from_central(fp_test, prefix=prefix) + print("@@@@@@@@@@@STEFAN 2", prefix, fp_test, fp_obs) # test a file of the freshly transferred directory from main has # expected file content - with open(join(fp_obs, 'job/2_test_folder/job/2_test_folder/', - 'source', 'testdir', 'fileA.txt'), 'r') as f: + with open(join(fp_obs, 'testdir', 'fileA.txt'), 'r') as f: self.assertIn('contentA', '\n'.join(f.readlines())) From e21171b8fd8c381fcf468ba126f2a5ed6d53a6bd Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 10:07:27 +0100 Subject: [PATCH 44/78] get files --- qiita_client/tests/test_qiita_client.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 28bc69e..9e9078f 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -550,6 +550,10 @@ def test_fetch_directory(self): fp_obs = self.tester.fetch_file_from_central(fp_test, prefix=prefix) print("@@@@@@@@@@@STEFAN 2", prefix, fp_test, fp_obs) + from glob import glob + for fp in glob("%s/**/*" % prefix, recursive=True): + print("@@@@@@@@@@@ STEFAN present: %s" % fp) + # test a file of the freshly transferred directory from main has # expected file content with open(join(fp_obs, 'testdir', 'fileA.txt'), 'r') as f: @@ -558,3 +562,15 @@ def test_fetch_directory(self): if __name__ == '__main__': main() + +# # [Errno 2] No such file or directory: +# # '/home/runner/localFetch +# # /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ +# # job/2_test_folder/testdir/fileA.txt' + +# [ +# , +# , +# , +# , +# ] \ No newline at end of file From 7cd6f1643d5bb07fbcee8c102c764fa53cef63f2 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 10:37:09 +0100 Subject: [PATCH 45/78] more debug --- .github/workflows/qiita-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 6d58a2c..ccd751d 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -54,9 +54,9 @@ jobs: # all config files # wget https://github.com/biocore/qiita/archive/dev.zip # unzip dev.zip - wget https://github.com/jlab/qiita/archive/refs/heads/tornado_FetchFileFromCentralHandler.zip - unzip tornado_FetchFileFromCentralHandler.zip - mv qiita-tornado_FetchFileFromCentralHandler qiita-dev + wget https://github.com/jlab/qiita/archive/refs/heads/tornado_FetchFileFromCentralHandler_alsoDirs_debug.zip + unzip tornado_FetchFileFromCentralHandler_alsoDirs_debug.zip + mv qiita-tornado_FetchFileFromCentralHandler_alsoDirs_debug qiita-dev # pull out the port so we can modify the configuration file easily pgport=${{ job.services.postgres.ports[5432] }} @@ -130,7 +130,8 @@ jobs: export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg export QIITA_CLIENT_DEBUG_LEVEL=DEBUG - nosetests --with-doctest --with-coverage --cover-package=qiita_client + nosetests --with-doctest --with-coverage --cover-package=qiita_client; true + cat /tmp/stefan.log - uses: codecov/codecov-action@v3 with: From 9e2baf05bc216ac3fcd79f423c78a18c4b44b88e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 10:43:48 +0100 Subject: [PATCH 46/78] forcer cat --- .github/workflows/qiita-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index ccd751d..114e3b7 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -130,8 +130,7 @@ jobs: export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg export QIITA_CLIENT_DEBUG_LEVEL=DEBUG - nosetests --with-doctest --with-coverage --cover-package=qiita_client; true - cat /tmp/stefan.log + nosetests --with-doctest --with-coverage --cover-package=qiita_client; cat /tmp/stefan.log - uses: codecov/codecov-action@v3 with: From c3cc119c0dd3e965aeb4a21e3ef2c4959cf4116b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 10:54:15 +0100 Subject: [PATCH 47/78] read log here --- qiita_client/tests/test_qiita_client.py | 32 ++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 9e9078f..ddcba6b 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -554,6 +554,9 @@ def test_fetch_directory(self): for fp in glob("%s/**/*" % prefix, recursive=True): print("@@@@@@@@@@@ STEFAN present: %s" % fp) + with open('/tmp/stefan.log') as f: + print("üüüüüüüüüü log from qiita\n %s" % f.readlines()) + # test a file of the freshly transferred directory from main has # expected file content with open(join(fp_obs, 'testdir', 'fileA.txt'), 'r') as f: @@ -573,4 +576,31 @@ def test_fetch_directory(self): # , # , # , -# ] \ No newline at end of file +# ] + +# tinqiita: +# base_data_dir: /qiita_data/ +# prefix: /root/localFetch/ +# deposited file locations +# //job/2_test_folder/testdir/fileA.txt /root/localFetch/qiita_data/job/2_test_folder/testdir/fileA.txt +# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff +# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna +# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log +# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq +# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/fileA.txt +# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff +# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna +# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log +# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq + + +# github: +# base_data_dir: /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ +# prefix: /home/runner/localFetch +# deposited file locations +# //job/2_test_folder/job/2_test_folder/testdir/fileA.txt +# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/fileA.txt +# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirB_l1/fileE.sff +# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/fileB.fna +# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq +# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log \ No newline at end of file From 7b11c640860299e82dac8fec6808a8ff797a85ec Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 10:55:52 +0100 Subject: [PATCH 48/78] debug in client --- qiita_client/qiita_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 9128d4e..39543b3 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -881,6 +881,7 @@ def push_file_to_central(self, filepath): for root, dirnames, filenames in os.walk(filepath): for filename in fnmatch.filter(filenames, "*"): fp = os.path.join(root, filename) + print("äääääääääää '%s' '%s' '%s' '%s' " % (root, dirnames, filenames, os.path.join(dirpath, os.path.dirname(fp)))) self.post('/cloud/push_file_to_central/', files={os.path.join( dirpath, From 13b226aa25d5f2ac5af122100dca1d845972bcee Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 11:22:07 +0100 Subject: [PATCH 49/78] remove cat --- .github/workflows/qiita-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 114e3b7..cf4ba64 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -130,7 +130,7 @@ jobs: export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg export QIITA_CLIENT_DEBUG_LEVEL=DEBUG - nosetests --with-doctest --with-coverage --cover-package=qiita_client; cat /tmp/stefan.log + nosetests --with-doctest --with-coverage --cover-package=qiita_client - uses: codecov/codecov-action@v3 with: From c8eaae62cfb593bbe265a9621713eec95621f61d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 11:51:02 +0100 Subject: [PATCH 50/78] trigger build --- qiita_client/tests/test_qiita_client.py | 96 +++++++++++++++---------- 1 file changed, 58 insertions(+), 38 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index ddcba6b..2c75536 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -566,41 +566,61 @@ def test_fetch_directory(self): if __name__ == '__main__': main() -# # [Errno 2] No such file or directory: -# # '/home/runner/localFetch -# # /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ -# # job/2_test_folder/testdir/fileA.txt' - -# [ -# , -# , -# , -# , -# ] - -# tinqiita: -# base_data_dir: /qiita_data/ -# prefix: /root/localFetch/ -# deposited file locations -# //job/2_test_folder/testdir/fileA.txt /root/localFetch/qiita_data/job/2_test_folder/testdir/fileA.txt -# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff -# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna -# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log -# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq -# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/fileA.txt -# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff -# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna -# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log -# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq - - -# github: -# base_data_dir: /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ -# prefix: /home/runner/localFetch -# deposited file locations -# //job/2_test_folder/job/2_test_folder/testdir/fileA.txt -# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/fileA.txt -# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirB_l1/fileE.sff -# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/fileB.fna -# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq -# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log \ No newline at end of file +# tinqiita +# base_data_dir: /qiita_data/ +# prefix: /root/localFetch +# actual files after fetch +# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/fileA.txt +# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff +# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna +# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log +# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq +# files at Main: +# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/fileA.txt\n', +# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff\n', +# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', +# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n', +# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n'] +# when fetching dir +# , +# , +# , +# , +# ] + +# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/fileA.txt testdir/fileA.txt +# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff testdir/subdirB_l1/fileE.sff +# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna testdir/subdirA_l1/fileB.fna +# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log testdir/subdirA_l1/subdirC_l2/fileC.log +# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq testdir/subdirA_l1/subdirC_l2/fileD.seq\n'] + + + + +# github +# base_data_dir: /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ +# prefix: /home/runner/localFetch +# actual files after fetch +# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/fileA.txt +# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirB_l1/fileE.sff +# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/fileB.fna +# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq +# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log +# files at Main: +# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/fileA.txt\n', +# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff\n', +# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', +# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n', +# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n'] +# when fetching dir +# , +# , +# , +# , +# ] +# all_files + # '- 9 /protected/job/2_test_folder/testdir/fileA.txt job/2_test_folder/testdir/fileA.txt\n', + # '- 5 /protected/job/2_test_folder/testdir/subdirB_l1/fileE.sff job/2_test_folder/testdir/subdirB_l1/fileE.sff\n', + # '- 10 /protected/job/2_test_folder/testdir/subdirA_l1/fileB.fna job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', + # '- 4 /protected/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n', + # '- 10 /protected/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n'] From c560eff91019c228615976631c2416f4e67fe6ed Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 11:56:35 +0100 Subject: [PATCH 51/78] trigger --- qiita_client/tests/test_qiita_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 2c75536..3271532 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -624,3 +624,4 @@ def test_fetch_directory(self): # '- 10 /protected/job/2_test_folder/testdir/subdirA_l1/fileB.fna job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', # '- 4 /protected/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n', # '- 10 /protected/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n'] + From 86f86fbd59d9444f4d09764559963f89ad4b7455 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 12:28:37 +0100 Subject: [PATCH 52/78] trigger --- qiita_client/tests/test_qiita_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 3271532..1a5a063 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -596,7 +596,6 @@ def test_fetch_directory(self): - # github # base_data_dir: /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ # prefix: /home/runner/localFetch From 119e14a1f6cf995fa8ddef544bcfcb2d6ee600ae Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 12:41:29 +0100 Subject: [PATCH 53/78] clean debug infos --- qiita_client/tests/test_qiita_client.py | 69 ------------------------- 1 file changed, 69 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 1a5a063..ddf843c 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -532,8 +532,6 @@ def test_fetch_directory(self): base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] - print("@@@@@@@@@@@STEFAN", base_data_dir) - # creating a LOCAL test directory within base_data_dir as the DB entry # but no files exist. "job" is the according mountpoint fp_test = join(base_data_dir, 'job', '2_test_folder') @@ -548,14 +546,6 @@ def test_fetch_directory(self): # (=prefix) than it was generated prefix = join(expanduser("~"), 'localFetch') fp_obs = self.tester.fetch_file_from_central(fp_test, prefix=prefix) - print("@@@@@@@@@@@STEFAN 2", prefix, fp_test, fp_obs) - - from glob import glob - for fp in glob("%s/**/*" % prefix, recursive=True): - print("@@@@@@@@@@@ STEFAN present: %s" % fp) - - with open('/tmp/stefan.log') as f: - print("üüüüüüüüüü log from qiita\n %s" % f.readlines()) # test a file of the freshly transferred directory from main has # expected file content @@ -565,62 +555,3 @@ def test_fetch_directory(self): if __name__ == '__main__': main() - -# tinqiita -# base_data_dir: /qiita_data/ -# prefix: /root/localFetch -# actual files after fetch -# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/fileA.txt -# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff -# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna -# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log -# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq -# files at Main: -# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/fileA.txt\n', -# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff\n', -# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', -# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n', -# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n'] -# when fetching dir -# , -# , -# , -# , -# ] - -# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/fileA.txt testdir/fileA.txt -# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff testdir/subdirB_l1/fileE.sff -# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna testdir/subdirA_l1/fileB.fna -# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log testdir/subdirA_l1/subdirC_l2/fileC.log -# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq testdir/subdirA_l1/subdirC_l2/fileD.seq\n'] - - - -# github -# base_data_dir: /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ -# prefix: /home/runner/localFetch -# actual files after fetch -# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/fileA.txt -# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirB_l1/fileE.sff -# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/fileB.fna -# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq -# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log -# files at Main: -# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/fileA.txt\n', -# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff\n', -# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', -# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n', -# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n'] -# when fetching dir -# , -# , -# , -# , -# ] -# all_files - # '- 9 /protected/job/2_test_folder/testdir/fileA.txt job/2_test_folder/testdir/fileA.txt\n', - # '- 5 /protected/job/2_test_folder/testdir/subdirB_l1/fileE.sff job/2_test_folder/testdir/subdirB_l1/fileE.sff\n', - # '- 10 /protected/job/2_test_folder/testdir/subdirA_l1/fileB.fna job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', - # '- 4 /protected/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n', - # '- 10 /protected/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n'] - From fa819be7dbb760a867bd70d530e745c64342ca73 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 12:42:48 +0100 Subject: [PATCH 54/78] remove debug infos --- qiita_client/qiita_client.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 39543b3..1c6e5fc 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -817,9 +817,6 @@ def fetch_file_from_central(self, filepath, prefix=None): # check if requested filepath is a single file OR a whole directory if 'Is-Qiita-Directory' in response.headers.keys(): with ZipFile(BytesIO(response.content)) as zf: - import sys - print("ÖÖÖÖÖ in client:\n%s" % zf.filelist, file=sys.stderr) - zf.extractall(path=target_filepath) else: content = response.content @@ -881,7 +878,6 @@ def push_file_to_central(self, filepath): for root, dirnames, filenames in os.walk(filepath): for filename in fnmatch.filter(filenames, "*"): fp = os.path.join(root, filename) - print("äääääääääää '%s' '%s' '%s' '%s' " % (root, dirnames, filenames, os.path.join(dirpath, os.path.dirname(fp)))) self.post('/cloud/push_file_to_central/', files={os.path.join( dirpath, From f2619e032f8acfe3802c5ffb776f59cc0cccd3e0 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 12:55:16 +0100 Subject: [PATCH 55/78] fall back to base branch --- .github/workflows/qiita-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index cf4ba64..6d58a2c 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -54,9 +54,9 @@ jobs: # all config files # wget https://github.com/biocore/qiita/archive/dev.zip # unzip dev.zip - wget https://github.com/jlab/qiita/archive/refs/heads/tornado_FetchFileFromCentralHandler_alsoDirs_debug.zip - unzip tornado_FetchFileFromCentralHandler_alsoDirs_debug.zip - mv qiita-tornado_FetchFileFromCentralHandler_alsoDirs_debug qiita-dev + wget https://github.com/jlab/qiita/archive/refs/heads/tornado_FetchFileFromCentralHandler.zip + unzip tornado_FetchFileFromCentralHandler.zip + mv qiita-tornado_FetchFileFromCentralHandler qiita-dev # pull out the port so we can modify the configuration file easily pgport=${{ job.services.postgres.ports[5432] }} From 9639d562b82723e6a46fa0b8de80a6824599509f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 11 Nov 2025 15:56:15 +0100 Subject: [PATCH 56/78] more verbose log message --- qiita_client/qiita_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 1c6e5fc..25553b8 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -863,8 +863,9 @@ def push_file_to_central(self, filepath): return filepath elif self._plugincoupling == 'https': - logger.debug('Submitting %s %s to qiita server.' % ( - 'directory' if os.path.isdir(filepath) else 'file', filepath)) + logger.debug('Submitting %s %s to qiita server via %s' % ( + 'directory' if os.path.isdir(filepath) else 'file', filepath, + self._plugincoupling)) # target path, i.e. without filename dirpath = os.path.dirname(filepath) From 793b287ffd0ccf9a176423df116942eb29061eda Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 12 Nov 2025 12:27:03 +0100 Subject: [PATCH 57/78] allow for file/dir deletion in test mode --- qiita_client/qiita_client.py | 19 +++++++++++++++---- qiita_client/tests/test_qiita_client.py | 18 ++++-------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 25553b8..158060b 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -897,10 +897,13 @@ def push_file_to_central(self, filepath): def delete_file_from_central(self, filepath): """Deletes a file in Qiita's central BASE_DATA_DIR directory. - I currently (2025-09-25) assess this operation to be too dangerous for + I currently (2025-11-12) assess this operation to be too dangerous for protocols other than "filesystem", i.e. on "https" the files are NOT deleted. - However, this might change in the future and since I don't want to + However, in plugin tests, this function is needed and I therefore + implemented an API endpoint which is only activated when Qiita main + instance is operated in test mode. + This might change in the future and since I don't want to touch every plugin's code again, I am adding this function here already and use it in according plugin code locations, e.g. function _gzip_file in qtp-sequencing. @@ -916,8 +919,16 @@ def delete_file_from_central(self, filepath): The given filepath - to be transparent in plugin code. """ if self._plugincoupling == 'filesystem': - os.remove(filepath) + if os.path.isdir(filepath): + shutil.rmtree(filepath) + else: + os.remove(filepath) elif self._plugincoupling == 'https': - pass + try: + response = self.get( + '/cloud/delete_file_from_central/' + filepath, + rettype='object') + except: + pass return filepath diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index ddf843c..6abe399 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -511,20 +511,10 @@ def test_delete_file_from_central(self): # delete file and test if it is gone fp_deleted = self.qclient.delete_file_from_central(fp_test) - if protocol == 'filesystem': - # all three fp should point to the same filepath - self.assertFalse(exists(fp_obs)) - self.assertFalse(exists(fp_test)) - self.assertFalse(exists(fp_deleted)) - elif protocol == 'https': - # as of 2025-09-26, I don't allow deletion of qiita main files - # through API endpoints. Thus, the file is NOT deleted! - # local version of the file - self.assertTrue(exists(fp_test)) - # qiita main filepath - self.assertTrue(exists(fp_obs)) - # qiita main filepath, returned by delete_file_from_central - self.assertTrue(exists(fp_deleted)) + # all three fp should point to the same filepath + self.assertFalse(exists(fp_obs)) + self.assertFalse(exists(fp_test)) + self.assertFalse(exists(fp_deleted)) def test_fetch_directory(self): # a bit hacky, but should work as long as test database does not change From 46b932e92f68001c8d1ec0e495c005023b7d581e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 12 Nov 2025 13:00:19 +0100 Subject: [PATCH 58/78] no exception handling --- qiita_client/qiita_client.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 158060b..b69c96a 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -924,11 +924,10 @@ def delete_file_from_central(self, filepath): else: os.remove(filepath) elif self._plugincoupling == 'https': - try: - response = self.get( - '/cloud/delete_file_from_central/' + filepath, - rettype='object') - except: - pass + # will return in internal server error, when qiita is in productive + # mode + response = self.get( + '/cloud/delete_file_from_central/' + filepath, + rettype='object') return filepath From 6a13ca0ab6289bbcb1c11ff6c94ed5fa27327041 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 12 Nov 2025 13:00:41 +0100 Subject: [PATCH 59/78] codestyle --- qiita_client/qiita_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index b69c96a..fb1beeb 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -926,7 +926,7 @@ def delete_file_from_central(self, filepath): elif self._plugincoupling == 'https': # will return in internal server error, when qiita is in productive # mode - response = self.get( + self.get( '/cloud/delete_file_from_central/' + filepath, rettype='object') From 6c60b02cb9c6697aee93a351bf2d5d73e13c846f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 13 Nov 2025 10:51:52 +0100 Subject: [PATCH 60/78] also return full filepath when in filesystem mode --- qiita_client/testing.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index 67ca68e..ef19893 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -113,9 +113,6 @@ def _stripRoot(fp): return fp[len(sep):] return fp - if self.qclient._plugincoupling == 'filesystem': - return fps - # use artifact 1 info to determine BASA_DATA_DIR, as we know that the # filepath ends with ....raw_data/1_s_G1_L001_sequences.fastq.gz, thus # BASE_DATA_DIR must be the prefix, e.g. /qiita_data/ From 0763558e943856c396ae004c393c87376bf99491 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 13 Nov 2025 11:06:02 +0100 Subject: [PATCH 61/78] test if fileobject exists prior to delete attempt --- qiita_client/qiita_client.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index fb1beeb..aed9845 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -919,10 +919,11 @@ def delete_file_from_central(self, filepath): The given filepath - to be transparent in plugin code. """ if self._plugincoupling == 'filesystem': - if os.path.isdir(filepath): - shutil.rmtree(filepath) - else: - os.remove(filepath) + if os.path.exists(filepath): + if os.path.isdir(filepath): + shutil.rmtree(filepath) + else: + os.remove(filepath) elif self._plugincoupling == 'https': # will return in internal server error, when qiita is in productive # mode From 5e4d2931ea915149f57bd982c30bc1ddecf97c0e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 13 Nov 2025 23:24:25 +0100 Subject: [PATCH 62/78] after calling a plugin function, push files of artifacts to qiita central --- qiita_client/plugin.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index 4c30bdc..6e7139d 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -100,9 +100,47 @@ def __init__(self, name, description, function, required_parameters, self.outputs = outputs self.analysis_only = analysis_only + def _push_artifacts_files_to_central(qclient, artifacts): + """Pushes all files of a list of artifacts to BASE_DATA_DIR. + + Parameters + ---------- + qclient : qiita_client.QiitaClient + The Qiita server client + artifacts : [ArtifactInfo] + A list of qiita Artifacts + + Returns + ------- + The input list of artifacts + """ + if artifacts is None: + return artifacts + + for artifact in artifacts: + if artifact is not None: + if 'files' in artifact.keys(): + artifact['files'] = { + filetype: [ + { + k: qclient.push_file_to_central(v) + if k == 'filepath' else v + for k, v + in file.items()} + for file + in artifact['files'][filetype]] + for filetype + in artifact['files'].keys() + } + + return artifacts + def __call__(self, qclient, server_url, job_id, output_dir): logger.debug('Entered QiitaCommand.__call__()') - return self.function(qclient, server_url, job_id, output_dir) + status, artifacts, error_message = self.function( + qclient, server_url, job_id, output_dir) + return status, QiitaCommand._push_artifacts_files_to_central( + qclient, artifacts), error_message class QiitaArtifactType(object): From 0d6a453e4420c13cab972e7afa52ebc81d6da958 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 13 Nov 2025 23:42:25 +0100 Subject: [PATCH 63/78] handle cases where Command function does NOT return typical 3-tuple --- qiita_client/plugin.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index 6e7139d..7bb0422 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -137,10 +137,17 @@ def _push_artifacts_files_to_central(qclient, artifacts): def __call__(self, qclient, server_url, job_id, output_dir): logger.debug('Entered QiitaCommand.__call__()') - status, artifacts, error_message = self.function( + results = self.function( qclient, server_url, job_id, output_dir) - return status, QiitaCommand._push_artifacts_files_to_central( - qclient, artifacts), error_message + # typical, but not all, functions of QiitaCommands return 3-tuple + # status=bool, list of artifacts, error_message=str + if isinstance(results, tuple) and (len(results) == 3) and \ + isinstance(results[0], bool) and \ + isinstance(results[1], list) and \ + isinstance(results[2], str): + results[1] = QiitaCommand._push_artifacts_files_to_central( + qclient, results[1]) + return results class QiitaArtifactType(object): From b5f9c0e634fd200006eee60a905dec4dade1ace5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 14 Nov 2025 10:52:00 +0100 Subject: [PATCH 64/78] adapt push function to list of ArtifactInfos --- qiita_client/plugin.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index 7bb0422..867ea35 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -15,7 +15,7 @@ from future import standard_library from json import dumps import urllib -from qiita_client import QiitaClient +from qiita_client import QiitaClient, ArtifactInfo import logging @@ -118,22 +118,13 @@ def _push_artifacts_files_to_central(qclient, artifacts): return artifacts for artifact in artifacts: - if artifact is not None: - if 'files' in artifact.keys(): - artifact['files'] = { - filetype: [ - { - k: qclient.push_file_to_central(v) - if k == 'filepath' else v - for k, v - in file.items()} - for file - in artifact['files'][filetype]] - for filetype - in artifact['files'].keys() - } - - return artifacts + if isinstance(artifact, ArtifactInfo): + for i in range(len(artifact.files)): + (fp, ftype) = artifact.files[i] + # send file to Qiita central and potentially update + # filepath, which is not done at the moment (2025-11-14) + fp = qclient.push_file_to_central(fp) + artifact.files[i] = (fp, ftype) def __call__(self, qclient, server_url, job_id, output_dir): logger.debug('Entered QiitaCommand.__call__()') From 9cc935a59e5aa4c668f143849791c878789ef050 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 14 Nov 2025 10:59:45 +0100 Subject: [PATCH 65/78] add tests --- qiita_client/tests/test_plugin.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index 43a8eca..d3f11f7 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -73,6 +73,23 @@ def func(a, b, c, d): self.exp_opt, self.exp_out, self.exp_dflt) self.assertEqual(obs('a', 'b', 'c', 'd'), 42) + def test__push_artifacts_files_to_central(self): + class fakeClient(): + def push_file_to_central(self, fp): + return 'pushed:%s' % fp + + obs = QiitaCommand._push_artifacts_files_to_central( + fakeClient(), [ + ArtifactInfo("stefArtiName", "Atype", [ + ("fp1", "preprocessed_fasta"), + ("fp2", "preprocessed_fastq")]), + None, + ArtifactInfo("artiName", "artiType", [])]) + + self.assertIn('pushed:', obs[0].files[0][0]) + self.assertIn('pushed:', obs[0].files[1][0]) + self.assertEqual([], obs[2].files) + class QiitaArtifactTypeTest(TestCase): def test_init(self): From 4a046cb633b56688fd785a79c81dd14952e27593 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 14 Nov 2025 11:13:31 +0100 Subject: [PATCH 66/78] cope with no return value --- qiita_client/tests/test_plugin.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index d3f11f7..e6d28ae 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -75,20 +75,20 @@ def func(a, b, c, d): def test__push_artifacts_files_to_central(self): class fakeClient(): - def push_file_to_central(self, fp): - return 'pushed:%s' % fp - - obs = QiitaCommand._push_artifacts_files_to_central( - fakeClient(), [ - ArtifactInfo("stefArtiName", "Atype", [ - ("fp1", "preprocessed_fasta"), - ("fp2", "preprocessed_fastq")]), - None, - ArtifactInfo("artiName", "artiType", [])]) - - self.assertIn('pushed:', obs[0].files[0][0]) - self.assertIn('pushed:', obs[0].files[1][0]) - self.assertEqual([], obs[2].files) + def push_file_to_central(self, filepath): + return 'pushed:%s' % filepath + + artifacts = [ + ArtifactInfo("stefArtiName", "Atype", [ + ("fp1", "preprocessed_fasta"), + ("fp2", "preprocessed_fastq")]), + None, + ArtifactInfo("artiName", "artiType", [])] + QiitaCommand._push_artifacts_files_to_central(fakeClient(), artifacts) + + self.assertIn('pushed:', artifacts[0].files[0][0]) + self.assertIn('pushed:', artifacts[0].files[1][0]) + self.assertEqual([], artifacts[2].files) class QiitaArtifactTypeTest(TestCase): From 2b08035d51ca5dfa336aeb9860e73dd251d53cec Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 14 Nov 2025 11:29:54 +0100 Subject: [PATCH 67/78] make method static --- qiita_client/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index 867ea35..ff2c1fc 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -100,6 +100,7 @@ def __init__(self, name, description, function, required_parameters, self.outputs = outputs self.analysis_only = analysis_only + @staticmethod def _push_artifacts_files_to_central(qclient, artifacts): """Pushes all files of a list of artifacts to BASE_DATA_DIR. @@ -136,8 +137,7 @@ def __call__(self, qclient, server_url, job_id, output_dir): isinstance(results[0], bool) and \ isinstance(results[1], list) and \ isinstance(results[2], str): - results[1] = QiitaCommand._push_artifacts_files_to_central( - qclient, results[1]) + QiitaCommand._push_artifacts_files_to_central(qclient, results[1]) return results From 5635f2bdc63c4fdc0088efb987704734ddc64406 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 14 Nov 2025 13:47:51 +0100 Subject: [PATCH 68/78] add an interception to automatically fetch files from qiita central --- qiita_client/qiita_client.py | 54 +++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index aed9845..a403552 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -17,6 +17,7 @@ import fnmatch from io import BytesIO from zipfile import ZipFile +import re try: @@ -412,6 +413,38 @@ def _request_retry(self, req, url, rettype='json', **kwargs): "Request '%s %s' did not succeed. Status code: %d. Message: %s" % (req.__name__, url, r.status_code, r.text)) + def _fetch_artifact_files(self, ainfo): + """helper method to fetch all files of an artifact from Qiita main. + + Parameters + ---------- + ainfo : json dict + Information about Qiita artifact + + Returns + ------- + Same as input BUT filepaths are adapated after downloading files from + Qiita main to local IF protocol coupling != filesystem. Otherwise, no + change occurs. + """ + if self._plugincoupling != 'filesystem': + if 'files' in ainfo.keys(): + ainfo['files'] = { + filetype: [ + { + k: self.fetch_file_from_central(v) + if k == 'filepath' else v + for k, v + in file.items()} + for file + in ainfo['files'][filetype]] + for filetype + in ainfo['files'].keys() + } + return ainfo + else: + return ainfo + def get(self, url, rettype='json', **kwargs): """Execute a get request against the Qiita server @@ -431,9 +464,28 @@ def get(self, url, rettype='json', **kwargs): The JSON response from the server """ logger.debug('Entered QiitaClient.get()') - return self._request_retry( + result = self._request_retry( self._session.get, url, rettype=rettype, **kwargs) + if self._plugincoupling != 'filesystem': + # intercept get requests from plugins that request metadata or + # artifact files and ensure they get transferred from Qiita + # central, when not using "filesystem" + if re.search(r"/qiita_db/prep_template/\d+/?$", url): + # client is requesting filepath to a prep/metadata file, see + # qiita/qiita_db/handlers/prep_template.py:: + # PrepTemplateDBHandler::get + # for the "result" data-structure + for fp in ['prep-file', 'sample-file']: + result[fp] = self.fetch_file_from_central(result[fp]) + elif re.search(r"/qiita_db/artifacts/\d+/?$", url): + # client is requesting an artifact, see + # qiita/qiita_db/handlers/artifact.py::ArtifactHandler::get + # for the "result" data-structure + result = self._fetch_artifact_files(result) + + return result + def post(self, url, **kwargs): """Execute a post request against the Qiita server From 1f1ed204a832d949f1c43d6bff6629e7b0adb362 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sat, 15 Nov 2025 17:18:50 +0100 Subject: [PATCH 69/78] some debug infos --- qiita_client/plugin.py | 3 +++ qiita_client/qiita_client.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index ff2c1fc..679ef4a 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -120,10 +120,13 @@ def _push_artifacts_files_to_central(qclient, artifacts): for artifact in artifacts: if isinstance(artifact, ArtifactInfo): + logger.debug('QiitaCommand::__call__: Push artifact files ' + 'via %s to central:' % qclient._plugincoupling) for i in range(len(artifact.files)): (fp, ftype) = artifact.files[i] # send file to Qiita central and potentially update # filepath, which is not done at the moment (2025-11-14) + logger.debug(' artifact files %s pushed to central' % fp) fp = qclient.push_file_to_central(fp) artifact.files[i] = (fp, ftype) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index a403552..24cfe80 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -428,6 +428,8 @@ def _fetch_artifact_files(self, ainfo): change occurs. """ if self._plugincoupling != 'filesystem': + logger.debug('QiitaClient::get: fetching artfiact file from ' + 'central: %s' % ainfo['files']) if 'files' in ainfo.keys(): ainfo['files'] = { filetype: [ @@ -476,7 +478,10 @@ def get(self, url, rettype='json', **kwargs): # qiita/qiita_db/handlers/prep_template.py:: # PrepTemplateDBHandler::get # for the "result" data-structure + logger.debug('QiitaClient::get: fetching artifact metadata' + '-file from central:') for fp in ['prep-file', 'sample-file']: + logger.debug('QiitaClient::get: file %s' % result[fp]) result[fp] = self.fetch_file_from_central(result[fp]) elif re.search(r"/qiita_db/artifacts/\d+/?$", url): # client is requesting an artifact, see From 38779ed97cdb3c1e5d41504b57acb9eeb6f8f25a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 16 Nov 2025 11:33:29 +0100 Subject: [PATCH 70/78] expose base_data_dir in self instead of extra method "deposit..." --- qiita_client/testing.py | 65 +++++++---------------------------------- 1 file changed, 11 insertions(+), 54 deletions(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index ef19893..4ca6bac 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -7,8 +7,7 @@ # ----------------------------------------------------------------------------- from unittest import TestCase -from os import environ, sep -from os.path import join, isabs +from os import environ from time import sleep from qiita_client import QiitaClient @@ -46,6 +45,16 @@ def setUpClass(cls): cls.qclient._plugincoupling = environ.get( 'QIITA_PLUGINCOUPLING', BaseQiitaPlugin._DEFAULT_PLUGIN_COUPLINGS) + # use artifact 1 info to determine BASA_DATA_DIR, as we know that the + # filepath ends with ....raw_data/1_s_G1_L001_sequences.fastq.gz, thus + # BASE_DATA_DIR must be the prefix, e.g. /qiita_data/ + # This might break IF file + # qiita-spots/qiita/qiita_db/support_files/populate_test_db.sql + # changes. + ainfo = cls.qclient.get('/qiita_db/artifacts/1/') + cls.base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ + :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] + # Give enough time for the plugins to register sleep(5) @@ -83,55 +92,3 @@ def _wait_for_running_job(self, job_id): break return status - - def deposite_in_qiita_basedir(self, fps, update_fp_only=False): - """Pushs a file to qiita main AND adapts given filepath accordingly. - - A helper function to fix file paths in tests such that they point to - the expected BASE_DATA_DIR. This becomes necessary when uncoupling the - plugin filesystem as some methods now actually fetches expected files - from BASE_DATA_DIR. This will fail for protocols other than filesystem - IF files are created locally by the plugin test. - - Parameters - ---------- - fps : str or [str] - Filepath or list of filepaths to file(s) that shall be part of - BASE_DATA_DIR, but currently points to some tmp file for testing. - update_fp_only : bool - Some tests operate on filepaths only - files do not actually need - to exist. Thus, we don't need to tranfer a file. - - Returns - ------- - The potentially modified filepaths. - """ - def _stripRoot(fp): - # chop off leading / for join to work properly when prepending - # the BASE_DATA_DIR - if isabs(fp): - return fp[len(sep):] - return fp - - # use artifact 1 info to determine BASA_DATA_DIR, as we know that the - # filepath ends with ....raw_data/1_s_G1_L001_sequences.fastq.gz, thus - # BASE_DATA_DIR must be the prefix, e.g. /qiita_data/ - # This might break IF file - # qiita-spots/qiita/qiita_db/support_files/populate_test_db.sql - # changes. - ainfo = self.qclient.get('/qiita_db/artifacts/1/') - base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ - :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] - if isinstance(fps, str): - if not update_fp_only: - self.qclient.push_file_to_central(fps) - return join(base_data_dir, _stripRoot(fps)) - elif isinstance(fps, list): - for fp in fps: - if not update_fp_only: - self.qclient.push_file_to_central(fp) - return [join(base_data_dir, _stripRoot(fp)) for fp in fps] - else: - raise ValueError( - "deposite_in_qiita_basedir is not implemented for type %s" - % type(fps)) From 01e18ddfed876c26f763a3bd68a19745a36d6bfb Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 16 Nov 2025 11:44:00 +0100 Subject: [PATCH 71/78] add attribute to fake client --- qiita_client/tests/test_plugin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index e6d28ae..1f73d7d 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -75,6 +75,9 @@ def func(a, b, c, d): def test__push_artifacts_files_to_central(self): class fakeClient(): + def __init__(): + self._plugincoupling = 'null protocol' + def push_file_to_central(self, filepath): return 'pushed:%s' % filepath From 267fe7da9bc7d59c6ff968b8542661eee71e96a9 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 16 Nov 2025 11:49:37 +0100 Subject: [PATCH 72/78] fix signature --- qiita_client/tests/test_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index 1f73d7d..f3010c8 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -75,7 +75,7 @@ def func(a, b, c, d): def test__push_artifacts_files_to_central(self): class fakeClient(): - def __init__(): + def __init__(self): self._plugincoupling = 'null protocol' def push_file_to_central(self, filepath): From 6e5ed9129a16b44cc0a9c4937d9d4a6c4e718263 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 17 Nov 2025 07:50:33 +0100 Subject: [PATCH 73/78] added option to turn off automatic file fetching --- qiita_client/qiita_client.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 24cfe80..bf6141f 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -447,7 +447,7 @@ def _fetch_artifact_files(self, ainfo): else: return ainfo - def get(self, url, rettype='json', **kwargs): + def get(self, url, rettype='json', no_file_fetching=False, **kwargs): """Execute a get request against the Qiita server Parameters @@ -457,6 +457,12 @@ def get(self, url, rettype='json', **kwargs): rettype : string The return type of the function, either "json" (default) or "object" for the response object itself + no_file_fetching : bool + If plugin is coupled through none "filesystem" protocols, artifact + files will automatically fetched from Qiita central when requesting + via "/qiita_db/prep_template/" or "/qiita_db/artifacts/". For + testing, you can turn off this behaviour. Handy if e.g. files not + yet exists in Qiita central. kwargs : dict The request kwargs @@ -469,7 +475,8 @@ def get(self, url, rettype='json', **kwargs): result = self._request_retry( self._session.get, url, rettype=rettype, **kwargs) - if self._plugincoupling != 'filesystem': + if (self._plugincoupling != 'filesystem') and \ + (no_file_fetching is False): # intercept get requests from plugins that request metadata or # artifact files and ensure they get transferred from Qiita # central, when not using "filesystem" From a6889082277573d4f79173f372e86bb962460110 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 17 Nov 2025 07:51:56 +0100 Subject: [PATCH 74/78] use a more simple request to determine base_data_dir --- qiita_client/testing.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index 4ca6bac..e4ff381 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -45,15 +45,15 @@ def setUpClass(cls): cls.qclient._plugincoupling = environ.get( 'QIITA_PLUGINCOUPLING', BaseQiitaPlugin._DEFAULT_PLUGIN_COUPLINGS) - # use artifact 1 info to determine BASA_DATA_DIR, as we know that the - # filepath ends with ....raw_data/1_s_G1_L001_sequences.fastq.gz, thus - # BASE_DATA_DIR must be the prefix, e.g. /qiita_data/ + # Determine BASE_DATA_DIR of qiita central, without having direct + # access to qiita's settings file. This is done by requesting + # information about prep 1, which should be in the test database. # This might break IF file # qiita-spots/qiita/qiita_db/support_files/populate_test_db.sql # changes. - ainfo = cls.qclient.get('/qiita_db/artifacts/1/') - cls.base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ - :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] + prep_info = cls.qclient.get('/qiita_db/prep_template/1/', + no_file_fetching=True) + cls.base_data_dir = prep_info['prep-file'].split('templates/')[0] # Give enough time for the plugins to register sleep(5) From 8508a8072eb2f1ecaed10d050d7f235ba888ac06 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 17 Nov 2025 16:07:52 +0100 Subject: [PATCH 75/78] push files when patching artifact summaries --- qiita_client/qiita_client.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index bf6141f..08abd1c 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -12,7 +12,7 @@ import requests import threading import pandas as pd -from json import dumps +from json import dumps, loads from random import randint import fnmatch from io import BytesIO @@ -389,7 +389,8 @@ def _request_retry(self, req, url, rettype='json', **kwargs): elif r.status_code in (500, 405): raise RuntimeError( "Request '%s %s' did not succeed. Status code: %d. " - "Message: %s" % (req.__name__, url, r.status_code, r.text)) + "Message: %s%s" % (req.__name__, url, r.status_code, + r.text, r.reason)) elif 0 <= (r.status_code - 200) < 100: try: if rettype is None or rettype == 'json': @@ -574,6 +575,26 @@ def patch(self, url, op, path, value=None, from_p=None, **kwargs): # we made sure that data is correctly formatted here kwargs['data'] = data + # similar to above get() injection mechanism, we are here pushing files + # to Qiita central, when patching artifact summaries + if (self._plugincoupling != 'filesystem') and \ + (path == '/html_summary/') and (op == 'add'): + if re.search(r"/qiita_db/artifacts/\d+/?$", url): + if value is not None: + logger.debug('QiitaClient::patch: push summary files to' \ + 'central: %s' % value) + try: + # values might be an json encoded dictioary with + # multiple filepaths... + dictValues = loads(value) + for ftype in ['html', 'dir']: + if (ftype in dictValues.keys()) and \ + (dictValues[ftype] is not None): + self.push_file_to_central(dictValues[ftype]) + except TypeError as e: + # or just a single string, i.e. filepath + self.push_file_to_central(value) + return self._request_retry( self._session.patch, url, rettype='json', **kwargs) From 3f38b1306eeba491e6e900436edb28e2f9724dcf Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 17 Nov 2025 16:31:21 +0100 Subject: [PATCH 76/78] codestyle --- qiita_client/qiita_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 08abd1c..696f310 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -581,7 +581,7 @@ def patch(self, url, op, path, value=None, from_p=None, **kwargs): (path == '/html_summary/') and (op == 'add'): if re.search(r"/qiita_db/artifacts/\d+/?$", url): if value is not None: - logger.debug('QiitaClient::patch: push summary files to' \ + logger.debug('QiitaClient::patch: push summary files to' 'central: %s' % value) try: # values might be an json encoded dictioary with @@ -591,7 +591,7 @@ def patch(self, url, op, path, value=None, from_p=None, **kwargs): if (ftype in dictValues.keys()) and \ (dictValues[ftype] is not None): self.push_file_to_central(dictValues[ftype]) - except TypeError as e: + except TypeError: # or just a single string, i.e. filepath self.push_file_to_central(value) From cdbf4e484e729e6699465df458bf7c7113fd806e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 17 Nov 2025 16:54:34 +0100 Subject: [PATCH 77/78] more elaborate test for plain string instead of json dict --- qiita_client/qiita_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 696f310..8ae3320 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -12,7 +12,7 @@ import requests import threading import pandas as pd -from json import dumps, loads +from json import dumps, loads, JSONDecodeError from random import randint import fnmatch from io import BytesIO @@ -581,7 +581,7 @@ def patch(self, url, op, path, value=None, from_p=None, **kwargs): (path == '/html_summary/') and (op == 'add'): if re.search(r"/qiita_db/artifacts/\d+/?$", url): if value is not None: - logger.debug('QiitaClient::patch: push summary files to' + logger.debug('QiitaClient::patch: push summary files to ' 'central: %s' % value) try: # values might be an json encoded dictioary with @@ -591,7 +591,7 @@ def patch(self, url, op, path, value=None, from_p=None, **kwargs): if (ftype in dictValues.keys()) and \ (dictValues[ftype] is not None): self.push_file_to_central(dictValues[ftype]) - except TypeError: + except (TypeError, JSONDecodeError): # or just a single string, i.e. filepath self.push_file_to_central(value) From 2321dbe973df0f17f4151c69ce73fa83afa017b6 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 17 Nov 2025 21:20:05 +0100 Subject: [PATCH 78/78] handle missing py27 JSONDecodeError --- qiita_client/qiita_client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 8ae3320..0ea317a 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -12,7 +12,13 @@ import requests import threading import pandas as pd -from json import dumps, loads, JSONDecodeError +from json import dumps, loads +try: + from json import JSONDecodeError +except ImportError: + # dirty hack to cope with the fact that python 2.7 does not have + # JSONDecodeError, but is needed for qp-target-gene plugin + JSONDecodeError = ValueError from random import randint import fnmatch from io import BytesIO