From 18dbaa820d324cb6d34aa5245aeead489a1bc40a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 09:59:03 +0100 Subject: [PATCH 01/12] prepare to clean up filepath mess --- qiita_client/qiita_client.py | 3 +++ qiita_client/tests/test_qiita_client.py | 32 +++++++++++++------------ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 1c6e5fc..9128d4e 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -817,6 +817,9 @@ def fetch_file_from_central(self, filepath, prefix=None): # check if requested filepath is a single file OR a whole directory if 'Is-Qiita-Directory' in response.headers.keys(): with ZipFile(BytesIO(response.content)) as zf: + import sys + print("ÖÖÖÖÖ in client:\n%s" % zf.filelist, file=sys.stderr) + zf.extractall(path=target_filepath) else: content = response.content diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 09db5b6..28bc69e 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -14,7 +14,6 @@ from json import dumps import pandas as pd from shutil import rmtree -from pathlib import Path from qiita_client.qiita_client import (QiitaClient, _format_payload, ArtifactInfo) @@ -528,29 +527,32 @@ def test_delete_file_from_central(self): self.assertTrue(exists(fp_deleted)) def test_fetch_directory(self): - # creating a test directory - fp_test = join('./job', '2_test_folder', 'source') - self._create_test_dir(prefix=fp_test) - - # transmitting test directory into qiita main - self.tester._plugincoupling = 'https' - self.tester.push_file_to_central(fp_test) # a bit hacky, but should work as long as test database does not change ainfo = self.qclient.get('/qiita_db/artifacts/1/') base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] - fp_main = join(base_data_dir, join(*Path(fp_test).parts)) - # fetch test directory from qiita main, this time storing it at - # QIITA_BASE_DIR + print("@@@@@@@@@@@STEFAN", base_data_dir) + + # creating a LOCAL test directory within base_data_dir as the DB entry + # but no files exist. "job" is the according mountpoint + fp_test = join(base_data_dir, 'job', '2_test_folder') + self._create_test_dir(prefix=fp_test) + + # transmitting test directory to qiita main (remote) + self.tester._plugincoupling = 'https' + self.tester.push_file_to_central(fp_test) + # fp_main = join(base_data_dir, join(*Path(fp_test).parts)) + + # fetch test directory from qiita main to a different location + # (=prefix) than it was generated prefix = join(expanduser("~"), 'localFetch') - fp_obs = self.tester.fetch_file_from_central( - dirname(fp_main), prefix=prefix) + fp_obs = self.tester.fetch_file_from_central(fp_test, prefix=prefix) + print("@@@@@@@@@@@STEFAN 2", prefix, fp_test, fp_obs) # test a file of the freshly transferred directory from main has # expected file content - with open(join(fp_obs, 'job/2_test_folder/job/2_test_folder/', - 'source', 'testdir', 'fileA.txt'), 'r') as f: + with open(join(fp_obs, 'testdir', 'fileA.txt'), 'r') as f: self.assertIn('contentA', '\n'.join(f.readlines())) From e21171b8fd8c381fcf468ba126f2a5ed6d53a6bd Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 10:07:27 +0100 Subject: [PATCH 02/12] get files --- qiita_client/tests/test_qiita_client.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 28bc69e..9e9078f 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -550,6 +550,10 @@ def test_fetch_directory(self): fp_obs = self.tester.fetch_file_from_central(fp_test, prefix=prefix) print("@@@@@@@@@@@STEFAN 2", prefix, fp_test, fp_obs) + from glob import glob + for fp in glob("%s/**/*" % prefix, recursive=True): + print("@@@@@@@@@@@ STEFAN present: %s" % fp) + # test a file of the freshly transferred directory from main has # expected file content with open(join(fp_obs, 'testdir', 'fileA.txt'), 'r') as f: @@ -558,3 +562,15 @@ def test_fetch_directory(self): if __name__ == '__main__': main() + +# # [Errno 2] No such file or directory: +# # '/home/runner/localFetch +# # /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ +# # job/2_test_folder/testdir/fileA.txt' + +# [ +# , +# , +# , +# , +# ] \ No newline at end of file From 7cd6f1643d5bb07fbcee8c102c764fa53cef63f2 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 10:37:09 +0100 Subject: [PATCH 03/12] more debug --- .github/workflows/qiita-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 6d58a2c..ccd751d 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -54,9 +54,9 @@ jobs: # all config files # wget https://github.com/biocore/qiita/archive/dev.zip # unzip dev.zip - wget https://github.com/jlab/qiita/archive/refs/heads/tornado_FetchFileFromCentralHandler.zip - unzip tornado_FetchFileFromCentralHandler.zip - mv qiita-tornado_FetchFileFromCentralHandler qiita-dev + wget https://github.com/jlab/qiita/archive/refs/heads/tornado_FetchFileFromCentralHandler_alsoDirs_debug.zip + unzip tornado_FetchFileFromCentralHandler_alsoDirs_debug.zip + mv qiita-tornado_FetchFileFromCentralHandler_alsoDirs_debug qiita-dev # pull out the port so we can modify the configuration file easily pgport=${{ job.services.postgres.ports[5432] }} @@ -130,7 +130,8 @@ jobs: export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg export QIITA_CLIENT_DEBUG_LEVEL=DEBUG - nosetests --with-doctest --with-coverage --cover-package=qiita_client + nosetests --with-doctest --with-coverage --cover-package=qiita_client; true + cat /tmp/stefan.log - uses: codecov/codecov-action@v3 with: From 9e2baf05bc216ac3fcd79f423c78a18c4b44b88e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 10:43:48 +0100 Subject: [PATCH 04/12] forcer cat --- .github/workflows/qiita-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index ccd751d..114e3b7 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -130,8 +130,7 @@ jobs: export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg export QIITA_CLIENT_DEBUG_LEVEL=DEBUG - nosetests --with-doctest --with-coverage --cover-package=qiita_client; true - cat /tmp/stefan.log + nosetests --with-doctest --with-coverage --cover-package=qiita_client; cat /tmp/stefan.log - uses: codecov/codecov-action@v3 with: From c3cc119c0dd3e965aeb4a21e3ef2c4959cf4116b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 10:54:15 +0100 Subject: [PATCH 05/12] read log here --- qiita_client/tests/test_qiita_client.py | 32 ++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 9e9078f..ddcba6b 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -554,6 +554,9 @@ def test_fetch_directory(self): for fp in glob("%s/**/*" % prefix, recursive=True): print("@@@@@@@@@@@ STEFAN present: %s" % fp) + with open('/tmp/stefan.log') as f: + print("üüüüüüüüüü log from qiita\n %s" % f.readlines()) + # test a file of the freshly transferred directory from main has # expected file content with open(join(fp_obs, 'testdir', 'fileA.txt'), 'r') as f: @@ -573,4 +576,31 @@ def test_fetch_directory(self): # , # , # , -# ] \ No newline at end of file +# ] + +# tinqiita: +# base_data_dir: /qiita_data/ +# prefix: /root/localFetch/ +# deposited file locations +# //job/2_test_folder/testdir/fileA.txt /root/localFetch/qiita_data/job/2_test_folder/testdir/fileA.txt +# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff +# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna +# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log +# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq +# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/fileA.txt +# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff +# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna +# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log +# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq + + +# github: +# base_data_dir: /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ +# prefix: /home/runner/localFetch +# deposited file locations +# //job/2_test_folder/job/2_test_folder/testdir/fileA.txt +# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/fileA.txt +# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirB_l1/fileE.sff +# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/fileB.fna +# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq +# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log \ No newline at end of file From 7b11c640860299e82dac8fec6808a8ff797a85ec Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 10:55:52 +0100 Subject: [PATCH 06/12] debug in client --- qiita_client/qiita_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 9128d4e..39543b3 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -881,6 +881,7 @@ def push_file_to_central(self, filepath): for root, dirnames, filenames in os.walk(filepath): for filename in fnmatch.filter(filenames, "*"): fp = os.path.join(root, filename) + print("äääääääääää '%s' '%s' '%s' '%s' " % (root, dirnames, filenames, os.path.join(dirpath, os.path.dirname(fp)))) self.post('/cloud/push_file_to_central/', files={os.path.join( dirpath, From 13b226aa25d5f2ac5af122100dca1d845972bcee Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 11:22:07 +0100 Subject: [PATCH 07/12] remove cat --- .github/workflows/qiita-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 114e3b7..cf4ba64 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -130,7 +130,7 @@ jobs: export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg export QIITA_CLIENT_DEBUG_LEVEL=DEBUG - nosetests --with-doctest --with-coverage --cover-package=qiita_client; cat /tmp/stefan.log + nosetests --with-doctest --with-coverage --cover-package=qiita_client - uses: codecov/codecov-action@v3 with: From c8eaae62cfb593bbe265a9621713eec95621f61d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 11:51:02 +0100 Subject: [PATCH 08/12] trigger build --- qiita_client/tests/test_qiita_client.py | 96 +++++++++++++++---------- 1 file changed, 58 insertions(+), 38 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index ddcba6b..2c75536 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -566,41 +566,61 @@ def test_fetch_directory(self): if __name__ == '__main__': main() -# # [Errno 2] No such file or directory: -# # '/home/runner/localFetch -# # /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ -# # job/2_test_folder/testdir/fileA.txt' - -# [ -# , -# , -# , -# , -# ] - -# tinqiita: -# base_data_dir: /qiita_data/ -# prefix: /root/localFetch/ -# deposited file locations -# //job/2_test_folder/testdir/fileA.txt /root/localFetch/qiita_data/job/2_test_folder/testdir/fileA.txt -# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff -# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna -# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log -# /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq -# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/fileA.txt -# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff -# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna -# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log -# üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq - - -# github: -# base_data_dir: /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ -# prefix: /home/runner/localFetch -# deposited file locations -# //job/2_test_folder/job/2_test_folder/testdir/fileA.txt -# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/fileA.txt -# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirB_l1/fileE.sff -# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/fileB.fna -# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq -# /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log \ No newline at end of file +# tinqiita +# base_data_dir: /qiita_data/ +# prefix: /root/localFetch +# actual files after fetch +# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/fileA.txt +# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff +# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna +# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log +# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq +# files at Main: +# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/fileA.txt\n', +# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff\n', +# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', +# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n', +# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n'] +# when fetching dir +# , +# , +# , +# , +# ] + +# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/fileA.txt testdir/fileA.txt +# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff testdir/subdirB_l1/fileE.sff +# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna testdir/subdirA_l1/fileB.fna +# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log testdir/subdirA_l1/subdirC_l2/fileC.log +# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq testdir/subdirA_l1/subdirC_l2/fileD.seq\n'] + + + + +# github +# base_data_dir: /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ +# prefix: /home/runner/localFetch +# actual files after fetch +# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/fileA.txt +# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirB_l1/fileE.sff +# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/fileB.fna +# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq +# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log +# files at Main: +# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/fileA.txt\n', +# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff\n', +# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', +# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n', +# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n'] +# when fetching dir +# , +# , +# , +# , +# ] +# all_files + # '- 9 /protected/job/2_test_folder/testdir/fileA.txt job/2_test_folder/testdir/fileA.txt\n', + # '- 5 /protected/job/2_test_folder/testdir/subdirB_l1/fileE.sff job/2_test_folder/testdir/subdirB_l1/fileE.sff\n', + # '- 10 /protected/job/2_test_folder/testdir/subdirA_l1/fileB.fna job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', + # '- 4 /protected/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n', + # '- 10 /protected/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n'] From c560eff91019c228615976631c2416f4e67fe6ed Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 11:56:35 +0100 Subject: [PATCH 09/12] trigger --- qiita_client/tests/test_qiita_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 2c75536..3271532 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -624,3 +624,4 @@ def test_fetch_directory(self): # '- 10 /protected/job/2_test_folder/testdir/subdirA_l1/fileB.fna job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', # '- 4 /protected/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n', # '- 10 /protected/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n'] + From 86f86fbd59d9444f4d09764559963f89ad4b7455 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 12:28:37 +0100 Subject: [PATCH 10/12] trigger --- qiita_client/tests/test_qiita_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 3271532..1a5a063 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -596,7 +596,6 @@ def test_fetch_directory(self): - # github # base_data_dir: /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ # prefix: /home/runner/localFetch From 119e14a1f6cf995fa8ddef544bcfcb2d6ee600ae Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 12:41:29 +0100 Subject: [PATCH 11/12] clean debug infos --- qiita_client/tests/test_qiita_client.py | 69 ------------------------- 1 file changed, 69 deletions(-) diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 1a5a063..ddf843c 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -532,8 +532,6 @@ def test_fetch_directory(self): base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] - print("@@@@@@@@@@@STEFAN", base_data_dir) - # creating a LOCAL test directory within base_data_dir as the DB entry # but no files exist. "job" is the according mountpoint fp_test = join(base_data_dir, 'job', '2_test_folder') @@ -548,14 +546,6 @@ def test_fetch_directory(self): # (=prefix) than it was generated prefix = join(expanduser("~"), 'localFetch') fp_obs = self.tester.fetch_file_from_central(fp_test, prefix=prefix) - print("@@@@@@@@@@@STEFAN 2", prefix, fp_test, fp_obs) - - from glob import glob - for fp in glob("%s/**/*" % prefix, recursive=True): - print("@@@@@@@@@@@ STEFAN present: %s" % fp) - - with open('/tmp/stefan.log') as f: - print("üüüüüüüüüü log from qiita\n %s" % f.readlines()) # test a file of the freshly transferred directory from main has # expected file content @@ -565,62 +555,3 @@ def test_fetch_directory(self): if __name__ == '__main__': main() - -# tinqiita -# base_data_dir: /qiita_data/ -# prefix: /root/localFetch -# actual files after fetch -# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/fileA.txt -# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff -# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna -# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log -# @@@@@@@@@@@ STEFAN present: /root/localFetch/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq -# files at Main: -# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/fileA.txt\n', -# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff\n', -# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', -# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n', -# 'üüüüüüüü qiita filepath=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n'] -# when fetching dir -# , -# , -# , -# , -# ] - -# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/fileA.txt testdir/fileA.txt -# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff testdir/subdirB_l1/fileE.sff -# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna testdir/subdirA_l1/fileB.fna -# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log testdir/subdirA_l1/subdirC_l2/fileC.log -# üüüüüüüü da bin ich baff=/qiita_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq testdir/subdirA_l1/subdirC_l2/fileD.seq\n'] - - - -# github -# base_data_dir: /home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/ -# prefix: /home/runner/localFetch -# actual files after fetch -# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/fileA.txt -# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirB_l1/fileE.sff -# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/fileB.fna -# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq -# @@@@@@@@@@@ STEFAN present: /home/runner/localFetch/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log -# files at Main: -# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/fileA.txt\n', -# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirB_l1/fileE.sff\n', -# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', -# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n', -# 'üüüüüüüü qiita filepath=/home/runner/work/qiita_client/qiita_client/qiita-dev/qiita_db/support_files/test_data/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n'] -# when fetching dir -# , -# , -# , -# , -# ] -# all_files - # '- 9 /protected/job/2_test_folder/testdir/fileA.txt job/2_test_folder/testdir/fileA.txt\n', - # '- 5 /protected/job/2_test_folder/testdir/subdirB_l1/fileE.sff job/2_test_folder/testdir/subdirB_l1/fileE.sff\n', - # '- 10 /protected/job/2_test_folder/testdir/subdirA_l1/fileB.fna job/2_test_folder/testdir/subdirA_l1/fileB.fna\n', - # '- 4 /protected/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileD.seq\n', - # '- 10 /protected/job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log job/2_test_folder/testdir/subdirA_l1/subdirC_l2/fileC.log\n'] - From fa819be7dbb760a867bd70d530e745c64342ca73 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 12:42:48 +0100 Subject: [PATCH 12/12] remove debug infos --- qiita_client/qiita_client.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 39543b3..1c6e5fc 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -817,9 +817,6 @@ def fetch_file_from_central(self, filepath, prefix=None): # check if requested filepath is a single file OR a whole directory if 'Is-Qiita-Directory' in response.headers.keys(): with ZipFile(BytesIO(response.content)) as zf: - import sys - print("ÖÖÖÖÖ in client:\n%s" % zf.filelist, file=sys.stderr) - zf.extractall(path=target_filepath) else: content = response.content @@ -881,7 +878,6 @@ def push_file_to_central(self, filepath): for root, dirnames, filenames in os.walk(filepath): for filename in fnmatch.filter(filenames, "*"): fp = os.path.join(root, filename) - print("äääääääääää '%s' '%s' '%s' '%s' " % (root, dirnames, filenames, os.path.join(dirpath, os.path.dirname(fp)))) self.post('/cloud/push_file_to_central/', files={os.path.join( dirpath,