From c6f8357f555a7cf500f593630710490223426e02 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 10:22:41 +0100 Subject: [PATCH 1/9] adding debug information --- qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py index 30f7e8ba1..833113109 100644 --- a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py +++ b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py @@ -220,6 +220,9 @@ def post(self): os.makedirs(os.path.dirname(filepath), exist_ok=True) if sent_directory: with zipfile.ZipFile(BytesIO(file['body'])) as zf: + import sys + print("üüüüüüüü qiita filepath=%s" % filepath, file=sys.stderr) + print("üüüüüüüü qiita inventory:%s\n" % zf.filelist, file=sys.stderr) zf.extractall(filepath) stored_directories.append(filepath) else: From 1ff9892012d7efbc00bb66858758b05b64a7eb05 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 10:34:40 +0100 Subject: [PATCH 2/9] move debug to individual files --- qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py index 833113109..298f930e4 100644 --- a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py +++ b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py @@ -220,12 +220,11 @@ def post(self): os.makedirs(os.path.dirname(filepath), exist_ok=True) if sent_directory: with zipfile.ZipFile(BytesIO(file['body'])) as zf: - import sys - print("üüüüüüüü qiita filepath=%s" % filepath, file=sys.stderr) - print("üüüüüüüü qiita inventory:%s\n" % zf.filelist, file=sys.stderr) zf.extractall(filepath) stored_directories.append(filepath) else: + with open("/tmp/stefan.log", "a") as f: + f.write("üüüüüüüü qiita filepath=%s\n" % filepath) with open(filepath, "wb") as f: f.write(file['body']) stored_files.append(filepath) From 4bd4a18d7ee2669e490eed489245e1a93a8ab0a9 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 11:21:23 +0100 Subject: [PATCH 3/9] more debug when composing zip --- qiita_pet/handlers/download.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qiita_pet/handlers/download.py b/qiita_pet/handlers/download.py index 60e029f29..8a9485684 100644 --- a/qiita_pet/handlers/download.py +++ b/qiita_pet/handlers/download.py @@ -150,6 +150,9 @@ def _write_nginx_file_list(self, to_download): for fp, fp_name, fp_checksum, fp_size in to_download]) self.set_header('X-Archive-Files', 'zip') + with open("/tmp/stefan.log", "a") as f: + f.write("üüüüüüüü all_files=%s\n" % all_files) + self.write("%s\n" % all_files) def _set_nginx_headers(self, fname): @@ -298,6 +301,10 @@ def get(self, study_id): self._write_nginx_file_list(to_download) + with open("/logs/download.log", "a") as f: + f.write("DownloadRawData::get(study_id=%s), to_download=%s\n" % (study_id, to_download)) + + zip_fn = 'study_raw_data_%d_%s.zip' % ( study_id, datetime.now().strftime('%m%d%y-%H%M%S')) From 14d43f1231da105f7d44b6f155b7e87238e58ca2 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 11:27:36 +0100 Subject: [PATCH 4/9] debug --- qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py index 298f930e4..8a0a7ea4c 100644 --- a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py +++ b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py @@ -168,6 +168,8 @@ def get(self, requested_filepath): # a whole directory to_download = BaseHandlerDownload._list_dir_files_nginx( self, filepath) + with open("/tmp/stefan.log", "a") as f: + f.write("üüüüüüüü to_download=%s\n" % to_download) BaseHandlerDownload._write_nginx_file_list(self, to_download) BaseHandlerDownload._set_nginx_headers( self, filename_directory) From fc367bbf8a55a3895af17c4394b85e88b99f9e52 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 11:50:28 +0100 Subject: [PATCH 5/9] modify nginx file list for ZIP --- .../handlers/cloud_handlers/file_transfer_handlers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py index 8a0a7ea4c..956e0af6e 100644 --- a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py +++ b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py @@ -145,6 +145,8 @@ def get(self, requested_filepath): # make path in zip file relative rel_path = os.path.relpath(full_path, filepath) zf.write(full_path, rel_path) + with open("/tmp/stefan.log", "a") as f: + f.write("üüüüüüüü da bin ich baff=%s %s\n" % (full_path, rel_path)) memfile.seek(0) self.set_header('Content-Type', 'application/zip') self.set_header('Content-Disposition', @@ -168,6 +170,12 @@ def get(self, requested_filepath): # a whole directory to_download = BaseHandlerDownload._list_dir_files_nginx( self, filepath) + # above function adds filepath to located files, which is + # different from the non-nginx version. Correct here: + to_download = [ + (fp, rel_path(fp_name, filepath), fp_checksum, fp_size) + for fp, fp_name, fp_checksum, fp_size + in to_download] with open("/tmp/stefan.log", "a") as f: f.write("üüüüüüüü to_download=%s\n" % to_download) BaseHandlerDownload._write_nginx_file_list(self, to_download) From 043c6beec29c3bd3edecdc4f5592db3b537415b3 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 11:55:57 +0100 Subject: [PATCH 6/9] use correct function --- qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py index 956e0af6e..e425fe217 100644 --- a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py +++ b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py @@ -173,7 +173,7 @@ def get(self, requested_filepath): # above function adds filepath to located files, which is # different from the non-nginx version. Correct here: to_download = [ - (fp, rel_path(fp_name, filepath), fp_checksum, fp_size) + (fp, os.path.relpath(fp_name, filepath), fp_checksum, fp_size) for fp, fp_name, fp_checksum, fp_size in to_download] with open("/tmp/stefan.log", "a") as f: From 3cb9cf786691dd3429bcbf9ecd2333852da876df Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 12:28:12 +0100 Subject: [PATCH 7/9] fix path computation --- .../handlers/cloud_handlers/file_transfer_handlers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py index e425fe217..c1b9ac059 100644 --- a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py +++ b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py @@ -170,10 +170,16 @@ def get(self, requested_filepath): # a whole directory to_download = BaseHandlerDownload._list_dir_files_nginx( self, filepath) + + # fp_subdir is the part of the filepath the user requested, + # without QIITA_BASE_DIR + fp_subdir = os.path.relpath(filepath, basedatadir) + # above function adds filepath to located files, which is # different from the non-nginx version. Correct here: to_download = [ - (fp, os.path.relpath(fp_name, filepath), fp_checksum, fp_size) + (fp, os.path.relpath(fp_name, fp_subdir), fp_checksum, + fp_size) for fp, fp_name, fp_checksum, fp_size in to_download] with open("/tmp/stefan.log", "a") as f: From 2cc787dd15b3030b76addcbb431586938d298450 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 12:36:57 +0100 Subject: [PATCH 8/9] clean up --- .../cloud_handlers/file_transfer_handlers.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py index c1b9ac059..74f994c79 100644 --- a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py +++ b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py @@ -145,8 +145,6 @@ def get(self, requested_filepath): # make path in zip file relative rel_path = os.path.relpath(full_path, filepath) zf.write(full_path, rel_path) - with open("/tmp/stefan.log", "a") as f: - f.write("üüüüüüüü da bin ich baff=%s %s\n" % (full_path, rel_path)) memfile.seek(0) self.set_header('Content-Type', 'application/zip') self.set_header('Content-Disposition', @@ -176,14 +174,19 @@ def get(self, requested_filepath): fp_subdir = os.path.relpath(filepath, basedatadir) # above function adds filepath to located files, which is - # different from the non-nginx version. Correct here: + # different from the non-nginx version, e.g. + # fp = /protected/job/2_test_folder/testdir/fileA.txt + # fp_name = job/2_test_folder/testdir/fileA.txt + # where "job/2_test_folder" is what user requested and + # "testdir/fileA.txt" is a file within this directory. + # When extracting by qiita_client, the "job/2_test_folder" + # part would be added twice (one by user request, second by + # unzipping). Therefore, we need to correct these names here: to_download = [ (fp, os.path.relpath(fp_name, fp_subdir), fp_checksum, fp_size) for fp, fp_name, fp_checksum, fp_size in to_download] - with open("/tmp/stefan.log", "a") as f: - f.write("üüüüüüüü to_download=%s\n" % to_download) BaseHandlerDownload._write_nginx_file_list(self, to_download) BaseHandlerDownload._set_nginx_headers( self, filename_directory) @@ -239,8 +242,6 @@ def post(self): zf.extractall(filepath) stored_directories.append(filepath) else: - with open("/tmp/stefan.log", "a") as f: - f.write("üüüüüüüü qiita filepath=%s\n" % filepath) with open(filepath, "wb") as f: f.write(file['body']) stored_files.append(filepath) From 94417720f30fc46ac32c48657cc582b78c9c76d0 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 7 Nov 2025 12:38:31 +0100 Subject: [PATCH 9/9] clean download.py --- qiita_pet/handlers/download.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/qiita_pet/handlers/download.py b/qiita_pet/handlers/download.py index 8a9485684..60e029f29 100644 --- a/qiita_pet/handlers/download.py +++ b/qiita_pet/handlers/download.py @@ -150,9 +150,6 @@ def _write_nginx_file_list(self, to_download): for fp, fp_name, fp_checksum, fp_size in to_download]) self.set_header('X-Archive-Files', 'zip') - with open("/tmp/stefan.log", "a") as f: - f.write("üüüüüüüü all_files=%s\n" % all_files) - self.write("%s\n" % all_files) def _set_nginx_headers(self, fname): @@ -301,10 +298,6 @@ def get(self, study_id): self._write_nginx_file_list(to_download) - with open("/logs/download.log", "a") as f: - f.write("DownloadRawData::get(study_id=%s), to_download=%s\n" % (study_id, to_download)) - - zip_fn = 'study_raw_data_%d_%s.zip' % ( study_id, datetime.now().strftime('%m%d%y-%H%M%S'))