diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index 2e65b5c2f..7692d28c7 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -623,9 +623,8 @@ def delete(cls, artifact_id): # move the files to the uploads folder. We also need # to nullify the column in the prep template table if not instance.parents and study is not None: - qdb.util.move_filepaths_to_upload_folder( - study.id, filepaths) - + qdb.util.move_filepaths_to_upload_folder(study.id, + filepaths) # there are cases that an artifact would not be linked to a # study pt_ids = [tuple([pt.id]) for a in all_artifacts diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index 08288c20c..08e4b3eca 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -384,12 +384,13 @@ def test_move_filepaths_to_upload_folder(self): # we are going to test the move_filepaths_to_upload_folder indirectly # by creating an artifact and deleting it. To accomplish this we need # to create a new prep info file, attach a biom with html_summary and - # then deleting it. However, we will do this twice to assure that + # then delete it. However, we will do this twice to assure that # there are no conflicts with this study_id = 1 # creating the 2 sets of files for the 2 artifacts fd, seqs_fp1 = mkstemp(suffix='_seqs.fastq') close(fd) + html_fp1 = mkdtemp() html_fp1 = join(html_fp1, 'support_files') mkdir(html_fp1) @@ -397,12 +398,33 @@ def test_move_filepaths_to_upload_folder(self): fp.write(">AAA\nAAA") fd, seqs_fp2 = mkstemp(suffix='_seqs.fastq') close(fd) + html_fp2 = mkdtemp() html_fp2 = join(html_fp2, 'support_files') mkdir(html_fp2) with open(join(html_fp2, 'index.html'), 'w') as fp: fp.write(">AAA\nAAA") + # create an additional directory named 'more_files' and populate it + # with two files that shouldn't be moved back into the uploads + # directory when an artifact is deleted, and one that should. + more_files_fp = mkdtemp() + more_files_fp = join(more_files_fp, 'more_files') + mkdir(more_files_fp) + test_files = [(join(more_files_fp, x), 1) for x in + ['qtp-sequencing-validate-data.csv', + 'feature-table.qza', + 'good_file.txt']] + + # create the files + for file_name, _ in test_files: + with open(file_name, 'w') as fp: + fp.write("This is a test file.\n") + + # verify that the files exist + for file_name, _ in test_files: + self.assertTrue(exists(file_name)) + # creating new prep info file metadata_dict = { 'SKB8.640193': {'center_name': 'ANL', @@ -422,7 +444,7 @@ def test_move_filepaths_to_upload_folder(self): # inserting artifact 1 artifact1 = qdb.artifact.Artifact.create( - [(seqs_fp1, 1), (html_fp1, 'html_summary')], "FASTQ", + [(seqs_fp1, 1), (html_fp1, 'html_summary')] + test_files, "FASTQ", prep_template=pt1) # inserting artifact 2 artifact2 = qdb.artifact.Artifact.create( @@ -433,8 +455,23 @@ def test_move_filepaths_to_upload_folder(self): filepaths = artifact1.filepaths filepaths.extend(artifact2.filepaths) - # delete artifacts + # delete artifact 1 qdb.artifact.Artifact.delete(artifact1.id) + + # now that artifact 1 is deleted, confirm that all three test files + # are removed. + for file_name, _ in test_files: + self.assertFalse(exists(file_name)) + + # confirm that the 'bad_files' were not moved back into the uploads + # directory, but the 'good_file.txt' was. + uploads = qdb.util.get_files_from_uploads_folders("1") + bad_files = [basename(x[0]) for x in test_files if + 'good_file.txt' not in x[0]] + for _, some_path, _ in uploads: + self.assertFalse(basename(some_path) in bad_files) + + # finish deleting artifacts qdb.artifact.Artifact.delete(artifact2.id) # now let's create another artifact with the same filenames that @@ -455,7 +492,17 @@ def test_move_filepaths_to_upload_folder(self): str(study_id)) for x in filepaths: self.assertFalse(exists(x['fp'])) - new_fp = join(path_for_removal, basename(x['fp'])) + + f_name = basename(x['fp']) + + if f_name in bad_files: + # skip the check for file-names in bad_files, + # because they were already checked above and they were + # already deleted. + continue + + new_fp = join(path_for_removal, f_name) + if x['fp_type'] == 'html_summary': # The html summary gets removed, not moved self.assertFalse(exists(new_fp)) diff --git a/qiita_db/util.py b/qiita_db/util.py index b8410434d..c65921712 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -958,19 +958,26 @@ def move_filepaths_to_upload_folder(study_id, filepaths): path_builder = partial(join, uploads_fp) + # do not move these files back to upload folder. + do_not_move = ['qtp-sequencing-validate-data.csv', 'feature-table.qza'] + # We can now go over and remove all the filepaths sql = """DELETE FROM qiita.filepath WHERE filepath_id = %s""" for x in filepaths: qdb.sql_connection.TRN.add(sql, [x['fp_id']]) - if x['fp_type'] in ('html_summary', 'html_summary_dir'): + if (x['fp_type'] in ('html_summary', + 'html_summary_dir') or + basename(x['fp']) in do_not_move): _rm_files(qdb.sql_connection.TRN, x['fp']) - else: - destination = path_builder(basename(x['fp'])) + continue - qdb.sql_connection.TRN.add_post_rollback_func( - move, destination, x['fp']) - move(x['fp'], destination) + # if files were not removed, then they should be moved. + destination = path_builder(basename(x['fp'])) + qdb.sql_connection.TRN.add_post_rollback_func(move, + destination, + x['fp']) + move(x['fp'], destination) qdb.sql_connection.TRN.execute()