diff --git a/qtp_job_output_folder/summary.py b/qtp_job_output_folder/summary.py index 3892ff1..c9489a2 100644 --- a/qtp_job_output_folder/summary.py +++ b/qtp_job_output_folder/summary.py @@ -16,7 +16,7 @@ def _folder_listing(folder): for f in glob(f'{folder}/*'): if isdir(f): results.append(('folder', f)) - results.extend(_folder_listing(f'{f}/*')) + results.extend(_folder_listing(f)) else: results.append(('file', f)) return results diff --git a/qtp_job_output_folder/tests/test_plugin.py b/qtp_job_output_folder/tests/test_plugin.py index bc30158..28c464f 100644 --- a/qtp_job_output_folder/tests/test_plugin.py +++ b/qtp_job_output_folder/tests/test_plugin.py @@ -23,10 +23,19 @@ class PluginTests(PluginTestCase): def setUp(self): self.out_dir = mkdtemp() - self.source_dir = join(mkdtemp(), 'test_data') + # need to refer to a valid mountpoint (here: job) and a directory + # that exists in Qiita DB as 'directory' as fetching via https will + # fail otherwise. To avoid collisions, a mid-level directory name is + # a random string (= _get_candidate_names()) + self.mountpoint = 'job' + # only adapt filepaths + self.source_dir = self.deposite_in_qiita_basedir( + join(self.mountpoint, '2_test_folder'), True) source = join(dirname(abspath(getfile(currentframe()))), 'test_data') copytree(source, self.source_dir) - self._clean_up_files = [self.out_dir] + self.qclient.push_file_to_central(self.source_dir) + self._clean_up_files = [self.out_dir, dirname(self.source_dir)] + self._clean_up_remote_files = [] def tearDown(self): for fp in self._clean_up_files: @@ -35,6 +44,8 @@ def tearDown(self): rmtree(fp) else: remove(fp) + for fp in self._clean_up_remote_files: + self.qclient.delete_file_from_central(fp) def _wait_job(self, job_id): for i in range(20): @@ -47,7 +58,8 @@ def _wait_job(self, job_id): def test_plugin_summary(self): # creating new artifact files = [(self.source_dir, 'directory')] - data = {'filepaths': dumps(files), 'type': 'job-output-folder', + atype = 'job-output-folder' + data = {'filepaths': dumps(files), 'type': atype, 'name': "A name", 'data_type': 'Job Output Folder'} aid = self.qclient.post('/apitest/artifact/', data=data)['artifact'] data = {'command': dumps(['qtp-job-output-folder', __version__, @@ -57,6 +69,10 @@ def test_plugin_summary(self): job_id = self.qclient.post( '/apitest/processing_job/', data=data)['job'] plugin("https://localhost:21174", job_id, self.out_dir) + fp_target_dir = '%s/%s' % (self.source_dir.split( + '/%s/' % self.mountpoint)[0], atype) + self._clean_up_remote_files.append(fp_target_dir) + self._clean_up_files.append(fp_target_dir) self._wait_job(job_id) obs = self.qclient.get_job_info(job_id) self.assertEqual(obs['status'], 'success') @@ -64,10 +80,11 @@ def test_plugin_summary(self): def test_plugin_validate(self): # test success files = {'directory': [self.source_dir]} + atype = 'job-output-folder' parameters = {'template': None, 'analysis': None, 'files': dumps(files), - 'artifact_type': 'job-output-folder'} + 'artifact_type': atype} data = { 'command': dumps( ['qtp-job-output-folder', __version__, 'Validate']), @@ -76,6 +93,10 @@ def test_plugin_validate(self): job_id = self.qclient.post( '/apitest/processing_job/', data=data)['job'] plugin("https://localhost:21174", job_id, self.out_dir) + fp_target_dir = '%s/%s' % (self.source_dir.split( + '/%s/' % self.mountpoint)[0], atype) + self._clean_up_remote_files.append(fp_target_dir) + self._clean_up_files.append(fp_target_dir) self._wait_job(job_id) obs = self.qclient.get_job_info(job_id) self.assertEqual(obs['status'], 'success') diff --git a/qtp_job_output_folder/tests/test_summary.py b/qtp_job_output_folder/tests/test_summary.py index a4aae89..137e7f6 100644 --- a/qtp_job_output_folder/tests/test_summary.py +++ b/qtp_job_output_folder/tests/test_summary.py @@ -23,10 +23,20 @@ class SummaryTests(PluginTestCase): def setUp(self): self.out_dir = mkdtemp() - self.source_dir = join(mkdtemp(), 'test_data') + # need to refer to a valid mountpoint (here: job) and a directory + # that exists in Qiita DB as 'directory' as fetching via https will + # fail otherwise. To avoid collisions, a mid-level directory name is + # a random string (= _get_candidate_names()) + self.mountpoint = 'job' + # only adapt filepaths + self.exp_dirname = '2_test_folder' + self.source_dir = self.deposite_in_qiita_basedir( + join(self.mountpoint, self.exp_dirname), True) source = join(dirname(abspath(getfile(currentframe()))), 'test_data') copytree(source, self.source_dir) - self._clean_up_files = [self.out_dir] + self.qclient.push_file_to_central(self.source_dir) + self._clean_up_files = [self.out_dir, dirname(self.source_dir)] + self._clean_up_remote_files = [] def tearDown(self): for fp in self._clean_up_files: @@ -35,10 +45,13 @@ def tearDown(self): rmtree(fp) else: remove(fp) + for fp in self._clean_up_remote_files: + self.qclient.delete_file_from_central(fp) def test_summary(self): files = [(self.source_dir, 'directory')] - data = {'filepaths': dumps(files), 'type': 'job-output-folder', + atype = 'job-output-folder' + data = {'filepaths': dumps(files), 'type': atype, 'name': "A name", 'data_type': 'Job Output Folder'} aid = self.qclient.post('/apitest/artifact/', data=data)['artifact'] parameters = {'input_data': aid} @@ -52,6 +65,10 @@ def test_summary(self): # Run the test obs_success, obs_ainfo, obs_error = generate_html_summary( self.qclient, job_id, parameters, self.out_dir) + fp_target_dir = '%s/%s' % (self.source_dir.split( + '/%s/' % self.mountpoint)[0], atype) + self._clean_up_remote_files.append(fp_target_dir) + self._clean_up_files.append(fp_target_dir) # asserting reply self.assertTrue(obs_success) @@ -63,33 +80,33 @@ def test_summary(self): # cleaning artifact files, to avoid errors [self._clean_up_files.extend([ff['filepath']]) for f in res['files'].values() for ff in f] - html_fp = res['files']['html_summary'][0]['filepath'] + html_fp = self.qclient.fetch_file_from_central( + res['files']['html_summary'][0]['filepath']) with open(html_fp) as html_f: html = html_f.read() - self.assertCountEqual( sorted(html.replace('
', '').split('\n')), - sorted(EXP_HTML.format(aid=aid).replace('
', '').split('\n'))) + sorted(EXP_HTML.format( + aid=aid, + dir=self.exp_dirname).replace('
', '').split('\n'))) EXP_HTML = ( - '' - 'test_data/folder_a
\n' - 'test_data/folder_a/folder_b/folder_c
\n' - '' - 'test_data/file_2
\n' - '' - 'test_data/file_1
\n' - '' - 'test_data/test_data
\n' - 'test_data/test_data/folder_a/folder_b
\n' - 'test_data/test_data/folder_a/folder_b/' + '' + '{dir}/folder_a
\n' + '{dir}/folder_a/folder_b/folder_c
\n' + '' + '{dir}/file_2
\n' + '' + '{dir}/file_1
\n' + '{dir}/folder_a/folder_b
\n' + '{dir}/folder_a/folder_b/' 'folder_c/file_c
\n' - 'test_data/test_data/folder_a/file_a') + '{dir}/folder_a/file_a') if __name__ == '__main__': diff --git a/qtp_job_output_folder/tests/test_validate.py b/qtp_job_output_folder/tests/test_validate.py index 0c67e84..f333170 100644 --- a/qtp_job_output_folder/tests/test_validate.py +++ b/qtp_job_output_folder/tests/test_validate.py @@ -25,10 +25,17 @@ class ValidateTests(PluginTestCase): def setUp(self): self.out_dir = mkdtemp() - self.source_dir = join(mkdtemp(), 'test_data') + # need to refer to a valid mountpoint (here: job) and a directory + # that exists in Qiita DB as 'directory' as fetching via https will + # fail otherwise + self.mountpoint = 'job' + self.source_dir = self.deposite_in_qiita_basedir( + join(self.mountpoint, '2_test_folder'), True) source = join(dirname(abspath(getfile(currentframe()))), 'test_data') copytree(source, self.source_dir) - self._clean_up_files = [self.out_dir] + self.qclient.push_file_to_central(self.source_dir) + self._clean_up_files = [self.out_dir, dirname(self.source_dir)] + self._clean_up_remote_files = [] def tearDown(self): for fp in self._clean_up_files: @@ -37,13 +44,16 @@ def tearDown(self): rmtree(fp) else: remove(fp) + for fp in self._clean_up_remote_files: + self.qclient.delete_file_from_central(fp) def test_validate(self): files = {'directory': [self.source_dir]} + atype = 'job-output-folder' parameters = {'template': None, 'analysis': None, 'files': dumps(files), - 'artifact_type': 'job-output-folder'} + 'artifact_type': atype} data = { 'command': dumps( ['qtp-job-output-folder', __version__, 'Validate']), @@ -52,15 +62,16 @@ def test_validate(self): job_id = self.qclient.post( '/apitest/processing_job/', data=data)['job'] - out_dir = mkdtemp() - self._clean_up_files.append(out_dir) obs_success, obs_ainfo, obs_error = validate( - self.qclient, job_id, parameters, out_dir) - + self.qclient, job_id, parameters, self.out_dir) + for suffix in [atype, self.mountpoint]: + self._clean_up_remote_files.append( + '%s/%s' % (self.source_dir.split('/%s/' % self.mountpoint)[0], + suffix)) self.assertTrue(obs_success) filepaths = [ (f'{self.source_dir}', 'directory'), - (f'{out_dir}/summary.html', 'html_summary')] + (f'{self.out_dir}/summary.html', 'html_summary')] exp = [ArtifactInfo(None, 'job-output-folder', filepaths)] self.assertEqual(obs_ainfo, exp) self.assertEqual(obs_error, "") diff --git a/scripts/configure_qtp_job_output_folder b/scripts/configure_qtp_job_output_folder index 815b728..2be7c26 100755 --- a/scripts/configure_qtp_job_output_folder +++ b/scripts/configure_qtp_job_output_folder @@ -17,11 +17,13 @@ from qtp_job_output_folder import plugin @click.option('--env-script', prompt='Environment script', default='conda activate qtp-job-output-folder') @click.option('--ca-cert', prompt='Server certificate', default='None') -def config(env_script, ca_cert): +@click.argument('plugincoupling', required=False, default='filesystem') +def config(env_script, ca_cert, plugincoupling): """Generates the Qiita configuration files""" if ca_cert == 'None': ca_cert = None - plugin.generate_config(env_script, 'start_qtp_job_output_folder', ca_cert) + plugin.generate_config(env_script, 'start_qtp_job_output_folder', ca_cert, + plugin_coupling=plugincoupling) if __name__ == '__main__':