Uncouple clientpush

qtp_job_output_folder/summary.py

@@ -16,7 +16,7 @@ def _folder_listing(folder):
     for f in glob(f'{folder}/*'):
         if isdir(f):
             results.append(('folder', f))
-            results.extend(_folder_listing(f'{f}/*'))
+            results.extend(_folder_listing(f))
         else:
             results.append(('file', f))
     return results

qtp_job_output_folder/tests/test_plugin.py

@@ -23,10 +23,19 @@
 class PluginTests(PluginTestCase):
     def setUp(self):
         self.out_dir = mkdtemp()
-        self.source_dir = join(mkdtemp(), 'test_data')
+        # need to refer to a valid mountpoint (here: job) and a directory
+        # that exists in Qiita DB as 'directory' as fetching via https will
+        # fail otherwise. To avoid collisions, a mid-level directory name is
+        # a random string (= _get_candidate_names())
+        self.mountpoint = 'job'
+        # only adapt filepaths
+        self.source_dir = join(self.base_data_dir, self.mountpoint,
+                               '2_test_folder')
         source = join(dirname(abspath(getfile(currentframe()))), 'test_data')
         copytree(source, self.source_dir)
-        self._clean_up_files = [self.out_dir]
+        self.qclient.push_file_to_central(self.source_dir)
+        self._clean_up_files = [self.out_dir, dirname(self.source_dir)]
+        self._clean_up_remote_files = []
 
     def tearDown(self):
         for fp in self._clean_up_files:
@@ -35,6 +44,8 @@ def tearDown(self):
                     rmtree(fp)
                 else:
                     remove(fp)
+        for fp in self._clean_up_remote_files:
+            self.qclient.delete_file_from_central(fp)
 
     def _wait_job(self, job_id):
         for i in range(20):
@@ -47,7 +58,8 @@ def _wait_job(self, job_id):
     def test_plugin_summary(self):
         # creating new artifact
         files = [(self.source_dir, 'directory')]
-        data = {'filepaths': dumps(files), 'type': 'job-output-folder',
+        atype = 'job-output-folder'
+        data = {'filepaths': dumps(files), 'type': atype,
                 'name': "A name", 'data_type': 'Job Output Folder'}
         aid = self.qclient.post('/apitest/artifact/', data=data)['artifact']
         data = {'command': dumps(['qtp-job-output-folder', __version__,
@@ -57,17 +69,22 @@ def test_plugin_summary(self):
         job_id = self.qclient.post(
             '/apitest/processing_job/', data=data)['job']
         plugin("https://localhost:21174", job_id, self.out_dir)
+        fp_target_dir = '%s/%s' % (self.source_dir.split(
+            '/%s/' % self.mountpoint)[0], atype)
+        self._clean_up_remote_files.append(fp_target_dir)
+        self._clean_up_files.append(fp_target_dir)
         self._wait_job(job_id)
         obs = self.qclient.get_job_info(job_id)
         self.assertEqual(obs['status'], 'success')
 
     def test_plugin_validate(self):
         # test success
         files = {'directory': [self.source_dir]}
+        atype = 'job-output-folder'
         parameters = {'template': None,
                       'analysis': None,
                       'files': dumps(files),
-                      'artifact_type': 'job-output-folder'}
+                      'artifact_type': atype}
         data = {
             'command': dumps(
                 ['qtp-job-output-folder', __version__, 'Validate']),
@@ -76,6 +93,10 @@ def test_plugin_validate(self):
         job_id = self.qclient.post(
             '/apitest/processing_job/', data=data)['job']
         plugin("https://localhost:21174", job_id, self.out_dir)
+        fp_target_dir = '%s/%s' % (self.source_dir.split(
+            '/%s/' % self.mountpoint)[0], atype)
+        self._clean_up_remote_files.append(fp_target_dir)
+        self._clean_up_files.append(fp_target_dir)
         self._wait_job(job_id)
         obs = self.qclient.get_job_info(job_id)
         self.assertEqual(obs['status'], 'success')

qtp_job_output_folder/tests/test_summary.py

@@ -23,10 +23,20 @@
 class SummaryTests(PluginTestCase):
     def setUp(self):
         self.out_dir = mkdtemp()
-        self.source_dir = join(mkdtemp(), 'test_data')
+        # need to refer to a valid mountpoint (here: job) and a directory
+        # that exists in Qiita DB as 'directory' as fetching via https will
+        # fail otherwise. To avoid collisions, a mid-level directory name is
+        # a random string (= _get_candidate_names())
+        self.mountpoint = 'job'
+        # only adapt filepaths
+        self.exp_dirname = '2_test_folder'
+        self.source_dir = join(self.base_data_dir, self.mountpoint,
+                               self.exp_dirname)
         source = join(dirname(abspath(getfile(currentframe()))), 'test_data')
         copytree(source, self.source_dir)
-        self._clean_up_files = [self.out_dir]
+        self.qclient.push_file_to_central(self.source_dir)
+        self._clean_up_files = [self.out_dir, dirname(self.source_dir)]
+        self._clean_up_remote_files = []
 
     def tearDown(self):
         for fp in self._clean_up_files:
@@ -35,10 +45,13 @@ def tearDown(self):
                     rmtree(fp)
                 else:
                     remove(fp)
+        for fp in self._clean_up_remote_files:
+            self.qclient.delete_file_from_central(fp)
 
     def test_summary(self):
         files = [(self.source_dir, 'directory')]
-        data = {'filepaths': dumps(files), 'type': 'job-output-folder',
+        atype = 'job-output-folder'
+        data = {'filepaths': dumps(files), 'type': atype,
                 'name': "A name", 'data_type': 'Job Output Folder'}
         aid = self.qclient.post('/apitest/artifact/', data=data)['artifact']
         parameters = {'input_data': aid}
@@ -52,6 +65,10 @@ def test_summary(self):
         # Run the test
         obs_success, obs_ainfo, obs_error = generate_html_summary(
             self.qclient, job_id, parameters, self.out_dir)
+        fp_target_dir = '%s/%s' % (self.source_dir.split(
+            '/%s/' % self.mountpoint)[0], atype)
+        self._clean_up_remote_files.append(fp_target_dir)
+        self._clean_up_files.append(fp_target_dir)
 
         # asserting reply
         self.assertTrue(obs_success)
@@ -63,33 +80,33 @@ def test_summary(self):
         # cleaning artifact files, to avoid errors
         [self._clean_up_files.extend([ff['filepath']])
          for f in res['files'].values() for ff in f]
-        html_fp = res['files']['html_summary'][0]['filepath']
+        html_fp = self.qclient.fetch_file_from_central(
+            res['files']['html_summary'][0]['filepath'])
         with open(html_fp) as html_f:
             html = html_f.read()
-
         self.assertCountEqual(
             sorted(html.replace('<br/>', '').split('\n')),
-            sorted(EXP_HTML.format(aid=aid).replace('<br/>', '').split('\n')))
+            sorted(EXP_HTML.format(
+                aid=aid,
+                dir=self.exp_dirname).replace('<br/>', '').split('\n')))
 
 
 EXP_HTML = (
-    '<a href="./{aid}/test_data/folder_a" type="folder" target="_blank">'
-    'test_data/folder_a</a><br/>\n'
-    '<a href="./{aid}/test_data/folder_a/folder_b/folder_c" type="folder" '
-    'target="_blank">test_data/folder_a/folder_b/folder_c</a><br/>\n'
-    '<a href="./{aid}/test_data/file_2" type="file" target="_blank">'
-    'test_data/file_2</a><br/>\n'
-    '<a href="./{aid}/test_data/file_1" type="file" target="_blank">'
-    'test_data/file_1</a><br/>\n'
-    '<a href="./{aid}/test_data/test_data" type="folder" target="_blank">'
-    'test_data/test_data</a><br/>\n'
-    '<a href="./{aid}/test_data/test_data/folder_a/folder_b" type="folder" '
-    'target="_blank">test_data/test_data/folder_a/folder_b</a><br/>\n'
-    '<a href="./{aid}/test_data/test_data/folder_a/folder_b/folder_c/file_c" '
-    'type="file" target="_blank">test_data/test_data/folder_a/folder_b/'
+    '<a href="./{aid}/{dir}/folder_a" type="folder" target="_blank">'
+    '{dir}/folder_a</a><br/>\n'
+    '<a href="./{aid}/{dir}/folder_a/folder_b/folder_c" type="folder" '
+    'target="_blank">{dir}/folder_a/folder_b/folder_c</a><br/>\n'
+    '<a href="./{aid}/{dir}/file_2" type="file" target="_blank">'
+    '{dir}/file_2</a><br/>\n'
+    '<a href="./{aid}/{dir}/file_1" type="file" target="_blank">'
+    '{dir}/file_1</a><br/>\n'
+    '<a href="./{aid}/{dir}/folder_a/folder_b" type="folder" '
+    'target="_blank">{dir}/folder_a/folder_b</a><br/>\n'
+    '<a href="./{aid}/{dir}/folder_a/folder_b/folder_c/file_c" '
+    'type="file" target="_blank">{dir}/folder_a/folder_b/'
     'folder_c/file_c</a><br/>\n'
-    '<a href="./{aid}/test_data/test_data/folder_a/file_a" type="file" '
-    'target="_blank">test_data/test_data/folder_a/file_a</a>')
+    '<a href="./{aid}/{dir}/folder_a/file_a" type="file" '
+    'target="_blank">{dir}/folder_a/file_a</a>')
 
 
 if __name__ == '__main__':

qtp_job_output_folder/tests/test_validate.py

@@ -25,10 +25,17 @@
 class ValidateTests(PluginTestCase):
     def setUp(self):
         self.out_dir = mkdtemp()
-        self.source_dir = join(mkdtemp(), 'test_data')
+        # need to refer to a valid mountpoint (here: job) and a directory
+        # that exists in Qiita DB as 'directory' as fetching via https will
+        # fail otherwise
+        self.mountpoint = 'job'
+        self.source_dir = join(self.base_data_dir, self.mountpoint,
+                               '2_test_folder')
         source = join(dirname(abspath(getfile(currentframe()))), 'test_data')
         copytree(source, self.source_dir)
-        self._clean_up_files = [self.out_dir]
+        self.qclient.push_file_to_central(self.source_dir)
+        self._clean_up_files = [self.out_dir, dirname(self.source_dir)]
+        self._clean_up_remote_files = []
 
     def tearDown(self):
         for fp in self._clean_up_files:
@@ -37,13 +44,16 @@ def tearDown(self):
                     rmtree(fp)
                 else:
                     remove(fp)
+        for fp in self._clean_up_remote_files:
+            self.qclient.delete_file_from_central(fp)
 
     def test_validate(self):
         files = {'directory': [self.source_dir]}
+        atype = 'job-output-folder'
         parameters = {'template': None,
                       'analysis': None,
                       'files': dumps(files),
-                      'artifact_type': 'job-output-folder'}
+                      'artifact_type': atype}
         data = {
             'command': dumps(
                 ['qtp-job-output-folder', __version__, 'Validate']),
@@ -52,15 +62,16 @@ def test_validate(self):
         job_id = self.qclient.post(
             '/apitest/processing_job/', data=data)['job']
 
-        out_dir = mkdtemp()
-        self._clean_up_files.append(out_dir)
         obs_success, obs_ainfo, obs_error = validate(
-            self.qclient, job_id, parameters, out_dir)
-
+            self.qclient, job_id, parameters, self.out_dir)
+        for suffix in [atype, self.mountpoint]:
+            self._clean_up_remote_files.append(
+                '%s/%s' % (self.source_dir.split('/%s/' % self.mountpoint)[0],
+                           suffix))
         self.assertTrue(obs_success)
         filepaths = [
             (f'{self.source_dir}', 'directory'),
-            (f'{out_dir}/summary.html', 'html_summary')]
+            (f'{self.out_dir}/summary.html', 'html_summary')]
         exp = [ArtifactInfo(None, 'job-output-folder', filepaths)]
         self.assertEqual(obs_ainfo, exp)
         self.assertEqual(obs_error, "")

scripts/configure_qtp_job_output_folder

@@ -17,11 +17,13 @@ from qtp_job_output_folder import plugin
 @click.option('--env-script', prompt='Environment script',
               default='conda activate qtp-job-output-folder')
 @click.option('--ca-cert', prompt='Server certificate', default='None')
-def config(env_script, ca_cert):
+@click.argument('plugincoupling', required=False, default='filesystem')
+def config(env_script, ca_cert, plugincoupling):
     """Generates the Qiita configuration files"""
     if ca_cert == 'None':
         ca_cert = None
-    plugin.generate_config(env_script, 'start_qtp_job_output_folder', ca_cert)
+    plugin.generate_config(env_script, 'start_qtp_job_output_folder', ca_cert,
+                           plugin_coupling=plugincoupling)
 
 
 if __name__ == '__main__':