expose base_data_dir in self instead of extra method "deposit..."

qiita_client/qiita_client.py

@@ -12,7 +12,13 @@
 import requests
 import threading
 import pandas as pd
-from json import dumps
+from json import dumps, loads
+try:
+    from json import JSONDecodeError
+except ImportError:
+    # dirty hack to cope with the fact that python 2.7 does not have
+    # JSONDecodeError, but is needed for qp-target-gene plugin
+    JSONDecodeError = ValueError
 from random import randint
 import fnmatch
 from io import BytesIO
@@ -389,7 +395,8 @@ def _request_retry(self, req, url, rettype='json', **kwargs):
             elif r.status_code in (500, 405):
                 raise RuntimeError(
                     "Request '%s %s' did not succeed. Status code: %d. "
-                    "Message: %s" % (req.__name__, url, r.status_code, r.text))
+                    "Message: %s%s" % (req.__name__, url, r.status_code,
+                                       r.text, r.reason))
             elif 0 <= (r.status_code - 200) < 100:
                 try:
                     if rettype is None or rettype == 'json':
@@ -447,7 +454,7 @@ def _fetch_artifact_files(self, ainfo):
         else:
             return ainfo
 
-    def get(self, url, rettype='json', **kwargs):
+    def get(self, url, rettype='json', no_file_fetching=False, **kwargs):
         """Execute a get request against the Qiita server
 
         Parameters
@@ -457,6 +464,12 @@ def get(self, url, rettype='json', **kwargs):
         rettype : string
             The return type of the function, either "json" (default) or
             "object" for the response object itself
+        no_file_fetching : bool
+            If plugin is coupled through none "filesystem" protocols, artifact
+            files will automatically fetched from Qiita central when requesting
+            via "/qiita_db/prep_template/" or "/qiita_db/artifacts/". For
+            testing, you can turn off this behaviour. Handy if e.g. files not
+            yet exists in Qiita central.
         kwargs : dict
             The request kwargs
 
@@ -469,7 +482,8 @@ def get(self, url, rettype='json', **kwargs):
         result = self._request_retry(
             self._session.get, url, rettype=rettype, **kwargs)
 
-        if self._plugincoupling != 'filesystem':
+        if (self._plugincoupling != 'filesystem') and \
+           (no_file_fetching is False):
             # intercept get requests from plugins that request metadata or
             # artifact files and ensure they get transferred from Qiita
             # central, when not using "filesystem"
@@ -567,6 +581,26 @@ def patch(self, url, op, path, value=None, from_p=None, **kwargs):
         # we made sure that data is correctly formatted here
         kwargs['data'] = data
 
+        # similar to above get() injection mechanism, we are here pushing files
+        # to Qiita central, when patching artifact summaries
+        if (self._plugincoupling != 'filesystem') and \
+           (path == '/html_summary/') and (op == 'add'):
+            if re.search(r"/qiita_db/artifacts/\d+/?$", url):
+                if value is not None:
+                    logger.debug('QiitaClient::patch: push summary files to '
+                                 'central: %s' % value)
+                    try:
+                        # values might be an json encoded dictioary with
+                        # multiple filepaths...
+                        dictValues = loads(value)
+                        for ftype in ['html', 'dir']:
+                            if (ftype in dictValues.keys()) and \
+                               (dictValues[ftype] is not None):
+                                self.push_file_to_central(dictValues[ftype])
+                    except (TypeError, JSONDecodeError):
+                        # or just a single string, i.e. filepath
+                        self.push_file_to_central(value)
+
         return self._request_retry(
             self._session.patch, url, rettype='json', **kwargs)
 

qiita_client/testing.py

@@ -7,8 +7,7 @@
 # -----------------------------------------------------------------------------
 
 from unittest import TestCase
-from os import environ, sep
-from os.path import join, isabs
+from os import environ
 from time import sleep
 
 from qiita_client import QiitaClient
@@ -46,6 +45,16 @@ def setUpClass(cls):
         cls.qclient._plugincoupling = environ.get(
             'QIITA_PLUGINCOUPLING', BaseQiitaPlugin._DEFAULT_PLUGIN_COUPLINGS)
 
+        # Determine BASE_DATA_DIR of qiita central, without having direct
+        # access to qiita's settings file. This is done by requesting
+        # information about prep 1, which should be in the test database.
+        # This might break IF file
+        #    qiita-spots/qiita/qiita_db/support_files/populate_test_db.sql
+        # changes.
+        prep_info = cls.qclient.get('/qiita_db/prep_template/1/',
+                                    no_file_fetching=True)
+        cls.base_data_dir = prep_info['prep-file'].split('templates/')[0]
+
         # Give enough time for the plugins to register
         sleep(5)
 
@@ -83,55 +92,3 @@ def _wait_for_running_job(self, job_id):
                 break
 
         return status
-
-    def deposite_in_qiita_basedir(self, fps, update_fp_only=False):
-        """Pushs a file to qiita main AND adapts given filepath accordingly.
-
-        A helper function to fix file paths in tests such that they point to
-        the expected BASE_DATA_DIR. This becomes necessary when uncoupling the
-        plugin filesystem as some methods now actually fetches expected files
-        from BASE_DATA_DIR. This will fail for protocols other than filesystem
-        IF files are created locally by the plugin test.
-
-        Parameters
-        ----------
-        fps : str or [str]
-            Filepath or list of filepaths to file(s) that shall be part of
-            BASE_DATA_DIR, but currently points to some tmp file for testing.
-        update_fp_only : bool
-            Some tests operate on filepaths only - files do not actually need
-            to exist. Thus, we don't need to tranfer a file.
-
-        Returns
-        -------
-        The potentially modified filepaths.
-        """
-        def _stripRoot(fp):
-            # chop off leading / for join to work properly when prepending
-            # the BASE_DATA_DIR
-            if isabs(fp):
-                return fp[len(sep):]
-            return fp
-
-        # use artifact 1 info to determine BASA_DATA_DIR, as we know that the
-        # filepath ends with ....raw_data/1_s_G1_L001_sequences.fastq.gz, thus
-        # BASE_DATA_DIR must be the prefix, e.g. /qiita_data/
-        # This might break IF file
-        #    qiita-spots/qiita/qiita_db/support_files/populate_test_db.sql
-        # changes.
-        ainfo = self.qclient.get('/qiita_db/artifacts/1/')
-        base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][
-            :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))]
-        if isinstance(fps, str):
-            if not update_fp_only:
-                self.qclient.push_file_to_central(fps)
-            return join(base_data_dir, _stripRoot(fps))
-        elif isinstance(fps, list):
-            for fp in fps:
-                if not update_fp_only:
-                    self.qclient.push_file_to_central(fp)
-            return [join(base_data_dir, _stripRoot(fp)) for fp in fps]
-        else:
-            raise ValueError(
-                "deposite_in_qiita_basedir is not implemented for type %s"
-                % type(fps))

qiita_client/tests/test_plugin.py

@@ -75,6 +75,9 @@ def func(a, b, c, d):
 
     def test__push_artifacts_files_to_central(self):
         class fakeClient():
+            def __init__(self):
+                self._plugincoupling = 'null protocol'
+
             def push_file_to_central(self, filepath):
                 return 'pushed:%s' % filepath