diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 24cfe80..0ea317a 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -12,7 +12,13 @@ import requests import threading import pandas as pd -from json import dumps +from json import dumps, loads +try: + from json import JSONDecodeError +except ImportError: + # dirty hack to cope with the fact that python 2.7 does not have + # JSONDecodeError, but is needed for qp-target-gene plugin + JSONDecodeError = ValueError from random import randint import fnmatch from io import BytesIO @@ -389,7 +395,8 @@ def _request_retry(self, req, url, rettype='json', **kwargs): elif r.status_code in (500, 405): raise RuntimeError( "Request '%s %s' did not succeed. Status code: %d. " - "Message: %s" % (req.__name__, url, r.status_code, r.text)) + "Message: %s%s" % (req.__name__, url, r.status_code, + r.text, r.reason)) elif 0 <= (r.status_code - 200) < 100: try: if rettype is None or rettype == 'json': @@ -447,7 +454,7 @@ def _fetch_artifact_files(self, ainfo): else: return ainfo - def get(self, url, rettype='json', **kwargs): + def get(self, url, rettype='json', no_file_fetching=False, **kwargs): """Execute a get request against the Qiita server Parameters @@ -457,6 +464,12 @@ def get(self, url, rettype='json', **kwargs): rettype : string The return type of the function, either "json" (default) or "object" for the response object itself + no_file_fetching : bool + If plugin is coupled through none "filesystem" protocols, artifact + files will automatically fetched from Qiita central when requesting + via "/qiita_db/prep_template/" or "/qiita_db/artifacts/". For + testing, you can turn off this behaviour. Handy if e.g. files not + yet exists in Qiita central. kwargs : dict The request kwargs @@ -469,7 +482,8 @@ def get(self, url, rettype='json', **kwargs): result = self._request_retry( self._session.get, url, rettype=rettype, **kwargs) - if self._plugincoupling != 'filesystem': + if (self._plugincoupling != 'filesystem') and \ + (no_file_fetching is False): # intercept get requests from plugins that request metadata or # artifact files and ensure they get transferred from Qiita # central, when not using "filesystem" @@ -567,6 +581,26 @@ def patch(self, url, op, path, value=None, from_p=None, **kwargs): # we made sure that data is correctly formatted here kwargs['data'] = data + # similar to above get() injection mechanism, we are here pushing files + # to Qiita central, when patching artifact summaries + if (self._plugincoupling != 'filesystem') and \ + (path == '/html_summary/') and (op == 'add'): + if re.search(r"/qiita_db/artifacts/\d+/?$", url): + if value is not None: + logger.debug('QiitaClient::patch: push summary files to ' + 'central: %s' % value) + try: + # values might be an json encoded dictioary with + # multiple filepaths... + dictValues = loads(value) + for ftype in ['html', 'dir']: + if (ftype in dictValues.keys()) and \ + (dictValues[ftype] is not None): + self.push_file_to_central(dictValues[ftype]) + except (TypeError, JSONDecodeError): + # or just a single string, i.e. filepath + self.push_file_to_central(value) + return self._request_retry( self._session.patch, url, rettype='json', **kwargs) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index ef19893..e4ff381 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -7,8 +7,7 @@ # ----------------------------------------------------------------------------- from unittest import TestCase -from os import environ, sep -from os.path import join, isabs +from os import environ from time import sleep from qiita_client import QiitaClient @@ -46,6 +45,16 @@ def setUpClass(cls): cls.qclient._plugincoupling = environ.get( 'QIITA_PLUGINCOUPLING', BaseQiitaPlugin._DEFAULT_PLUGIN_COUPLINGS) + # Determine BASE_DATA_DIR of qiita central, without having direct + # access to qiita's settings file. This is done by requesting + # information about prep 1, which should be in the test database. + # This might break IF file + # qiita-spots/qiita/qiita_db/support_files/populate_test_db.sql + # changes. + prep_info = cls.qclient.get('/qiita_db/prep_template/1/', + no_file_fetching=True) + cls.base_data_dir = prep_info['prep-file'].split('templates/')[0] + # Give enough time for the plugins to register sleep(5) @@ -83,55 +92,3 @@ def _wait_for_running_job(self, job_id): break return status - - def deposite_in_qiita_basedir(self, fps, update_fp_only=False): - """Pushs a file to qiita main AND adapts given filepath accordingly. - - A helper function to fix file paths in tests such that they point to - the expected BASE_DATA_DIR. This becomes necessary when uncoupling the - plugin filesystem as some methods now actually fetches expected files - from BASE_DATA_DIR. This will fail for protocols other than filesystem - IF files are created locally by the plugin test. - - Parameters - ---------- - fps : str or [str] - Filepath or list of filepaths to file(s) that shall be part of - BASE_DATA_DIR, but currently points to some tmp file for testing. - update_fp_only : bool - Some tests operate on filepaths only - files do not actually need - to exist. Thus, we don't need to tranfer a file. - - Returns - ------- - The potentially modified filepaths. - """ - def _stripRoot(fp): - # chop off leading / for join to work properly when prepending - # the BASE_DATA_DIR - if isabs(fp): - return fp[len(sep):] - return fp - - # use artifact 1 info to determine BASA_DATA_DIR, as we know that the - # filepath ends with ....raw_data/1_s_G1_L001_sequences.fastq.gz, thus - # BASE_DATA_DIR must be the prefix, e.g. /qiita_data/ - # This might break IF file - # qiita-spots/qiita/qiita_db/support_files/populate_test_db.sql - # changes. - ainfo = self.qclient.get('/qiita_db/artifacts/1/') - base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][ - :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))] - if isinstance(fps, str): - if not update_fp_only: - self.qclient.push_file_to_central(fps) - return join(base_data_dir, _stripRoot(fps)) - elif isinstance(fps, list): - for fp in fps: - if not update_fp_only: - self.qclient.push_file_to_central(fp) - return [join(base_data_dir, _stripRoot(fp)) for fp in fps] - else: - raise ValueError( - "deposite_in_qiita_basedir is not implemented for type %s" - % type(fps)) diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index e6d28ae..f3010c8 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -75,6 +75,9 @@ def func(a, b, c, d): def test__push_artifacts_files_to_central(self): class fakeClient(): + def __init__(self): + self._plugincoupling = 'null protocol' + def push_file_to_central(self, filepath): return 'pushed:%s' % filepath