FTP upload integration tests.

Test various configuration options (ftp_upload_dir, ftp_upload_identifier, ftp_upload_dir_template) and various upload API parameters (space_to_tab, to_posix_lines, auto_decompress).

test/api/test_tools_upload.py

@@ -1,16 +1,16 @@
 from base import api
-
+from base.constants import (
+    ONE_TO_SIX_ON_WINDOWS,
+    ONE_TO_SIX_WITH_SPACES,
+    ONE_TO_SIX_WITH_TABS,
+)
 from base.populators import (
     DatasetPopulator,
     skip_without_datatype,
 )
 
 from galaxy.tools.verify.test_data import TestDataResolver
 
-ONE_TO_SIX_WITH_SPACES = "1 2 3\n4 5 6\n"
-ONE_TO_SIX_WITH_TABS = "1\t2\t3\n4\t5\t6\n"
-ONE_TO_SIX_ON_WINDOWS = "1\t2\t3\r4\t5\t6\r"
-
 
 class ToolsUploadTestCase(api.ApiTestCase):
 

test/base/api.py

@@ -8,14 +8,16 @@
     assert_not_has_keys,
     assert_status_code_is,
 )
-from .api_util import get_master_api_key, get_user_api_key
+from .api_util import (
+    ADMIN_TEST_USER,
+    get_master_api_key,
+    get_user_api_key,
+    OTHER_USER,
+    TEST_USER,
+)
 from .interactor import GalaxyInteractorApi as BaseInteractor
 from .testcase import FunctionalTestCase
 
-TEST_USER = "user@bx.psu.edu"
-ADMIN_TEST_USER = "test@bx.psu.edu"
-DEFAULT_OTHER_USER = "otheruser@bx.psu.edu"  # A second user for API testing.
-
 
 class UsesApiTestCaseMixin:
 
@@ -48,7 +50,7 @@ def _setup_user_get_key(self, email):
         return self._post("users/%s/api_key" % user["id"], admin=True).json()
 
     @contextmanager
-    def _different_user(self, email=DEFAULT_OTHER_USER):
+    def _different_user(self, email=OTHER_USER):
         """ Use in test cases to switch get/post operations to act as new user,
 
             with self._different_user( "other_user@bx.psu.edu" ):

test/base/api_util.py

@@ -3,6 +3,14 @@
 DEFAULT_GALAXY_MASTER_API_KEY = "TEST123"
 DEFAULT_GALAXY_USER_API_KEY = None
 
+DEFAULT_TEST_USER = "user@bx.psu.edu"
+DEFAULT_ADMIN_TEST_USER = "test@bx.psu.edu"
+DEFAULT_OTHER_USER = "otheruser@bx.psu.edu"  # A second user for API testing.
+
+TEST_USER = os.environ.get("GALAXY_TEST_USER_EMAIL", DEFAULT_TEST_USER)
+ADMIN_TEST_USER = os.environ.get("GALAXY_TEST_ADMIN_USER_EMAIL", DEFAULT_ADMIN_TEST_USER)
+OTHER_USER = os.environ.get("GALAXY_TEST_OTHER_USER_EMAIL", DEFAULT_OTHER_USER)
+
 
 def get_master_api_key():
     """ Test master API key to use for functional test. This key should be

test/base/constants.py

@@ -0,0 +1,6 @@
+"""Just constants useful for testing across test types."""
+
+# Following constants used by upload tests.
+ONE_TO_SIX_WITH_SPACES = "1 2 3\n4 5 6\n"
+ONE_TO_SIX_WITH_TABS = "1\t2\t3\n4\t5\t6\n"
+ONE_TO_SIX_ON_WINDOWS = "1\t2\t3\r4\t5\t6\r"

test/base/populators.py

@@ -104,12 +104,14 @@ class BaseDatasetPopulator(object):
     Galaxy - implementations must implement _get and _post.
     """
 
-    def new_dataset(self, history_id, content='TestData123', wait=False, **kwds):
+    def new_dataset(self, history_id, content=None, wait=False, **kwds):
         run_response = self.new_dataset_request(history_id, content=content, wait=wait, **kwds)
         return run_response.json()["outputs"][0]
 
-    def new_dataset_request(self, history_id, content='TestData123', wait=False, **kwds):
-        payload = self.upload_payload(history_id, content, **kwds)
+    def new_dataset_request(self, history_id, content=None, wait=False, **kwds):
+        if content is None and "ftp_files" not in kwds:
+            content = "TestData123"
+        payload = self.upload_payload(history_id, content=content, **kwds)
         run_response = self.tools_post(payload)
         if wait:
             self.wait_for_tool_run(history_id, run_response)
@@ -157,7 +159,7 @@ def new_history(self, **kwds):
         history_id = create_history_response.json()["id"]
         return history_id
 
-    def upload_payload(self, history_id, content, **kwds):
+    def upload_payload(self, history_id, content=None, **kwds):
         name = kwds.get("name", "Test Dataset")
         dbkey = kwds.get("dbkey", "?")
         file_type = kwds.get("file_type", 'txt')
@@ -166,7 +168,9 @@ def upload_payload(self, history_id, content, **kwds):
             'dbkey': dbkey,
             'file_type': file_type,
         }
-        if hasattr(content, 'read'):
+        if content is None:
+            upload_params["files_0|ftp_files"] = kwds.get("ftp_files")
+        elif hasattr(content, 'read'):
             upload_params["files_0|file_data"] = content
         else:
             upload_params['files_0|url_paste'] = content
@@ -185,6 +189,9 @@ def upload_payload(self, history_id, content, **kwds):
             upload_type='upload_dataset'
         )
 
+    def get_remote_files(self, target="ftp"):
+        return self._get("remote_files", data={"target": target}).json()
+
     def run_tool_payload(self, tool_id, inputs, history_id, **kwds):
         if "files_0|file_data" in inputs:
             kwds["__files"] = {"files_0|file_data": inputs["files_0|file_data"]}