test looper runp, add test files

.gitignore

@@ -2,7 +2,6 @@
 oldtests/test/*
 
 # toy/experimental files
-*.tsv
 *.pkl
 
 # ignore eggs

requirements/requirements-test.txt

@@ -1,3 +1,3 @@
-coveralls==1.1
-pytest-cov==2.4.0
+coveralls>=1.1
+pytest-cov==2.6.1
 pytest-remotedata

tests/data/example_peps/example_piface/pipeline_interface1.yaml

@@ -11,7 +11,7 @@ project_pipeline:
   path: pipelines/col_pipeline1.py
   output_schema: output_schema.yaml
   command_template: >
-    {pipeline.path} --sample-name {sample.sample_name}
+    {pipeline.path} --project-name {project.name}
   bioconductor:
     readFunName: readData
     readFunPath: readData.R

tests/data/example_peps/example_piface/pipeline_interface2.yaml

@@ -5,13 +5,16 @@ sample_pipeline:
   output_schema: output_schema.yaml
   command_template: >
     {pipeline.path} --sample-name {sample.sample_name}
-
+  compute:
+    size_dependent_variables: resources-sample.tsv
 project_pipeline:
   name: OTHER_PIPELINE2
   path: pipelines/col_pipeline2.py
   output_schema: output_schema.yaml
   command_template: >
-    {pipeline.path} --sample-name {sample.sample_name}
+    {pipeline.path} --project-name {project.name}
+  compute:
+    size_dependent_variables: resources-project.tsv
 
 bioconductor:
   readFunName: readData

tests/data/example_peps/example_piface/resources-project.tsv

@@ -0,0 +1,6 @@
+max_file_size	cores	mem	time
+0.05	1	12000	00-01:00:00
+0.5	1	16000	00-01:00:00
+1	1	16000	00-01:00:00
+10	1	16000	00-01:00:00
+NaN	1	32000	00-02:00:00

tests/data/example_peps/example_piface/resources-sample.tsv

@@ -0,0 +1,7 @@
+max_file_size	cores	mem	time
+0.001	1	8000	00-04:00:00
+0.05	2	12000	00-08:00:00
+0.5	4	16000	00-12:00:00
+1	8	16000	00-24:00:00
+10	16	32000	02-00:00:00
+NaN	32	32000	04-00:00:00

tests/smoketests/conftest.py

@@ -7,6 +7,7 @@
 ST = "annotation_sheet.csv"
 PI = "pipeline_interface{}.yaml"
 OS = "output_schema.yaml"
+RES = "resources-{}.tsv"
 
 @pytest.fixture
 def example_pep_piface_path():
@@ -30,16 +31,22 @@ def prep_temp_pep(example_pep_piface_path):
     piface1_path = os.path.join(example_pep_piface_path, PI.format("1"))
     piface2_path = os.path.join(example_pep_piface_path, PI.format("2"))
     schema_path = os.path.join(example_pep_piface_path, OS)
+    res_proj_path = os.path.join(example_pep_piface_path, RES.format("project"))
+    res_samp_path = os.path.join(example_pep_piface_path, RES.format("sample"))
     # temp copies
     temp_path_cfg = os.path.join(td, CFG)
     temp_path_sample_table = os.path.join(td, ST)
     temp_path_piface1 = os.path.join(td, PI.format("1"))
     temp_path_piface2 = os.path.join(td, PI.format("2"))
     temp_path_schema = os.path.join(td, OS)
+    temp_path_res_proj = os.path.join(td, RES.format("project"))
+    temp_path_res_samp = os.path.join(td, RES.format("sample"))
     # copying
     cpf(cfg_path, temp_path_cfg)
     cpf(sample_table_path, temp_path_sample_table)
     cpf(piface1_path, temp_path_piface1)
     cpf(piface2_path, temp_path_piface2)
     cpf(schema_path, temp_path_schema)
+    cpf(res_proj_path, temp_path_res_proj)
+    cpf(res_samp_path, temp_path_res_samp)
     return temp_path_cfg

tests/smoketests/test_run.py

@@ -5,11 +5,12 @@
 import subprocess
 from yaml import safe_load, dump
 
-def subp_run(pth):
+
+def subp_exec(pth, cmd):
     """
     looper run in a subprocess, example cfg
     """
-    proc = subprocess.Popen(["looper", "run", "-d", pth],
+    proc = subprocess.Popen(["looper", cmd, "-d", pth],
                             stderr=subprocess.PIPE, stdout=subprocess.PIPE)
     stdout, stderr = proc.communicate()
     return str(stdout), str(stderr), proc.returncode
@@ -18,11 +19,11 @@ def subp_run(pth):
 class LooperRunTests:
     def test_looper_run_basic(self, example_pep_piface_path_cfg):
         """ Verify looper runs in a basic case and return code is 0 """
-        stdout, stderr, rc = subp_run(example_pep_piface_path_cfg)
+        stdout, stderr, rc = subp_exec(example_pep_piface_path_cfg, "run")
         assert rc == 0
 
     def test_looper_multi_pipeline(self, example_pep_piface_path_cfg):
-        stdout, stderr, rc = subp_run(example_pep_piface_path_cfg)
+        stdout, stderr, rc = subp_exec(example_pep_piface_path_cfg, "run")
         assert "Commands submitted: 6 of 6" in stderr
 
     def test_looper_single_pipeline(self, prep_temp_pep):
@@ -35,6 +36,31 @@ def test_looper_single_pipeline(self, prep_temp_pep):
         # print("\nconfig_data: \n{}\n".format(config_data))
         with open(tp, 'w') as conf_file:
             dump(config_data, conf_file)
-        stdout, stderr, rc = subp_run(tp)
+        stdout, stderr, rc = subp_exec(tp, "run")
+        assert rc == 0
+        assert "Commands submitted: 6 of 6" not in stderr
+
+
+class LooperRunpTests:
+    def test_looper_runp_basic(self, example_pep_piface_path_cfg):
+        """ Verify looper runps in a basic case and return code is 0 """
+        stdout, stderr, rc = subp_exec(example_pep_piface_path_cfg, "runp")
+        assert rc == 0
+
+    def test_looper_multi_pipeline(self, example_pep_piface_path_cfg):
+        stdout, stderr, rc = subp_exec(example_pep_piface_path_cfg, "runp")
+        assert "Jobs submitted: 2" in stderr
+
+    def test_looper_single_pipeline(self, prep_temp_pep):
+        tp = prep_temp_pep
+        with open(tp, 'r') as conf_file:
+            config_data = safe_load(conf_file)
+        # print("\nconfig_data: \n{}\n".format(config_data))
+        pifaces = config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY]
+        config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][PIPELINE_INTERFACES_KEY] = pifaces[1]
+        # print("\nconfig_data: \n{}\n".format(config_data))
+        with open(tp, 'w') as conf_file:
+            dump(config_data, conf_file)
+        stdout, stderr, rc = subp_exec(tp, "runp")
         assert rc == 0
-        assert "Commands submitted: 6 of 6" not in stderr
+        assert "Jobs submitted: 2" not in stderr