From 4352a1e0bca79652198008e6a66a03c8e176a330 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sun, 14 Oct 2018 15:01:29 -0700 Subject: [PATCH 1/9] TEST: Get ds114 files for tests --- .travis.yml | 7 ++++ pydra/engine/tests/test_newnode.py | 56 +++++++++++++++--------------- tools/get_test_data.sh | 5 +++ 3 files changed, 40 insertions(+), 28 deletions(-) create mode 100644 tools/get_test_data.sh diff --git a/.travis.yml b/.travis.yml index 45e4bddcef..326f3c8bc6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,17 @@ python: - 3.6 - 3.7 +env: + - PYDRA_TEST_DATA=/tmp/pydra_test_data + install: - pip install --upgrade pip setuptools wheel - pip install ".[all]" +before_script: + - pip install datalad + - bash tools/get_test_data.sh + script: - py.test -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra diff --git a/pydra/engine/tests/test_newnode.py b/pydra/engine/tests/test_newnode.py index ca36acdb91..1cb36ba815 100644 --- a/pydra/engine/tests/test_newnode.py +++ b/pydra/engine/tests/test_newnode.py @@ -1,3 +1,9 @@ +import sys +import os +import time +import numpy as np +from pathlib import Path + from nipype.utils.filemanip import save_json, makedirs, to_str from nipype.interfaces import fsl @@ -5,9 +11,11 @@ from ..auxiliary import FunctionInterface, CurrentInterface from ..submitter import Submitter -import sys, time, os -import numpy as np -import pytest, pdb +import pytest +import pdb + +TEST_DATA_DIR = Path(os.getenv('PYDRA_TEST_DATA', '/nonexistent/path')) +DS114_DIR = TEST_DATA_DIR / 'ds000114' python35_only = pytest.mark.skipif(sys.version_info < (3, 5), reason="requires Python>3.4") @@ -1441,8 +1449,7 @@ def test_workflow_16a(plugin, change_dir): # testing CurrentInterface that is a temporary wrapper for current interfaces -@pytest.mark.skipif( - not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_node_1(change_dir, plugin): @@ -1453,7 +1460,7 @@ def test_current_node_1(change_dir, plugin): name="NA", inputs={ "in_file": - "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz" + str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz") }, interface=interf_bet, workingdir="test_cnd1_{}".format(plugin), @@ -1466,8 +1473,7 @@ def test_current_node_1(change_dir, plugin): assert "out_file" in nn.output.keys() -@pytest.mark.skipif( - not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_node_2(change_dir, plugin): @@ -1475,8 +1481,8 @@ def test_current_node_2(change_dir, plugin): interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface") in_file_l = [ - "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz", - "/Users/dorota/nipype_workshop/data/ds000114/sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz" + str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"), + str(DS114_DIR / "sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz") ] nn = Node( name="NA", @@ -1496,8 +1502,7 @@ def test_current_node_2(change_dir, plugin): assert "NA.in_file:1" in nn.output["out_file"].keys() -@pytest.mark.skipif( - not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_wf_1(change_dir, plugin): @@ -1508,7 +1513,7 @@ def test_current_wf_1(change_dir, plugin): name="fsl", inputs={ "in_file": - "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz" + str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz") }, interface=interf_bet, workingdir="nn", @@ -1529,8 +1534,7 @@ def test_current_wf_1(change_dir, plugin): assert "fsl_out" in wf.output.keys() -@pytest.mark.skipif( - not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_wf_1a(change_dir, plugin): @@ -1541,7 +1545,7 @@ def test_current_wf_1a(change_dir, plugin): name="fsl", inputs={ "in_file": - "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz" + str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz") }, interface=interf_bet, workingdir="nn", @@ -1562,8 +1566,7 @@ def test_current_wf_1a(change_dir, plugin): assert "fsl_out" in wf.output.keys() -@pytest.mark.skipif( - not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_wf_1b(change_dir, plugin): @@ -1583,7 +1586,7 @@ def test_current_wf_1b(change_dir, plugin): write_state=False, inputs={ "in_file": - "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz" + str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz") }) sub = Submitter(plugin=plugin, runnable=wf) @@ -1593,8 +1596,7 @@ def test_current_wf_1b(change_dir, plugin): assert "fsl_out" in wf.output.keys() -@pytest.mark.skipif( - not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_wf_1c(change_dir, plugin): @@ -1613,7 +1615,7 @@ def test_current_wf_1c(change_dir, plugin): write_state=False, inputs={ "in_file": - "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz" + str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz") }) sub = Submitter(plugin=plugin, runnable=wf) @@ -1623,8 +1625,7 @@ def test_current_wf_1c(change_dir, plugin): assert "fsl_out" in wf.output.keys() -@pytest.mark.skipif( - not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_wf_2(change_dir, plugin): @@ -1632,8 +1633,8 @@ def test_current_wf_2(change_dir, plugin): interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface") in_file_l = [ - "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz", - "/Users/dorota/nipype_workshop/data/ds000114/sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz" + str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"), + str(DS114_DIR / "sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz") ] nn = Node( @@ -1662,8 +1663,7 @@ def test_current_wf_2(change_dir, plugin): assert 'cw2.in_file:1' in wf.output["fsl_out"].keys() -@pytest.mark.skipif( - not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_wf_2a(change_dir, plugin): diff --git a/tools/get_test_data.sh b/tools/get_test_data.sh new file mode 100644 index 0000000000..4cf2d25c29 --- /dev/null +++ b/tools/get_test_data.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +mkdir -p $PYDRA_TEST_DATA +datalad install -r -s ///openfmri/ds000114 $PYDRA_TEST_DATA/ds000114 +datalad get $PYDRA_TEST_DATA/ds000114/sub-0{1,2}/ses-test/anat/sub-0?_ses-test_T1w.nii.gz From f3307d7c7f30a131c0135c1a55f6990956aff481 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sun, 14 Oct 2018 15:19:10 -0700 Subject: [PATCH 2/9] CI: Update Travis install --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 326f3c8bc6..d4f1360a26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,12 +9,16 @@ python: env: - PYDRA_TEST_DATA=/tmp/pydra_test_data +before_install: + - travis_retry bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh); + - travis_retry sudo apt-get update + - travis_retry sudo apt-get install fsl-core git-annex-standalone datalad + - travis_retry pip install --upgrade pip setuptools wheel + install: - - pip install --upgrade pip setuptools wheel - pip install ".[all]" before_script: - - pip install datalad - bash tools/get_test_data.sh script: From b705713129efa33ef166cb7d016ccd4bf2e84d08 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sun, 14 Oct 2018 15:35:26 -0700 Subject: [PATCH 3/9] STY: make format --- pydra/engine/tests/test_newnode.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/pydra/engine/tests/test_newnode.py b/pydra/engine/tests/test_newnode.py index 1cb36ba815..d669d4077a 100644 --- a/pydra/engine/tests/test_newnode.py +++ b/pydra/engine/tests/test_newnode.py @@ -1458,10 +1458,7 @@ def test_current_node_1(change_dir, plugin): nn = Node( name="NA", - inputs={ - "in_file": - str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz") - }, + inputs={"in_file": str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz")}, interface=interf_bet, workingdir="test_cnd1_{}".format(plugin), output_names=["out_file"]) @@ -1511,10 +1508,7 @@ def test_current_wf_1(change_dir, plugin): nn = Node( name="fsl", - inputs={ - "in_file": - str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz") - }, + inputs={"in_file": str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz")}, interface=interf_bet, workingdir="nn", output_names=["out_file"], @@ -1543,10 +1537,7 @@ def test_current_wf_1a(change_dir, plugin): nn = Node( name="fsl", - inputs={ - "in_file": - str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz") - }, + inputs={"in_file": str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz")}, interface=interf_bet, workingdir="nn", output_names=["out_file"], @@ -1584,10 +1575,7 @@ def test_current_wf_1b(change_dir, plugin): workingdir="nn", output_names=["out_file"], write_state=False, - inputs={ - "in_file": - str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz") - }) + inputs={"in_file": str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz")}) sub = Submitter(plugin=plugin, runnable=wf) sub.run() @@ -1613,10 +1601,7 @@ def test_current_wf_1c(change_dir, plugin): workingdir="nn", output_names=["out_file"], write_state=False, - inputs={ - "in_file": - str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz") - }) + inputs={"in_file": str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz")}) sub = Submitter(plugin=plugin, runnable=wf) sub.run() From 894dd2e7074cd37dce58725f776132252fc23540 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sun, 14 Oct 2018 15:37:05 -0700 Subject: [PATCH 4/9] MNT: Rename test files --- pydra/engine/tests/{test_newnode.py => test_node.py} | 0 pydra/engine/tests/{test_newnode_neuro.py => test_node_neuro.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename pydra/engine/tests/{test_newnode.py => test_node.py} (100%) rename pydra/engine/tests/{test_newnode_neuro.py => test_node_neuro.py} (100%) diff --git a/pydra/engine/tests/test_newnode.py b/pydra/engine/tests/test_node.py similarity index 100% rename from pydra/engine/tests/test_newnode.py rename to pydra/engine/tests/test_node.py diff --git a/pydra/engine/tests/test_newnode_neuro.py b/pydra/engine/tests/test_node_neuro.py similarity index 100% rename from pydra/engine/tests/test_newnode_neuro.py rename to pydra/engine/tests/test_node_neuro.py From 6422115675e298bacd4fe2b308a352d220304bae Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sun, 14 Oct 2018 15:43:58 -0700 Subject: [PATCH 5/9] CI: Install fsl, not fsl-core --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d4f1360a26..20b42ed044 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ env: before_install: - travis_retry bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh); - travis_retry sudo apt-get update - - travis_retry sudo apt-get install fsl-core git-annex-standalone datalad + - travis_retry sudo apt-get install fsl git-annex-standalone datalad - travis_retry pip install --upgrade pip setuptools wheel install: From 156889d0e2154606221cb342b3439e88ee8cb91b Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sun, 14 Oct 2018 15:51:20 -0700 Subject: [PATCH 6/9] MNT: Add dev install extra, currently tests + yapf --- pydra/__about__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pydra/__about__.py b/pydra/__about__.py index e8a8230245..7e63d0bb77 100644 --- a/pydra/__about__.py +++ b/pydra/__about__.py @@ -63,6 +63,7 @@ EXTRA_REQUIRES = { 'tests': TESTS_REQUIRES, + 'dev': TESTS_REQUIRES + ['yapf>=0.22'], 'plugins': ['dask', 'distributed'], } From b1e31a27f70309cbbea4ba48c8f7aebeb8c7ef62 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sun, 14 Oct 2018 15:56:31 -0700 Subject: [PATCH 7/9] CI: Source FSL config --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 20b42ed044..8cc5a86e63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ before_install: - travis_retry bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh); - travis_retry sudo apt-get update - travis_retry sudo apt-get install fsl git-annex-standalone datalad + - source /etc/fsl/fsl.sh - travis_retry pip install --upgrade pip setuptools wheel install: From c18bb91dcbe63c63ce6bc1c94973626dffce8cbb Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 15 Oct 2018 14:42:02 -0700 Subject: [PATCH 8/9] FIX: Correct environment variable name --- pydra/engine/tests/test_node.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pydra/engine/tests/test_node.py b/pydra/engine/tests/test_node.py index d669d4077a..402290b26e 100644 --- a/pydra/engine/tests/test_node.py +++ b/pydra/engine/tests/test_node.py @@ -1449,7 +1449,7 @@ def test_workflow_16a(plugin, change_dir): # testing CurrentInterface that is a temporary wrapper for current interfaces -@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_node_1(change_dir, plugin): @@ -1470,7 +1470,7 @@ def test_current_node_1(change_dir, plugin): assert "out_file" in nn.output.keys() -@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_node_2(change_dir, plugin): @@ -1499,7 +1499,7 @@ def test_current_node_2(change_dir, plugin): assert "NA.in_file:1" in nn.output["out_file"].keys() -@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_wf_1(change_dir, plugin): @@ -1528,7 +1528,7 @@ def test_current_wf_1(change_dir, plugin): assert "fsl_out" in wf.output.keys() -@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_wf_1a(change_dir, plugin): @@ -1557,7 +1557,7 @@ def test_current_wf_1a(change_dir, plugin): assert "fsl_out" in wf.output.keys() -@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_wf_1b(change_dir, plugin): @@ -1584,7 +1584,7 @@ def test_current_wf_1b(change_dir, plugin): assert "fsl_out" in wf.output.keys() -@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_wf_1c(change_dir, plugin): @@ -1610,7 +1610,7 @@ def test_current_wf_1c(change_dir, plugin): assert "fsl_out" in wf.output.keys() -@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_wf_2(change_dir, plugin): @@ -1648,7 +1648,7 @@ def test_current_wf_2(change_dir, plugin): assert 'cw2.in_file:1' in wf.output["fsl_out"].keys() -@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_DATA_DIR/ds000114") +@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114") @pytest.mark.parametrize("plugin", Plugins) @python35_only def test_current_wf_2a(change_dir, plugin): From 39b95631298a0d9e2cd8c68f009f1bd2af21d032 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 15 Oct 2018 14:42:24 -0700 Subject: [PATCH 9/9] FIX: Correct data path --- pydra/engine/tests/test_node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_node.py b/pydra/engine/tests/test_node.py index 402290b26e..17c2e6d90e 100644 --- a/pydra/engine/tests/test_node.py +++ b/pydra/engine/tests/test_node.py @@ -1656,8 +1656,8 @@ def test_current_wf_2a(change_dir, plugin): interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface") in_file_l = [ - "/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz", - "/data/ds000114/sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz" + str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"), + str(DS114_DIR / "sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz") ] nn = Node(