diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000..5008ddfcf5 Binary files /dev/null and b/.DS_Store differ diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml new file mode 100644 index 0000000000..f2907a5db1 --- /dev/null +++ b/.azure-pipelines/windows.yml @@ -0,0 +1,38 @@ +parameters: + name: '' + vmImage: '' + matrix: [] + +jobs: +- job: ${{ parameters.name }} + pool: + vmImage: ${{ parameters.vmImage }} + variables: + DEPENDS: "-r min-requirements.txt" + CHECK_TYPE: test + strategy: + matrix: + ${{ insert }}: ${{ parameters.matrix }} + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(PYTHON_VERSION)' + addToPath: true + architecture: '$(PYTHON_ARCH)' + - script: | + echo %PYTHONHASHSEED% + displayName: 'Display hash seed' + - script: | + python -m pip install --upgrade pip==18.1 setuptools==30.2.1 wheel + displayName: 'Update build tools' + - script: | + python -m pip install %DEPENDS% + displayName: 'Install dependencies' + - script: | + python -m pip install .[$(CHECK_TYPE)] + displayName: 'Install pydra' + - script: | + pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra + displayName: 'Pytest tests' + diff --git a/.gitignore b/.gitignore index 40e1319c82..bced49b1bb 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ cov.xml .*.swp *~ .idea + +.DS_Store diff --git a/.travis.yml b/.travis.yml index 911bdac53a..0466ad546a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,7 +57,6 @@ matrix: allow_failures: - python: 3.7 env: INSTALL_DEPENDS="pip==10.0.1 setuptools==30.3.0" - - os: windows before_install: diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000000..fc32bebd67 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,12 @@ +jobs: +- template: .azure-pipelines/windows.yml + parameters: + name: Windows + vmImage: windows-2019 + matrix: + py37-x86: + PYTHON_VERSION: '3.7' + PYTHON_ARCH: 'x86' + py37-x64: + PYTHON_VERSION: '3.7' + PYTHON_ARCH: 'x64' diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index 8014d43cce..6b114c2d1d 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -66,47 +66,6 @@ def split_filename(fname): return pth, fname, ext -def fname_presuffix(fname, prefix="", suffix="", newpath=None, use_ext=True): - """ - Manipulate path and name of input filename. - - Parameters - ---------- - fname : :obj:`str` - A filename (may or may not include path) - prefix : :obj:`str` - Characters to prepend to the filename - suffix : :obj:`str` - Characters to append to the filename - newpath : :obj:`str` - Path to replace the path of the input fname - use_ext : :obj:`bool` - If True (default), appends the extension of the original file - to the output name. - - Return - ------ - path : :obj:`str` - Absolute path of the modified filename - - Examples - -------- - >>> from pydra.engine.helpers_file import fname_presuffix - >>> fname = 'foo.nii.gz' - >>> fname_presuffix(fname,'pre','post','/tmp') - '/tmp/prefoopost.nii.gz' - - """ - pth, fname, ext = split_filename(fname) - if not use_ext: - ext = "" - - # No need for isdefined: bool(Undefined) evaluates to False - if newpath: - pth = op.abspath(newpath) - return op.join(pth, prefix + fname + suffix + ext) - - def hash_file(afile, chunk_len=8192, crypto=sha256, raise_notfound=True): """Compute hash of a file using 'crypto' module.""" from .specs import LazyField @@ -402,7 +361,7 @@ def get_related_files(filename, include_this_file=True): if this_type in type_set: for related_type in type_set: if include_this_file or related_type != this_type: - related_files.append(op.join(path, name + related_type)) + related_files.append(Path(path) / (name + related_type)) if not len(related_files): related_files = [filename] return related_files diff --git a/pydra/engine/tests/test_dockertask.py b/pydra/engine/tests/test_dockertask.py index 97abc56e5f..73c7f778a0 100644 --- a/pydra/engine/tests/test_dockertask.py +++ b/pydra/engine/tests/test_dockertask.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -import os, shutil +import os, sys, shutil import subprocess as sp import pytest import attr @@ -19,8 +19,13 @@ shutil.which("docker") is None or sp.call(["docker", "info"]), reason="no docker within the container", ) +no_win = pytest.mark.skipif( + sys.platform.startswith("win"), + reason="docker command not adjusted for windows docker", +) +@no_win @need_docker def test_docker_1_nosubm(): """ simple command in a container, a default bindings and working directory is added @@ -42,6 +47,7 @@ def test_docker_1_nosubm(): assert "Unable to find image" in res.output.stderr +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_1(plugin): @@ -61,6 +67,7 @@ def test_docker_1(plugin): assert "Unable to find image" in res.output.stderr +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_1_dockerflag(plugin): @@ -82,6 +89,7 @@ def test_docker_1_dockerflag(plugin): assert "Unable to find image" in res.output.stderr +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_1_dockerflag_exception(plugin): @@ -94,6 +102,7 @@ def test_docker_1_dockerflag_exception(plugin): assert "container_info has to have 2 or 3 elements" in str(excinfo.value) +@no_win @need_docker def test_docker_2_nosubm(): """ a command with arguments, cmd and args given as executable @@ -113,6 +122,7 @@ def test_docker_2_nosubm(): assert "Unable to find image" in res.output.stderr +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_2(plugin): @@ -135,6 +145,7 @@ def test_docker_2(plugin): assert "Unable to find image" in res.output.stderr +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_2_dockerflag(plugin): @@ -159,6 +170,7 @@ def test_docker_2_dockerflag(plugin): assert "Unable to find image" in res.output.stderr +@no_win @need_docker def test_docker_2a_nosubm(): """ a command with arguments, using executable and args @@ -183,6 +195,7 @@ def test_docker_2a_nosubm(): assert "Unable to find image" in res.output.stderr +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_2a(plugin): @@ -210,6 +223,7 @@ def test_docker_2a(plugin): assert "Unable to find image" in res.output.stderr +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_3(plugin, tmpdir): @@ -233,6 +247,7 @@ def test_docker_3(plugin, tmpdir): assert "Unable to find image" in res.output.stderr +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_3_dockerflag(plugin, tmpdir): @@ -259,6 +274,7 @@ def test_docker_3_dockerflag(plugin, tmpdir): assert "Unable to find image" in res.output.stderr +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_3_dockerflagbind(plugin, tmpdir): @@ -285,6 +301,7 @@ def test_docker_3_dockerflagbind(plugin, tmpdir): assert "Unable to find image" in res.output.stderr +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_4(plugin, tmpdir): @@ -311,6 +328,7 @@ def test_docker_4(plugin, tmpdir): assert res.output.return_code == 0 +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_4_dockerflag(plugin, tmpdir): @@ -340,6 +358,7 @@ def test_docker_4_dockerflag(plugin, tmpdir): # tests with State +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_st_1(plugin): @@ -364,6 +383,7 @@ def test_docker_st_1(plugin): assert res[0].output.return_code == res[1].output.return_code == 0 +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_st_2(plugin): @@ -388,6 +408,7 @@ def test_docker_st_2(plugin): assert res[0].output.return_code == res[1].output.return_code == 0 +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_st_3(plugin): @@ -406,6 +427,7 @@ def test_docker_st_3(plugin): assert "Ubuntu" in res[3].output.stdout +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_st_4(plugin): @@ -448,6 +470,7 @@ def test_docker_st_4(plugin): # tests with workflows +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_wf_docker_1(plugin, tmpdir): @@ -492,6 +515,7 @@ def test_wf_docker_1(plugin, tmpdir): assert res.output.out == "message from the previous task: hello from pydra" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_wf_docker_1_dockerflag(plugin, tmpdir): @@ -532,6 +556,7 @@ def test_wf_docker_1_dockerflag(plugin, tmpdir): assert res.output.out == "message from the previous task: hello from pydra" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_wf_docker_2pre(plugin, tmpdir): @@ -553,6 +578,7 @@ def test_wf_docker_2pre(plugin, tmpdir): assert res.output.stdout == "/outputs/tmp.txt" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_wf_docker_2(plugin, tmpdir): @@ -593,6 +619,7 @@ def test_wf_docker_2(plugin, tmpdir): assert res.output.out == "Hello!" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_wf_docker_3(plugin, tmpdir): @@ -636,6 +663,7 @@ def test_wf_docker_3(plugin, tmpdir): # tests with customized output_spec +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_outputspec_1(plugin, tmpdir): @@ -664,6 +692,7 @@ def test_docker_outputspec_1(plugin, tmpdir): # tests with customised input_spec +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_inputspec_1(plugin, tmpdir): @@ -705,6 +734,7 @@ def test_docker_inputspec_1(plugin, tmpdir): assert res.output.stdout == "hello from pydra" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_inputspec_1a(plugin, tmpdir): @@ -744,6 +774,7 @@ def test_docker_inputspec_1a(plugin, tmpdir): assert res.output.stdout == "hello from pydra" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_inputspec_2(plugin, tmpdir): @@ -792,6 +823,7 @@ def test_docker_inputspec_2(plugin, tmpdir): assert res.output.stdout == "hello from pydra\nhave a nice one" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_inputspec_2a_except(plugin, tmpdir): @@ -843,6 +875,7 @@ def test_docker_inputspec_2a_except(plugin, tmpdir): assert res.output.stdout == "hello from pydra\nhave a nice one" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_inputspec_2a(plugin, tmpdir): @@ -894,6 +927,7 @@ def test_docker_inputspec_2a(plugin, tmpdir): assert res.output.stdout == "hello from pydra\nhave a nice one" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_inputspec_3(plugin, tmpdir): @@ -937,6 +971,7 @@ def test_docker_inputspec_3(plugin, tmpdir): assert cmdline == docky.cmdline +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_inputspec_3a(plugin, tmpdir): @@ -980,6 +1015,7 @@ def test_docker_inputspec_3a(plugin, tmpdir): assert "use field.metadata['container_path']=True" in str(excinfo.value) +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_cmd_inputspec_copyfile_1(plugin, tmpdir): @@ -1042,6 +1078,7 @@ def test_docker_cmd_inputspec_copyfile_1(plugin, tmpdir): assert "hello from pydra\n" == f.read() +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_inputspec_state_1(plugin, tmpdir): @@ -1090,6 +1127,7 @@ def test_docker_inputspec_state_1(plugin, tmpdir): assert res[1].output.stdout == "have a nice one" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_inputspec_state_1b(plugin, tmpdir): @@ -1139,6 +1177,7 @@ def test_docker_inputspec_state_1b(plugin, tmpdir): assert res[1].output.stdout == "have a nice one" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_wf_inputspec_1(plugin, tmpdir): @@ -1190,6 +1229,7 @@ def test_docker_wf_inputspec_1(plugin, tmpdir): assert res.output.out == "hello from pydra" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_wf_state_inputspec_1(plugin, tmpdir): @@ -1247,6 +1287,7 @@ def test_docker_wf_state_inputspec_1(plugin, tmpdir): assert res[1].output.out == "have a nice one" +@no_win @need_docker @pytest.mark.parametrize("plugin", Plugins) def test_docker_wf_ndst_inputspec_1(plugin, tmpdir): diff --git a/pydra/engine/tests/test_helpers_file.py b/pydra/engine/tests/test_helpers_file.py index 381b7a54e3..56cc8000c6 100644 --- a/pydra/engine/tests/test_helpers_file.py +++ b/pydra/engine/tests/test_helpers_file.py @@ -6,7 +6,6 @@ from ..helpers_file import ( split_filename, - fname_presuffix, copyfile, copyfiles, on_cifs, @@ -38,17 +37,6 @@ def test_split_filename(filename, split): assert res == split -def test_fname_presuffix(): - fname = "foo.nii" - pth = fname_presuffix(fname, "pre_", "_post", "/tmp") - assert pth == "/tmp/pre_foo_post.nii" - fname += ".gz" - pth = fname_presuffix(fname, "pre_", "_post", "/tmp") - assert pth == "/tmp/pre_foo_post.nii.gz" - pth = fname_presuffix(fname, "pre_", "_post", "/tmp", use_ext=False) - assert pth == "/tmp/pre_foo_post" - - @pytest.fixture() def _temp_analyze_files(tmpdir): """Generate temporary analyze file pair.""" @@ -56,7 +44,7 @@ def _temp_analyze_files(tmpdir): orig_hdr = tmpdir.join("orig.hdr") orig_img.open("w+").close() orig_hdr.open("w+").close() - return str(orig_img), str(orig_hdr) + return Path(orig_img), Path(orig_hdr) @pytest.fixture() @@ -66,7 +54,7 @@ def _temp_analyze_files_prime(tmpdir): orig_hdr = tmpdir.join("orig_prime.hdr") orig_img.open("w+").close() orig_hdr.open("w+").close() - return orig_img.strpath, orig_hdr.strpath + return Path(orig_img.strpath), Path(orig_hdr.strpath) def test_copyfile(_temp_analyze_files): @@ -213,11 +201,19 @@ def test_ensure_list(filename, expected): @pytest.mark.parametrize( "file, length, expected_files", [ - ("/path/test.img", 3, ["/path/test.hdr", "/path/test.img", "/path/test.mat"]), - ("/path/test.hdr", 3, ["/path/test.hdr", "/path/test.img", "/path/test.mat"]), - ("/path/test.BRIK", 2, ["/path/test.BRIK", "/path/test.HEAD"]), - ("/path/test.HEAD", 2, ["/path/test.BRIK", "/path/test.HEAD"]), - ("/path/foo.nii", 2, ["/path/foo.nii", "/path/foo.mat"]), + ( + "/path/test.img", + 3, + [Path("/path/test.hdr"), Path("/path/test.img"), Path("/path/test.mat")], + ), + ( + "/path/test.hdr", + 3, + [Path("/path/test.hdr"), Path("/path/test.img"), Path("/path/test.mat")], + ), + ("/path/test.BRIK", 2, [Path("/path/test.BRIK"), Path("/path/test.HEAD")]), + ("/path/test.HEAD", 2, [Path("/path/test.BRIK"), Path("/path/test.HEAD")]), + ("/path/foo.nii", 2, [Path("/path/foo.nii"), Path("/path/foo.mat")]), ], ) def test_related_files(file, length, expected_files): diff --git a/pydra/engine/tests/test_node_task.py b/pydra/engine/tests/test_node_task.py index ff49ea79a2..9611d467b7 100644 --- a/pydra/engine/tests/test_node_task.py +++ b/pydra/engine/tests/test_node_task.py @@ -543,6 +543,8 @@ def test_task_nostate_cachedir_relativepath(tmpdir, plugin): """ task with provided cache_dir as relative path""" cwd = tmpdir.chdir() cache_dir = "test_task_nostate" + tmpdir.mkdir(cache_dir) + nn = fun_addtwo(name="NA", a=3, cache_dir=cache_dir) assert np.allclose(nn.inputs.a, [3]) assert nn.state is None diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 011e83c706..671c65a31e 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -2,7 +2,7 @@ import attr import typing as ty -import os, shutil +import os, sys, shutil import pytest from pathlib import Path @@ -12,6 +12,10 @@ from ..core import Workflow from ..specs import ShellOutSpec, ShellSpec, SpecInfo, File + +if sys.platform.startswith("win"): + pytest.skip("SLURM not available in windows", allow_module_level=True) + if bool(shutil.which("sbatch")): Plugins = ["cf", "slurm"] else: @@ -41,7 +45,7 @@ def test_shell_cmd_1(plugin, results_function): assert shelly.cmdline == " ".join(cmd) res = results_function(shelly, plugin=plugin) - assert res.output.stdout == str(shelly.output_dir) + "\n" + assert Path(res.output.stdout.rstrip()) == shelly.output_dir assert res.output.return_code == 0 assert res.output.stderr == "" @@ -57,7 +61,7 @@ def test_shell_cmd_1_strip(plugin, results_function): assert shelly.cmdline == " ".join(cmd) res = results_function(shelly, plugin) - assert res.output.stdout == str(shelly.output_dir) + assert Path(res.output.stdout) == Path(shelly.output_dir) assert res.output.return_code == 0 assert res.output.stderr == "" @@ -124,7 +128,8 @@ def test_shell_cmd_3(plugin): shelly = ShellCommandTask(name="shelly", executable=cmd).split("executable") assert shelly.cmdline == ["pwd", "whoami"] res = shelly(plugin=plugin) - assert res[0].output.stdout == f"{str(shelly.output_dir[0])}\n" + assert Path(res[0].output.stdout.rstrip()) == shelly.output_dir[0] + if "USER" in os.environ: assert res[1].output.stdout == f"{os.environ['USER']}\n" else: diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index e624a39ca2..f7f4182ef6 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import typing as ty -import os +import os, sys import pytest from ... import mark @@ -15,6 +15,11 @@ from ...utils.messenger import FileMessenger, PrintMessenger, collect_messages from .utils import gen_basic_wf +no_win = pytest.mark.skipif( + sys.platform.startswith("win"), + reason="docker/singularity command not adjusted for windows", +) + @mark.task def funaddtwo(a): @@ -345,6 +350,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] assert (tmpdir / funky.checksum / "messages.jsonld").exists() +@no_win def test_shell_cmd(tmpdir): cmd = ["echo", "hail", "pydra"] @@ -372,6 +378,7 @@ def test_container_cmds(tmpdir): assert containy.cmdline +@no_win def test_docker_cmd(tmpdir): docky = DockerTask(name="docky", executable="pwd", image="busybox") assert ( @@ -393,6 +400,7 @@ def test_docker_cmd(tmpdir): ) +@no_win def test_singularity_cmd(tmpdir): # todo how this should be done? image = "library://sylabsed/linux/alpine" diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 46cfee660c..091feba54a 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -1986,6 +1986,7 @@ def test_wf_nostate_cachedir_relativepath(tmpdir, plugin): """ wf with provided cache_dir as relative path""" tmpdir.chdir() cache_dir = "test_wf_cache_2" + tmpdir.mkdir(cache_dir) wf = Workflow(name="wf_2", input_spec=["x", "y"], cache_dir=cache_dir) wf.add(multiply(name="mult", x=wf.lzin.x, y=wf.lzin.y))