ENH: add singularity builds to CI tests (#154)

* add singularity client * add singularity build tests * minor fixes * add singularity cache * minor change * prepend sudo * use correct sudo invocation * add bash_test_file arg * mkdir for sing files * do not save in new dir * bind path to singularity exec test * fix tests for singularity * pip install quietly + always save cache * error on unset variables * use singularity run instead of exec * remove execute privileges * use a cache dir in tmp * fix runscript syntax * tmp only test miniconda * prepend miniconda to path * use local cache directory * use local cache * enh cache path * run all tests * suppress find warnings in singularity * only test miniconda * tmp minor change * tmp: fail miniconda test * fix path * revert changes * update new args * add todos * fix templates
ReproNim · Apr 20, 2018 · ea5425b · ea5425b
1 parent 8c74e09
commit ea5425b
Show file tree

Hide file tree

Showing 28 changed files with 195 additions and 88 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -16,9 +16,9 @@ _setup_env: &setup_kwds
     conda update -n base -yq conda
     conda update -yq --all
 
-    pip install -U --no-cache-dir pip
-    pip install --no-cache-dir dropbox pytest-cov reprozip
-    pip install -e ~/neurodocker
+    pip install -q --no-cache-dir -U pip
+    pip install -q --no-cache-dir dropbox pytest-cov reprozip
+    pip install -q --no-cache-dir -e ~/neurodocker
 
 
 version: 2
@@ -44,6 +44,7 @@ jobs:
             pytest -k 'test_docker' ~/neurodocker/neurodocker
       - save_cache:
           key: dfs-v0-{{ .Branch }}-{{ .Revision }}
+          when: always
           paths:
             - /tmp/cache
 
@@ -55,12 +56,32 @@ jobs:
           *checkout_kwds
       - run:
           *setup_kwds
+      - restore_cache:
+          keys:
+            - srs-v0-{{ .Branch }}-{{ .Revision }}
+            - srs-v0-{{ .Branch }}
+            - srs-v0
+      - run:
+          name: Install singularity
+          command: |
+            source ~/.bashrc
+            curl -fsSL http://neuro.debian.net/lists/trusty.us-nh.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list
+            curl -fsSL https://dl.dropbox.com/s/zxs209o955q6vkg/neurodebian.gpg | sudo apt-key add -
+            (sudo apt-key adv --refresh-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 || true)
+            sudo apt-get -qq update
+            sudo apt-get install -yq singularity-container
+            pip install -q --no-cache-dir singularity
       - run:
           name: Test singularity image builds
           no_output_timeout: 360m
           command: |
             source ~/.bashrc
             pytest -k 'test_singularity' ~/neurodocker/neurodocker
+      - save_cache:
+          key: srs-v0-{{ .Branch }}-{{ .Revision }}
+          when: always
+          paths:
+            - ~/tmp/cache
 
   test_others:
     machine:

diff --git a/README.md b/README.md
@@ -31,50 +31,60 @@ Note: it is not yet possible to minimize Docker containers using the _Neurodocke
 
 # Supported Software
 
-Valid options for each software package are the keyword arguments for the class that installs that package. These classes live in [`neurodocker.interfaces`](neurodocker/interfaces/). The default installation behavior for every software package (except Miniconda) is to install by downloading and un-compressing the binaries.
-
-
 | software | argument | description |
 | -------- | -------- | ----------- |
-| **AFNI** | version* | Either 17.2.02 or latest. |
-|          | install_r | If true, install R and AFNI R packages. False by default. |
+| **AFNI** | version* | latest |
+|          | method   | binaries (default), source. Install pre-compiled binaries or build form source. |
+|          | install_path | Installation path. Default `/opt/afni-{version}`. |
+|          | install_r | If true, install R. |
+|          | install_r_pkgs | If true, install R and AFNI's R packages. |
 |          | install_python2 | If true, install Python 2. |
 |          | install_python3 | If true, install Python 3. |
-| **ANTs** | version* | 2.2.0, 2.1.0, 2.0.3, or 2.0.0 |
-|          | use_binaries | If true (default), use pre-compiled binaries. If false, build from source. |
-|          | git_hash  | Git hash to checkout to before building from source (only used if use_binaries is false). |
-| **Convert3D** | version* | "1.0.0" or "nightly". |
-| **dcm2niix** | version* | "latest", "master", git commit hash, or git tag. |
-| **FreeSurfer** | version* | Any version for which binaries are provided. |
+| **ANTs** | version* | 2.2.0, 2.1.0, 2.0.3, or 2.0.0. If `method=source`, version can be a git commit hash or branch. |
+|          | method   | binaries (default), source. |
+|          | install_path | Installation path. Default `/opt/ants-{version}`. |
+|          | cmake_opts | If `method=source`, options for `cmake`. |
+|          | make_opts | If `method=source`, options for `make`. |
+| **Convert3D** | version* | 1.0.0 or nightly. |
+|               | method | binaries (default) |
+|               | install_path | Installation path. Default `/opt/convert3d-{version}`. |
+| **dcm2niix** | version* | latest, git commit hash or branch. |
+|              | method | source (default) |
+|              | install_path | Installation path. Default `/opt/dcm2niix-{version}`. |
+|              | cmake_opts | If `method=source`, options for `cmake`. |
+|              | make_opts | If `method=source`, options for `make`. |
+| **FreeSurfer** | version* | 6.0.0-min |
+|                | method | binaries (default) |
+|                | install_path | Installation path. Default `/opt/freesurfer-{version}`. |
+|                | exclude_paths | Sequence of path(s) to exclude when inflating the tarball. |
 |                | license_path | Relative path to license file. If provided, this file will be copied into the Docker image. Must be within the build context. |
-|                | min | If true, install a version of FreeSurfer minimized for recon-all. See [freesurfer/freesurfer#70](https://github.com/freesurfer/freesurfer/issues/70). False by default. |
-| **FSL**** | version* | Any version for which binaries are provided. |
-|           | eddy_5011 | If true, use pre-release version of FSL eddy v5.0.11 |
-|           | eddy_5011_cuda | 6.5, 7.0, 7.5, 8.0; only valid if using eddy pre-release |
-|           | use_binaries | If true (default), use pre-compiled binaries. Building from source is not available now but might be added in the future. |
-|           | use_installer | If true, use FSL's Python installer. Only valid on CentOS images. |
+| **FSL**** | version* | 5.0.11, 5.0.10, 5.0.9, 5.0.8 |
+|           | method | binaries (default) |
+|           | install_path | Installation path. Default `/opt/fsl-{version}`. |
+| **Matlab Compiler Runtime** | version* | 2018a, 2012-17[a-b], 2010a |
+|                             | method | binaries (default) |
+|                             | install_path | Installation path. Default `/opt/matlabmcr-{version}`. |
 | **MINC** | version* | 1.9.15 |
-| **Miniconda** | env_name* | Name of this conda environment. |
-|               | yaml_file | Environment specification file. Can be path on host or URL. |
-|               | conda_install | Packages to install with conda. e.g., `conda_install="python=3.6 numpy traits"` |
-|               | pip_install | Packages to install with pip. |
-|               | conda_opts  | Command-line options to pass to [`conda create`](https://conda.io/docs/commands/conda-create.html). e.g., `conda_opts="-c vida-nyu"` |
-|               | pip_opts    | Command-line options to pass to [`pip install`](https://pip.pypa.io/en/stable/reference/pip_install/#options). |
+|          | method | binaries (default) |
+|          | install_path | Installation path. Default `/opt/minc-{version}`. |
+| **Miniconda** | version | latest (default), all other hosted versions. |
+|               | install_path | Installation path. Default `/opt/miniconda-{version}`. |
+|               | env_name* | Name of this conda environment. |
+|               | conda_install | Packages to install with `conda`. E.g., `conda_install="python=3.6 numpy traits"` |
+|               | pip_install | Packages to install with `pip`. |
 |               | activate | If true (default), activate this environment in container entrypoint. |
-|               | miniconda_version | Version of Miniconda. Latest by default. |
-| **MRtrix3** | use_binaries | If true (default), use pre-compiled binaries. If false, build from source. |
-|             | git_hash | Git hash to checkout to before building from source (only used if use_binaries is false). |
+| **MRtrix3** | version* | 3.0 |
+|             | method | binaries (default) |
+|             | install_path | Installation path. Default `/opt/mrtrix3-{version}`. |
 | **NeuroDebian** | os_codename* | Codename of the operating system (e.g., stretch, zesty). |
-|                 | download_server* | Server to download NeuroDebian packages from. Choose the one closest to you. See `neurodocker generate --help` for the full list of servers. |
-|                 | pkgs | Packages to download from NeuroDebian. |
+|                 | server* | Server to download NeuroDebian packages from. Choose the one closest to you. See `neurodocker generate docker --help` for the full list of servers. |
 |                 | full | If true (default), use non-free sources. If false, use libre sources. |
-| **PETPVC** | version* | 1.2.0-b, 1.2.0-a, 1.1.0, 1.0.0 |
-| **SPM** | version*        | 12 (earlier versions will be supported in the future). |
-|         | matlab_version* | R2017a (other MCR versions will be supported once earlier SPM versions are supported). |
-
+| **SPM12** | version* | r7219, r6914, r6685, r6472, r6225 |
+|           | install_path | Installation path. Default `/opt/spm12-{version}`. |
+|           |              | _Note: Matlab Compiler Runtime is installed when SPM12 is installed._ |
 
-\* required argument.
 
+\* required argument.  
 ** FSL is non-free. If you are considering commercial use of FSL, please consult the [relevant license](https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/Licence).
 
 

diff --git a/neurodocker/generators/singularity.py b/neurodocker/generators/singularity.py
@@ -65,7 +65,7 @@ def __init__(self, specs):
         self._post = []
         self._environment = OrderedDict()
         self._files = []
-        self._runscript = ["/neurodocker/startup.sh"]
+        self._runscript = ['/neurodocker/startup.sh "$@"']
         self._test = []
         self._labels = []
 

diff --git a/neurodocker/interfaces/interfaces.py b/neurodocker/interfaces/interfaces.py
@@ -32,23 +32,13 @@ def __init__(self, *args, install_python2=False, install_python3=False,
             self._dependencies.append('python')
         if self.install_python3:
             self._dependencies.append('python3')
-        if self.install_r:
+        if self.install_r or self.install_r_pkgs:
             r = {
                 'apt': ['r-base', 'r-base-dev'],
                 'yum': ['R-devel'],
             }
             self._dependencies.extend(r[self._pkg_manager])
 
-        if self.install_r_pkgs and not self.install_r:
-            raise ValueError(
-                "option `install_r=True` must be specified if"
-                " `install_r_pkgs=True`."
-            )
-
-    def _install_r_pkgs(self):
-        """Return string of instructions to install AFNI's R packages."""
-        return "{{ afni.install_path }}/rPkgsInstall -pkgs ALL"
-
 
 class ANTs(_BaseInterface):
     """Create instance of ANTs object."""
@@ -102,6 +92,8 @@ class FreeSurfer(_BaseInterface):
         'trctrain',
     )
 
+    # TODO(kaczmarj): add option to add license file.
+
     def __init__(self, *args, **kwargs):
         super().__init__(self._name, *args, **kwargs)
 
@@ -183,6 +175,9 @@ class Miniconda(_BaseInterface):
     _installed = False
     _environments = set()
 
+    # TODO(kaczmarj): use create_env and use_env options.
+    # TODO(kaczmarj): add method to create environment from file.
+    # TODO(kaczmarj): add conda_opts and pip_opts
     def __init__(self, *args, env_name, conda_install=None, pip_install=None,
                  preinstalled=False, **kwargs):
         self.env_name = env_name

diff --git a/neurodocker/interfaces/tests/test_afni.py b/neurodocker/interfaces/tests/test_afni.py
@@ -32,7 +32,9 @@ def test_singularity(self):
                 ('user', 'neuro'),
             ],
         }
-        utils.test_singularity_container_from_specs(specs=specs)
+        bash_test_file = "test_afni.sh"
+        utils.test_singularity_container_from_specs(
+            specs=specs, bash_test_file=bash_test_file)
 
     def test_invalid_binaries(self):
         with pytest.raises(ValueError):

diff --git a/neurodocker/interfaces/tests/test_ants.py b/neurodocker/interfaces/tests/test_ants.py
@@ -31,8 +31,9 @@ def test_singularity(self):
                 ('user', 'neuro'),
             ]
         }
-
-        utils.test_singularity_container_from_specs(specs=specs)
+        bash_test_file = "test_ants.sh"
+        utils.test_singularity_container_from_specs(
+            specs=specs, bash_test_file=bash_test_file)
 
     def test_invalid_binaries(self):
         with pytest.raises(ValueError):

diff --git a/neurodocker/interfaces/tests/test_convert3d.py b/neurodocker/interfaces/tests/test_convert3d.py
@@ -29,4 +29,6 @@ def test_singularity(self):
             ]
         }
 
-        utils.test_singularity_container_from_specs(specs)
+        bash_test_file = "test_convert3d.sh"
+        utils.test_singularity_container_from_specs(
+            specs=specs, bash_test_file=bash_test_file)
diff --git a/neurodocker/interfaces/tests/test_dcm2niix.py b/neurodocker/interfaces/tests/test_dcm2niix.py
@@ -29,4 +29,6 @@ def test_singularity(self):
             ],
         }
 
-        utils.test_singularity_container_from_specs(specs)
+        bash_test_file = "test_dcm2niix.sh"
+        utils.test_singularity_container_from_specs(
+            specs=specs, bash_test_file=bash_test_file)
diff --git a/neurodocker/interfaces/tests/test_freesurfer.py b/neurodocker/interfaces/tests/test_freesurfer.py
@@ -28,4 +28,6 @@ def test_singularity(self):
                 ('user', 'neuro'),
             ]
         }
-        utils.test_singularity_container_from_specs(specs=specs)
+        bash_test_file = "test_freesurfer.sh"
+        utils.test_singularity_container_from_specs(
+            specs=specs, bash_test_file=bash_test_file)
diff --git a/neurodocker/interfaces/tests/test_fsl.py b/neurodocker/interfaces/tests/test_fsl.py
@@ -28,4 +28,6 @@ def test_singularity(self):
                 ('user', 'neuro'),
             ]
         }
-        utils.test_singularity_container_from_specs(specs=specs)
+        bash_test_file = "test_fsl.sh"
+        utils.test_singularity_container_from_specs(
+            specs=specs, bash_test_file=bash_test_file)
diff --git a/neurodocker/interfaces/tests/test_minc.py b/neurodocker/interfaces/tests/test_minc.py
@@ -22,11 +22,13 @@ def test_docker(self):
 
     def test_singularity(self):
         specs = {
-            'pkg_manager': 'apt',
+            'pkg_manager': 'yum',
             'instructions': [
                 ('base', 'docker://centos:7'),
                 ('minc', {'version': '1.9.15'}),
                 ('user', 'neuro'),
             ]
         }
-        utils.test_singularity_container_from_specs(specs=specs)
+        bash_test_file = "test_minc.sh"
+        utils.test_singularity_container_from_specs(
+            specs=specs, bash_test_file=bash_test_file)
diff --git a/neurodocker/interfaces/tests/test_minc.sh b/neurodocker/interfaces/tests/test_minc.sh
diff --git a/neurodocker/interfaces/tests/test_miniconda.py b/neurodocker/interfaces/tests/test_miniconda.py
@@ -36,16 +36,17 @@ def test_docker(self):
 
     def test_singularity(self):
         specs = {
-            'pkg_manager': 'yum',
+            'pkg_manager': 'apt',
             'instructions': [
-                ('base', 'docker://centos:7'),
+                ('base', 'docker://debian:stretch-slim'),
                 ('user', 'neuro'),
                 (
                     'miniconda',
                     {
                         'env_name': 'default',
-                        'conda_install': ['python=3.5.1', 'traits'],
+                        'conda_install': ['python=3.6.5', 'traits'],
                         'pip_install': ['nipype'],
+                        'activate': True,
                     }
                 ),
                 (
@@ -55,4 +56,6 @@ def test_singularity(self):
             ],
         }
 
-        utils.test_singularity_container_from_specs(specs=specs)
+        bash_test_file = "test_miniconda.sh"
+        utils.test_singularity_container_from_specs(
+            specs=specs, bash_test_file=bash_test_file)
diff --git a/neurodocker/interfaces/tests/test_mrtrix.py b/neurodocker/interfaces/tests/test_mrtrix.py
@@ -29,4 +29,6 @@ def test_singularity(self):
             ],
         }
 
-        utils.test_singularity_container_from_specs(specs=specs)
+        bash_test_file = "test_mrtrix.sh"
+        utils.test_singularity_container_from_specs(
+            specs=specs, bash_test_file=bash_test_file)
diff --git a/neurodocker/interfaces/tests/test_neurodebian.py b/neurodocker/interfaces/tests/test_neurodebian.py
@@ -45,4 +45,6 @@ def test_singularity(self):
             ]
         }
 
-        utils.test_singularity_container_from_specs(specs=specs)
+        bash_test_file = "test_neurodebian.sh"
+        utils.test_singularity_container_from_specs(
+            specs=specs, bash_test_file=bash_test_file)
diff --git a/neurodocker/interfaces/tests/test_petpvc.py b/neurodocker/interfaces/tests/test_petpvc.py
@@ -36,4 +36,6 @@ def test_singularity(self):
             ]
         }
 
-        utils.test_singularity_container_from_specs(specs=specs)
+        bash_test_file = "test_petpvc.sh"
+        utils.test_singularity_container_from_specs(
+            specs=specs, bash_test_file=bash_test_file)
diff --git a/neurodocker/interfaces/tests/test_petpvc.sh b/neurodocker/interfaces/tests/test_petpvc.sh
diff --git a/neurodocker/interfaces/tests/test_spm.py b/neurodocker/interfaces/tests/test_spm.py
@@ -29,4 +29,6 @@ def test_singularity(self):
             ],
         }
 
-        utils.test_singularity_container_from_specs(specs=specs)
+        bash_test_file = "test_spm.sh"
+        utils.test_singularity_container_from_specs(
+            specs=specs, bash_test_file=bash_test_file)
diff --git a/neurodocker/interfaces/tests/test_spm.sh b/neurodocker/interfaces/tests/test_spm.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -ex
+set -exu
 
 echo 'fprintf("testing")' > /tmp/test.m
 $SPMMCRCMD /tmp/test.m