Merge pull request #159 from oesteban/fix/ShowCircleArtifacts

[ENH] Miscelaneous improvements

build/files/run_fmriprep

@@ -1,5 +1,6 @@
 #!/bin/bash
 source /etc/fsl/fsl.sh
 source /etc/afni/afni.sh
-
 fmriprep $@
+
+chmod 755 -R /out/*

circle.yml

@@ -3,22 +3,25 @@ machine:
     OSF_PROJECT: "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage"
     DS005_URL: "${OSF_PROJECT}/57f32a429ad5a101f977eb75"
     DS054_URL: "${OSF_PROJECT}/57f32c22594d9001ef91bf9e"
-    SCRATCH: $HOME/scratch
   services:
     - docker
 
 dependencies:
   cache_directories:
     - "~/data"
     - "~/docker"
+    - "~/.cache/stanford-crn"
 
   pre:
     - mkdir -p $HOME/data
     - mkdir -p $HOME/docker
-    - mkdir -p $SCRATCH
+    - mkdir -p $HOME/ds005/out $HOME/ds054/out
     # Download test data
     - if [[ ! -d $HOME/data/ds005 ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O ds005_downsampled.tar.gz "${DS005_URL}" && tar xzf ds005_downsampled.tar.gz -C $HOME/data/; fi
     - if [[ ! -d $HOME/data/ds054 ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O ds054_downsampled.tar.gz "${DS054_URL}" && tar xzf ds054_downsampled.tar.gz -C $HOME/data/; fi
+    # Cache templates
+    - pip install "niworkflows>=0.0.3a7"
+    - python -c "from niworkflows.data import get_mni_template; get_mni_template()"
 
   override:
     - if [[ -e $HOME/docker/image.tar ]]; then docker load -i $HOME/docker/image.tar; fi
@@ -27,11 +30,12 @@ dependencies:
 test:
   override:
     - docker run -ti --rm --entrypoint="/usr/bin/run_unittests" poldracklab/fmriprep:latest
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v $HOME/data:/data:ro -v $SCRATCH/ds054:/scratch -w /scratch poldracklab/fmriprep:latest /data/ds054 out/ participant -w work/ -t ds054 --debug :
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v $HOME/data:/data:ro -v ~/.cache/stanford-crn:/root/.cache/stanford-crn -v $HOME/ds054/out:/out -w /scratch poldracklab/fmriprep:latest /data/ds054 /out/ participant -w work/ -t ds054 --debug :
         timeout: 4800
-    - docker run -i -v /etc/localtime:/etc/localtime:ro -v $HOME/data:/data:ro -v $SCRATCH/ds005:/scratch -w /scratch poldracklab/fmriprep:latest /data/ds005 out/ participant -w work/ -t ds005 --debug :
+    - docker run -i -v /etc/localtime:/etc/localtime:ro -v $HOME/data:/data:ro -v ~/.cache/stanford-crn:/root/.cache/stanford-crn -v $HOME/ds005/out:/out -w /scratch poldracklab/fmriprep:latest /data/ds005 /out/ participant -w work/ -t ds005 --debug :
         timeout: 4800
 
 general:
   artifacts:
-    - "~/scratch"
+    - "~/ds054/out"
+    - "~/ds005/out"

fmriprep/interfaces/bids.py

@@ -6,7 +6,7 @@
 # @Author: oesteban
 # @Date:   2016-06-03 09:35:13
 # @Last Modified by:   oesteban
-# @Last Modified time: 2016-08-17 15:32:43
+# @Last Modified time: 2016-10-05 15:03:25
 from lockfile import LockFile
 import os
 import os.path as op
@@ -19,7 +19,7 @@
 from nipype.interfaces.base import (traits, isdefined, TraitedSpec, BaseInterface,
                                     BaseInterfaceInputSpec, File, InputMultiPath, OutputMultiPath)
 
-from fmriprep.utils.misc import collect_bids_data
+from fmriprep.utils.misc import collect_bids_data, make_folder
 
 LOGGER = logging.getLogger('interface')
 
@@ -125,9 +125,7 @@ def _run_interface(self, runtime):
         out_path += '/{}'.format(mod)
 
         out_path = op.join(base_directory, out_path)
-        with LockFile(op.join(base_directory, '.fmriprep-lock')):
-            if not op.exists(out_path):
-                os.makedirs(out_path)
+        make_folder(out_path)
 
         base_fname = op.join(out_path, fname)
 

fmriprep/run_workflow.py

@@ -3,7 +3,7 @@
 # @Author: oesteban
 # @Date:   2015-11-19 16:44:27
 # @Last Modified by:   oesteban
-# @Last Modified time: 2016-10-03 09:06:10
+# @Last Modified time: 2016-10-05 15:03:18
 """
 fMRI preprocessing workflow
 =====
@@ -16,9 +16,6 @@
 from argparse import ArgumentParser
 from argparse import RawTextHelpFormatter
 from multiprocessing import cpu_count
-from lockfile import LockFile
-
-
 
 def main():
     """Entry point"""
@@ -72,6 +69,7 @@ def main():
 def create_workflow(opts):
     import logging
     from nipype import config as ncfg
+    from fmriprep.utils import make_folder
     from fmriprep.workflows import base as fwb
     from fmriprep.workflows.base import base_workflow_enumerator
 
@@ -95,23 +93,15 @@ def create_workflow(opts):
     if opts.ants_nthreads is not None:
         settings['ants_threads'] = opts.ants_nthreads
 
-    log_dir = op.join(settings['work_dir'], 'log')
+    log_dir = op.join(settings['output_dir'], 'log')
+    derivatives = op.join(settings['output_dir'], 'derivatives')
 
     # Check and create output and working directories
-    # Using locks to prevent https://github.com/poldracklab/mriqc/issues/111
-    with LockFile('.fmriprep-folders-lock'):
-        if not op.exists(settings['output_dir']):
-            os.makedirs(settings['output_dir'])
-
-        derivatives = op.join(settings['output_dir'], 'derivatives')
-        if not op.exists(derivatives):
-            os.makedirs(derivatives)
-
-        if not op.exists(settings['work_dir']):
-            os.makedirs(settings['work_dir'])
-
-        if not op.exists(log_dir):
-            os.makedirs(log_dir)
+    # Using make_folder to prevent https://github.com/poldracklab/mriqc/issues/111
+    make_folder(settings['output_dir'])
+    make_folder(settings['work_dir'])
+    make_folder(derivatives)
+    make_folder(log_dir)
 
     logger.addHandler(logging.FileHandler(op.join(log_dir, 'run_workflow')))
 

fmriprep/utils/__init__.py

@@ -2,4 +2,4 @@
 # -*- coding: utf-8 -*-
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
 # vi: set ft=python sts=4 ts=4 sw=4 et:
-from fmriprep.utils.misc import collect_bids_data
+from fmriprep.utils.misc import collect_bids_data, make_folder

fmriprep/utils/misc.py

@@ -3,12 +3,22 @@
 import itertools
 import json
 import os
+from os import path as op
+from errno import EEXIST
 import re
 
 from bids.grabbids import BIDSLayout
 
 INPUTS_SPEC = {'fieldmaps': [], 'func': [], 't1': [], 'sbref': []}
 
+def make_folder(folder):
+    try:
+        os.makedirs(folder)
+    except OSError as exc:
+        if exc.errno == EEXIST:
+            pass
+    return folder
+
 def gen_list(inlist, base=1):
     return range(base, len(inlist) + base)