ENH: Use TemplateFlow and niworkflows-0.8.x

.circleci/config.yml

@@ -4,7 +4,7 @@ jobs:
   build:
     environment:
       - TZ: "/usr/share/zoneinfo/America/Los_Angeles"
-      - MRIQC_API_TAG: 1.0.5
+      - MRIQC_API_TAG: 1.0.6
 
     docker:
       - image: docker:18.01.0-ce-git
@@ -133,8 +133,8 @@ jobs:
           name: Create Nipype config files
           command: |
             mkdir -p /tmp/t1w /tmp/bold
-            printf "[execution]\nstop_on_first_crash = true\nremove_unnecessary_outputs = false\n" > /tmp/t1w/nipype.cfg
-            echo "poll_sleep_duration = 0.2" >> /tmp/t1w/nipype.cfg
+            printf "[execution]\nstop_on_first_crash = true\n" > /tmp/t1w/nipype.cfg
+            echo "poll_sleep_duration = 0.01" >> /tmp/t1w/nipype.cfg
             echo "hash_method = content" >> /tmp/t1w/nipype.cfg
             cp /tmp/t1w/nipype.cfg /tmp/bold/nipype.cfg
       - persist_to_workspace:
@@ -187,9 +187,8 @@ jobs:
           no_output_timeout: 2h
           command: |
             docker run -ti --rm=false \
-              -v $PWD:/scratch -w /usr/local/src/mriqc \
-              --entrypoint="py.test" \
-              poldracklab/mriqc:latest mriqc/ \
+              -v $PWD:/scratch --entrypoint="py.test" -w /src/mriqc \
+              poldracklab/mriqc:latest mriqc \
               --junitxml=/scratch/tests.xml \
               --doctest-modules --ignore=docs --ignore=src --ignore=mriqc/bin \
               --ignore=mriqc/classifier/sklearn --ignore=mriqc/interfaces/transitional.py
@@ -219,10 +218,10 @@ jobs:
           name: Build MRIQC documentation
           no_output_timeout: 2h
           command: |
-            docker run -ti --rm=false -v /home/circleci/out/docs:/scratch/docs \
-              -w /usr/local/src/mriqc/docs \
+            docker run -u $( id -u ) -e PYTHONPATH=/src/mriqc -ti --rm=false \
+              -v /home/circleci/out/docs:/scratch/docs \
               --entrypoint=sphinx-build poldracklab/mriqc:latest \
-              -T -E -W -D language=en -b html source/ /scratch/docs 2>&1 \
+              -T -D language=en -b html /src/mriqc/docs/source/ /scratch/docs 2>&1 \
               | tee $PWD/builddocs.log
             cat $PWD/builddocs.log
             grep -qv "ERROR" $PWD/builddocs.log
@@ -261,29 +260,36 @@ jobs:
             docker-compose -f /tmp/src/mriqcwebapi/dockereve-master/docker-compose.yml --verbose up -d
           background: true
 
+      - restore_cache:
+          keys:
+            - t1w-v2-{{ epoch }}
+            - t1w-v2-
       - run:
           name: Run participant-level on T1w images
           no_output_timeout: 2h
           command: |
             mkdir -p /tmp/t1w/work /tmp/t1w/derivatives
-            sudo setfacl -d -m group:$(id -gn):rwx /tmp/t1w/derivatives
-            sudo setfacl -m group:$(id -gn):rwx /tmp/t1w/derivatives
-            sudo setfacl -d -m group:$(id -gn):rwx /tmp/t1w/work
-            sudo setfacl -m group:$(id -gn):rwx /tmp/t1w/work
             # Run MRIQC
-            docker run --rm=false -ti -v /tmp/data/${TEST_DATA_NAME}:/data:ro \
-                       -v $PWD:/scratch -w /scratch \
+            docker run -u $( id -u ) --rm=false -ti \
+                       -v /tmp/data/${TEST_DATA_NAME}:/data:ro \
+                       -v /tmp/t1w:/scratch -w /scratch \
                        poldracklab/mriqc:latest \
                        /data derivatives/ participant \
                        -vv --verbose-reports --profile -m T1w --dsname circletests \
                        --n_procs 2 --ants-nthreads 1 --ants-float \
                        --webapi-url http://$( hostname -I | awk '{print $1}' )/api/v1 --upload-strict
+      - save_cache:
+         key: t1w-v2-{{ epoch }}
+         paths:
+            - /tmp/t1w/work
+
       - run:
           name: Run group-level on T1w images
           no_output_timeout: 2h
           command: |
-            docker run --rm=false -ti -v /tmp/data/${TEST_DATA_NAME}:/data:ro \
-                       -v $PWD:/scratch -w /scratch \
+            docker run -u $( id -u ) --rm=false -ti \
+                       -v /tmp/data/${TEST_DATA_NAME}:/data:ro \
+                       -v /tmp/t1w:/scratch -w /scratch \
                        poldracklab/mriqc:latest \
                        /data derivatives/ group \
                        -m T1w -vv
@@ -303,7 +309,7 @@ jobs:
           name: Checking changes on MD5 sums of intermediate files of MRIQC
           command: |
             mkdir -p /tmp/t1w/test
-            export HASHCMD="docker run --rm=false -i -v $PWD:/scratch -w /scratch \
+            export HASHCMD="docker run -u $( id -u ) --rm=false -i -v $PWD:/scratch -w /scratch \
                                        --entrypoint=/usr/local/miniconda/bin/nib-hash \
                                        poldracklab/mriqc:latest"
             find /tmp/t1w/work -name "*.nii.gz" -type f  | sed s+/tmp/t1w/++ | sort | xargs -n1 $HASHCMD >> /tmp/t1w/test/nii_outputs.txt
@@ -324,7 +330,7 @@ jobs:
             docker run --rm=false -ti -v $PWD:/scratch -w /scratch \
               --entrypoint="dfcheck" poldracklab/mriqc:latest \
               -i /scratch/derivatives/group_T1w.tsv \
-              -r /usr/local/src/mriqc/mriqc/data/testdata/group_T1w.tsv
+              -r /src/mriqc/mriqc/data/testdata/group_T1w.tsv
 
       - run:
           name: WebAPI - Check records
@@ -340,7 +346,7 @@ jobs:
           command: |
             docker run --rm=false -ti -v $PWD:/scratch -w /scratch \
                        --entrypoint=mriqc_clf poldracklab/mriqc:latest \
-                       --train --test -P /usr/local/miniconda/lib/python3.6/site-packages/mriqc/data/mclf_run-20170724-191452_mod-rfc_ver-0.9.7-rc8_class-2_cv-loso_data-all_settings.yml -v
+                       --train --test -P /usr/local/miniconda/lib/python3.7/site-packages/mriqc/data/mclf_run-20170724-191452_mod-rfc_ver-0.9.7-rc8_class-2_cv-loso_data-all_settings.yml -v
             # Run the classifier on the test data
             docker run --rm=false -ti -v $PWD:/scratch -w /scratch \
                        --entrypoint=mriqc_clf poldracklab/mriqc:latest \
@@ -380,30 +386,35 @@ jobs:
             docker-compose -f /tmp/src/mriqcwebapi/dockereve-master/docker-compose.yml --verbose up -d
           background: true
 
+      - restore_cache:
+          keys:
+            - bold-v1-{{ epoch }}
+            - bold-v1-
+
       - run:
           name: Run participant-level on BOLD images
           no_output_timeout: 2h
           command: |
             mkdir -p /tmp/bold/work /tmp/bold/derivatives
-            sudo setfacl -d -m group:$(id -gn):rwx /tmp/bold/derivatives
-            sudo setfacl -m group:$(id -gn):rwx /tmp/bold/derivatives
-            sudo setfacl -d -m group:$(id -gn):rwx /tmp/bold/work
-            sudo setfacl -m group:$(id -gn):rwx /tmp/bold/work
             # Run MRIQC
-            docker run --rm=false -ti -v /tmp/data/${TEST_DATA_NAME}:/data:ro \
+            docker run -u $( id -u ) --rm=false -ti -v /tmp/data/${TEST_DATA_NAME}:/data:ro \
                        -v $PWD:/scratch -w /scratch \
                        poldracklab/mriqc:latest \
                        /data derivatives/ participant \
                        -vv --verbose-reports --profile -m bold --dsname circletests \
                        --n_procs 2 --ants-nthreads 1 --ants-float \
                        --testing --ica \
                        --webapi-url http://$( hostname -I | awk '{print $1}' )/api/v1 --upload-strict
+      - save_cache:
+         key: bold-v1-{{ epoch }}
+         paths:
+            - /tmp/bold/work
 
       - run:
           name: Run group-level on BOLD images
           no_output_timeout: 2h
           command: |
-            docker run --rm=false -ti -v /tmp/data/${TEST_DATA_NAME}:/data:ro \
+            docker run -u $( id -u ) --rm=false -ti -v /tmp/data/${TEST_DATA_NAME}:/data:ro \
                        -v $PWD:/scratch -w /scratch \
                        poldracklab/mriqc:latest \
                        /data derivatives/ group \
@@ -424,7 +435,7 @@ jobs:
           name: Checking changes on MD5 sums of intermediate files of MRIQC
           command: |
             mkdir -p /tmp/bold/test
-            export HASHCMD="docker run --rm=false -i -v $PWD:/scratch -w /scratch \
+            export HASHCMD="docker run -u $( id -u ) --rm=false -i -v $PWD:/scratch -w /scratch \
                                        --entrypoint=/usr/local/miniconda/bin/nib-hash \
                                        poldracklab/mriqc:latest"
             find /tmp/bold/work -name "*.nii.gz" -type f  | sed s+/tmp/bold/++ | sort | xargs -n1 $HASHCMD >> /tmp/bold/test/nii_outputs.txt
@@ -442,10 +453,10 @@ jobs:
       - run:
           name: Checking changes on IQMs
           command: |
-            docker run --rm=false -ti -v $PWD:/scratch -w /scratch \
+            docker run -u $( id -u ) --rm=false -ti -v $PWD:/scratch -w /scratch \
               --entrypoint="dfcheck" poldracklab/mriqc:latest \
               -i /scratch/derivatives/group_bold.tsv \
-              -r /usr/local/src/mriqc/mriqc/data/testdata/group_bold.tsv
+              -r /src/mriqc/mriqc/data/testdata/group_bold.tsv
 
       - run:
           name: WebAPI - Check records