ENH: prepare docker image using neurodebian snapshots

README.md

@@ -5,7 +5,10 @@
 #### Information queried from NITRC-IR and stored in a google spreadsheet
 https://docs.google.com/spreadsheets/d/11an55u9t2TAf0EV2pHN0vOd8Ww2Gie-tHp9xGULh_dA
 
-#### Setup your own environment
+#### Within your current environment
+
+##### Setup
+
 1. Make sure FSL is available in your environment and accessible from the command line.
 
 2. If you already have a `conda` environment, please follow the detailed steps below. 
@@ -17,7 +20,7 @@ curl -Ok https://raw.githubusercontent.com/ReproNim/simple_workflow/e4063fa95cb4
 source Simple_Prep.sh
 ```
 
-#### Run the demo
+##### Run the demo
 
 ```bash
 python run_demo_workflow.py --key 11an55u9t2TAf0EV2pHN0vOd8Ww2Gie-tHp9xGULh_dA
@@ -29,7 +32,7 @@ To run on one subject you can do:
 python run_demo_workflow.py --key 11an55u9t2TAf0EV2pHN0vOd8Ww2Gie-tHp9xGULh_dA -n 1
 ```
 
-### Detailed steps for setting up environment
+#### Detailed steps for setting up environment
 
 Install miniconda if you do not have it.
 
@@ -64,3 +67,44 @@ conda env create -f environment.yml
 source activate bh_demo
 pip install https://github.com/satra/prov/archive/enh/rdf-1.x.zip
 ```
+
+### Within Docker
+
+Using containerization solutions, such as docker, allows to create
+multiple complete computation environments while varying versions of any
+analysis pipeline components or inputs.  You could use [Simple_Prep_docker](Simple_Prep_docker)
+script to generate environments based on previous [Debian](http://www.debian.org) or [Ubuntu](http://ubuntu.com) releases
+for which [NeuroDebian](http://neuro.debian.net) builds of FSL [were available in the past](http://snapshot-neuro.debian.net:5002/package/fsl).
+
+N.B.  ATM NeuroDebian snapshots repository is not widely open yet, so if
+you would like to browse it, please "knock" first by running
+`curl -s http://neuro.debian.net/_files/knock-snapshots` command in your shell.
+
+#### Generate an environment
+
+For an example we will generate an environment based on Debian jessie
+release with FSL 5.0.8-3 as it was available in March of 2015:
+
+```bash
+./Simple_Prep_docker jessie 20150306T060524Z
+```
+
+which will generate a local docker image `repronim:simple_prep_USER_jessie_20150306T060524Z`
+(`USER` will correspond to your user name), with all necessary for computation
+components installed.
+
+#### Run the demo
+
+You can normally run a demo with only **one additional step** necessary -- setting up
+environment variables (to point to FSL binaries and enable  conda environment):
+
+```bash
+docker run -it --rm repronim:simple_prep_USER_jessie_20150306T060524Z /bin/bash -c '
+  . setup_environment;
+  cd simple_workflow-master
+  && python run_demo_workflow.py --key 11an55u9t2TAf0EV2pHN0vOd8Ww2Gie-tHp9xGULh_dA
+  && python check_output.py'
+```
+
+which would generate a new temporary container, perform analysis, run
+the check, and quit.

Simple_Prep_docker

@@ -0,0 +1,60 @@
+#!/bin/bash
+#emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: t -*- 
+#ex: set sts=4 ts=4 sw=4 noet:
+# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
+#
+#   See COPYING file distributed along with the XXX package for the
+#   copyright and license terms.
+#
+# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
+#
+# Helper to generate a Docker instance given release and snapshots date specification
+# and run Simple_Prep.sh analysis
+#
+set -e
+export PS4=+
+#set -x
+set -u
+
+DL_DIST=${1:-jessie}
+DL_DATE=${2:-}
+
+topdir=$(readlink -f `dirname $0`)
+cd `dirname $0`
+dockerfile=$topdir/Simple_Prep_docker-Dockerfile
+# echo "D: $DL_APT"
+# Some substitutes aren't used but left for possibly later use
+sed -e "s,DL_DIST,$DL_DIST,g" \
+    -e "s,DL_USER,$USER,g" \
+    -e "s,DL_UID,`id -u`,g" \
+    -e "s,DL_GID,`id -g`,g" \
+    -e "s,DL_GIT_USER_EMAIL,`git config --get user.email`,g" \
+    -e "s,DL_GIT_USER_NAME,`git config --get user.name`,g" \
+    $dockerfile.in >| $dockerfile
+
+if [ ! -z "$DL_DATE" ]; then
+{
+    # replace with snapshots
+    echo "sed -i -e 's,http://neuro.debian.net/debian/* ,http://snapshot-neuro.debian.net:5002/archive/neurodebian/$DL_DATE/ ,g' etc/apt/sources.list.d/neurodebian.sources.list;"
+    # for now we need to knock to expose snapshots repository
+    echo "curl -s http://neuro.debian.net/_files/knock-snapshots;"
+    # TODO: figure out and add snapshot for DL_DIST itself if available and force possible downgrade somehow
+    echo "eatmydata apt-get update; eatmydata apt-get dist-upgrade -y;"
+} | while read aptline; do
+    sed -i -e "s|\(\(.*\)DL_APT\(.*\)\)|\2$aptline\3\n\1|g" $dockerfile
+    :
+done
+fi
+sed -e '/DL_APT/d' -i $dockerfile
+
+tag_=simple_prep_${USER}_${DL_DIST}_${DL_DATE}
+tag=repronim:${tag_}
+echo "I: tag $tag"
+if docker images | grep -q repronim.*${tag_}; then
+    echo "I: tag $tag already exists -- skipping rebuilding"
+else
+    docker build -t $tag -f $dockerfile .
+    #docker build --no-cache=True -t $tag -f $dockerfile . #&& rm Dockerfile
+    :
+fi
+

Simple_Prep_docker-Dockerfile.in

@@ -0,0 +1,67 @@
+FROM neurodebian:DL_DIST
+MAINTAINER Yaroslav Halchenko <debian@onerussian.com>
+
+USER root
+
+# Speed up installation using our apt cacher - modify conf/etc/apt/apt.conf.d/99apt-cacher if you have any
+RUN mkdir -p /etc/apt/apt.conf.d/
+COPY conf/etc/apt/apt.conf.d/99apt-cacher /etc/apt/apt.conf.d/99apt-cacher
+RUN chmod a+r /etc/apt/apt.conf.d/99apt-cacher
+
+# Make deb-src avail
+# RUN sed -i -e 's,^deb\(.*\),deb\1\ndeb-src\1,g' /etc/apt/sources.list.d/neurodebian.sources.list /etc/apt/sources.list
+
+# Make contrib and non-free avail for FSL
+RUN sed -i -e 's, main$, main contrib non-free,g' /etc/apt/sources.list.d/neurodebian.sources.list
+
+# Assure popcon doesn't kick in
+RUN bash -c "echo 'debconf debconf/frontend select noninteractive' | debconf-set-selections -"
+
+RUN apt-get update
+# Use bash for extended syntax
+RUN apt-get install -y -q eatmydata
+# Some rudimentary tools if we need to do anything within docker and curl and unzip needed for setting up conda
+RUN bash -c "eatmydata apt-get install -y -q vim less man-db curl unzip bzip2"
+# Run additional lines, primarily to setup/enable snapshots repository etc
+RUN bash -c "DL_APT"
+# Install fsl-complete
+#RUN bash -c "eatmydata apt-get install -y -q fsl-complete"
+# We might be just fine with the core here
+RUN bash -c "eatmydata apt-get install -y -q fsl-core fsl-first-data"
+
+RUN apt-get clean
+
+# Setting up conda environment given simple_workflow specifications
+WORKDIR /opt/repronim/simple_workflow
+RUN curl -Ok https://raw.githubusercontent.com/ReproNim/simple_workflow/e4063fa95cb494da496565ec27c4ffe8a4901c45/Simple_Prep.sh
+# conda installations take way too long -- might benefit from setting up
+# to use proxy
+# http://stackoverflow.com/a/31120854
+RUN bash Simple_Prep.sh
+
+#
+# There seems to be no easy consistent way to load our customizations to env
+# variables for both interactive and non-interactive shells. So let's create
+# a file which would have all the necessary tuneups which would be passed
+# explicitly into bash
+#
+RUN bash -c 'echo -e "echo IN setup_environments\n. /etc/fsl/fsl.sh\nexport PATH=/root/miniconda2/bin:\$PATH\nsource activate bh_demo\n" > setup_environment'
+
+# Make fsl available in the containers by pointing ENV variable to it
+# which both bash and dash (ubuntu) should warrant
+# ENV ENV /etc/fsl/fsl.sh
+# not effective :-/ for now let's try to place in bashrc
+# RUN bash -c "echo '. /etc/fsl/fsl.sh' >> /etc/bash.bashrc"
+
+# Tune the environment variables
+#ENV PATH /root/miniconda2/bin:$PATH
+# RUN bash -c "echo 'source activate bh_demo' >> /etc/bash.bashrc"
+
+# Tune bash behavior so it loads our environment setup even in non-interactive mode
+ENV ENV /opt/repronim/simple_workflow/setup_environment
+RUN bash -c 'echo -e ". /opt/repronim/simple_workflow/setup_environment" >> /etc/profile'
+
+# Let's setup user matching user
+## RUN groupadd --gid DL_GID -r DL_USER && useradd -m --uid DL_UID -g DL_USER DL_USER
+
+## CMD ["/bin/bash"]

conf/etc/apt/apt.conf.d/99apt-cacher

@@ -0,0 +1,2 @@
+#Acquire::http { Proxy "http://172.17.42.1:3142"; };
+#Acquire::http { Proxy "http://smaug.datalad.org:3142"; };