diff --git a/README.md b/README.md index d6849f4..1ff5288 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,10 @@ #### Information queried from NITRC-IR and stored in a google spreadsheet https://docs.google.com/spreadsheets/d/11an55u9t2TAf0EV2pHN0vOd8Ww2Gie-tHp9xGULh_dA -#### Setup your own environment +#### Within your current environment + +##### Setup + 1. Make sure FSL is available in your environment and accessible from the command line. 2. If you already have a `conda` environment, please follow the detailed steps below. @@ -17,7 +20,7 @@ curl -Ok https://raw.githubusercontent.com/ReproNim/simple_workflow/e4063fa95cb4 source Simple_Prep.sh ``` -#### Run the demo +##### Run the demo ```bash python run_demo_workflow.py --key 11an55u9t2TAf0EV2pHN0vOd8Ww2Gie-tHp9xGULh_dA @@ -29,7 +32,7 @@ To run on one subject you can do: python run_demo_workflow.py --key 11an55u9t2TAf0EV2pHN0vOd8Ww2Gie-tHp9xGULh_dA -n 1 ``` -### Detailed steps for setting up environment +#### Detailed steps for setting up environment Install miniconda if you do not have it. @@ -64,3 +67,44 @@ conda env create -f environment.yml source activate bh_demo pip install https://github.com/satra/prov/archive/enh/rdf-1.x.zip ``` + +### Within Docker + +Using containerization solutions, such as docker, allows to create +multiple complete computation environments while varying versions of any +analysis pipeline components or inputs. You could use [Simple_Prep_docker](Simple_Prep_docker) +script to generate environments based on previous [Debian](http://www.debian.org) or [Ubuntu](http://ubuntu.com) releases +for which [NeuroDebian](http://neuro.debian.net) builds of FSL [were available in the past](http://snapshot-neuro.debian.net:5002/package/fsl). + +N.B. ATM NeuroDebian snapshots repository is not widely open yet, so if +you would like to browse it, please "knock" first by running +`curl -s http://neuro.debian.net/_files/knock-snapshots` command in your shell. + +#### Generate an environment + +For an example we will generate an environment based on Debian jessie +release with FSL 5.0.8-3 as it was available in March of 2015: + +```bash +./Simple_Prep_docker jessie 20150306T060524Z +``` + +which will generate a local docker image `repronim:simple_prep_USER_jessie_20150306T060524Z` +(`USER` will correspond to your user name), with all necessary for computation +components installed. + +#### Run the demo + +You can normally run a demo with only **one additional step** necessary -- setting up +environment variables (to point to FSL binaries and enable conda environment): + +```bash +docker run -it --rm repronim:simple_prep_USER_jessie_20150306T060524Z /bin/bash -c ' + . setup_environment; + cd simple_workflow-master + && python run_demo_workflow.py --key 11an55u9t2TAf0EV2pHN0vOd8Ww2Gie-tHp9xGULh_dA + && python check_output.py' +``` + +which would generate a new temporary container, perform analysis, run +the check, and quit. \ No newline at end of file diff --git a/Simple_Prep_docker b/Simple_Prep_docker new file mode 100755 index 0000000..f558c52 --- /dev/null +++ b/Simple_Prep_docker @@ -0,0 +1,60 @@ +#!/bin/bash +#emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: t -*- +#ex: set sts=4 ts=4 sw=4 noet: +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +# +# See COPYING file distributed along with the XXX package for the +# copyright and license terms. +# +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +# +# Helper to generate a Docker instance given release and snapshots date specification +# and run Simple_Prep.sh analysis +# +set -e +export PS4=+ +#set -x +set -u + +DL_DIST=${1:-jessie} +DL_DATE=${2:-} + +topdir=$(readlink -f `dirname $0`) +cd `dirname $0` +dockerfile=$topdir/Simple_Prep_docker-Dockerfile +# echo "D: $DL_APT" +# Some substitutes aren't used but left for possibly later use +sed -e "s,DL_DIST,$DL_DIST,g" \ + -e "s,DL_USER,$USER,g" \ + -e "s,DL_UID,`id -u`,g" \ + -e "s,DL_GID,`id -g`,g" \ + -e "s,DL_GIT_USER_EMAIL,`git config --get user.email`,g" \ + -e "s,DL_GIT_USER_NAME,`git config --get user.name`,g" \ + $dockerfile.in >| $dockerfile + +if [ ! -z "$DL_DATE" ]; then +{ + # replace with snapshots + echo "sed -i -e 's,http://neuro.debian.net/debian/* ,http://snapshot-neuro.debian.net:5002/archive/neurodebian/$DL_DATE/ ,g' etc/apt/sources.list.d/neurodebian.sources.list;" + # for now we need to knock to expose snapshots repository + echo "curl -s http://neuro.debian.net/_files/knock-snapshots;" + # TODO: figure out and add snapshot for DL_DIST itself if available and force possible downgrade somehow + echo "eatmydata apt-get update; eatmydata apt-get dist-upgrade -y;" +} | while read aptline; do + sed -i -e "s|\(\(.*\)DL_APT\(.*\)\)|\2$aptline\3\n\1|g" $dockerfile + : +done +fi +sed -e '/DL_APT/d' -i $dockerfile + +tag_=simple_prep_${USER}_${DL_DIST}_${DL_DATE} +tag=repronim:${tag_} +echo "I: tag $tag" +if docker images | grep -q repronim.*${tag_}; then + echo "I: tag $tag already exists -- skipping rebuilding" +else + docker build -t $tag -f $dockerfile . + #docker build --no-cache=True -t $tag -f $dockerfile . #&& rm Dockerfile + : +fi + diff --git a/Simple_Prep_docker-Dockerfile.in b/Simple_Prep_docker-Dockerfile.in new file mode 100644 index 0000000..7a7bd5e --- /dev/null +++ b/Simple_Prep_docker-Dockerfile.in @@ -0,0 +1,67 @@ +FROM neurodebian:DL_DIST +MAINTAINER Yaroslav Halchenko + +USER root + +# Speed up installation using our apt cacher - modify conf/etc/apt/apt.conf.d/99apt-cacher if you have any +RUN mkdir -p /etc/apt/apt.conf.d/ +COPY conf/etc/apt/apt.conf.d/99apt-cacher /etc/apt/apt.conf.d/99apt-cacher +RUN chmod a+r /etc/apt/apt.conf.d/99apt-cacher + +# Make deb-src avail +# RUN sed -i -e 's,^deb\(.*\),deb\1\ndeb-src\1,g' /etc/apt/sources.list.d/neurodebian.sources.list /etc/apt/sources.list + +# Make contrib and non-free avail for FSL +RUN sed -i -e 's, main$, main contrib non-free,g' /etc/apt/sources.list.d/neurodebian.sources.list + +# Assure popcon doesn't kick in +RUN bash -c "echo 'debconf debconf/frontend select noninteractive' | debconf-set-selections -" + +RUN apt-get update +# Use bash for extended syntax +RUN apt-get install -y -q eatmydata +# Some rudimentary tools if we need to do anything within docker and curl and unzip needed for setting up conda +RUN bash -c "eatmydata apt-get install -y -q vim less man-db curl unzip bzip2" +# Run additional lines, primarily to setup/enable snapshots repository etc +RUN bash -c "DL_APT" +# Install fsl-complete +#RUN bash -c "eatmydata apt-get install -y -q fsl-complete" +# We might be just fine with the core here +RUN bash -c "eatmydata apt-get install -y -q fsl-core fsl-first-data" + +RUN apt-get clean + +# Setting up conda environment given simple_workflow specifications +WORKDIR /opt/repronim/simple_workflow +RUN curl -Ok https://raw.githubusercontent.com/ReproNim/simple_workflow/e4063fa95cb494da496565ec27c4ffe8a4901c45/Simple_Prep.sh +# conda installations take way too long -- might benefit from setting up +# to use proxy +# http://stackoverflow.com/a/31120854 +RUN bash Simple_Prep.sh + +# +# There seems to be no easy consistent way to load our customizations to env +# variables for both interactive and non-interactive shells. So let's create +# a file which would have all the necessary tuneups which would be passed +# explicitly into bash +# +RUN bash -c 'echo -e "echo IN setup_environments\n. /etc/fsl/fsl.sh\nexport PATH=/root/miniconda2/bin:\$PATH\nsource activate bh_demo\n" > setup_environment' + +# Make fsl available in the containers by pointing ENV variable to it +# which both bash and dash (ubuntu) should warrant +# ENV ENV /etc/fsl/fsl.sh +# not effective :-/ for now let's try to place in bashrc +# RUN bash -c "echo '. /etc/fsl/fsl.sh' >> /etc/bash.bashrc" + +# Tune the environment variables +#ENV PATH /root/miniconda2/bin:$PATH +# RUN bash -c "echo 'source activate bh_demo' >> /etc/bash.bashrc" + +# Tune bash behavior so it loads our environment setup even in non-interactive mode +ENV ENV /opt/repronim/simple_workflow/setup_environment +RUN bash -c 'echo -e ". /opt/repronim/simple_workflow/setup_environment" >> /etc/profile' + +# Let's setup user matching user +## RUN groupadd --gid DL_GID -r DL_USER && useradd -m --uid DL_UID -g DL_USER DL_USER + +## CMD ["/bin/bash"] \ No newline at end of file diff --git a/conf/etc/apt/apt.conf.d/99apt-cacher b/conf/etc/apt/apt.conf.d/99apt-cacher new file mode 100644 index 0000000..428172f --- /dev/null +++ b/conf/etc/apt/apt.conf.d/99apt-cacher @@ -0,0 +1,2 @@ +#Acquire::http { Proxy "http://172.17.42.1:3142"; }; +#Acquire::http { Proxy "http://smaug.datalad.org:3142"; };