new Docker configs

supernnova · Apr 1, 2020 · 3154d8a · 3154d8a
1 parent 756d6c1
commit 3154d8a
Show file tree

Hide file tree

Showing 8 changed files with 244 additions and 57 deletions.
diff --git a/docs/installation/python.rst b/docs/installation/python.rst
@@ -22,15 +22,14 @@ Docker
 You can also use docker:
 
 - Install docker: `Docker`_.
-- For GPU support, install nvidia docker: `NVDocker`_.
 
 Create a docker image:
 
 .. code::
 
-    cd docker && make {device}
+    cd env && make {image}
 
-where ``device`` is one of ``cpu`` or ``gpu``
+where ``image`` is one of ``cpu`` or ``gpu`` (for cuda 9.) or ``gpu10`` (for cuda 10.)
 
 - This images contains all of this repository's dependencies.
 - Image construction will typically take a few minutes
@@ -39,12 +38,11 @@ Enter docker environment by calling:
 
 .. code::
 
-    python launch_docker.py
+    python launch_docker.py --image <image> --dump_dir </path/to/data>
 
-- Add ``--use_cuda`` to launch a GPU-supported container
+- Add ``--image image`` where image is ``cpu`` or ``gpu`` (for cuda 9.) or ``gpu10`` (for cuda 10.)
 - Add ``--dump_dir /path/to/data`` to mount the folder where you stored the data (see :ref:`DataStructure`) into the container. If unspecified, will use the default location (i.e. ``snndump``)
 
 This will launch an interactive session in the docker container, with zsh support.
 
-.. _Docker: https://docs.docker.com/install/linux/docker-ce/ubuntu/
-.. _NVDocker: https://github.com/NVIDIA/nvidia-docker
+.. _Docker: https://docs.docker.com/install/linux/docker-ce/ubuntu/
diff --git a/env/Dockerfile.gpu b/env/Dockerfile.gpu
@@ -65,22 +65,58 @@ RUN    /home/miniconda3/bin/pip install \
     /home/miniconda3/bin/pip install sphinx sphinx-autobuild sphinxcontrib-napoleon sphinx_rtd_theme tabulate
 
 # Extra python packages
-RUN /home/miniconda3/bin/pip install seaborn pytest-sugar pytest-cov mxnet-cu90
+RUN /home/miniconda3/bin/pip install seaborn pytest-sugar pytest-cov
+
+
+ENV PATH=$PATH:/u/home/.local/bin
 
 ####################################
 # Set up locale to avoid zsh errors
 ####################################
 RUN echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \
     && locale-gen --purge --lang en_US \
     && locale-gen
-ENV LANG en_US.utf8
+
+################################################
+# Set up the environment variables
+# Used in installing following libs as well
+################################################
+
+ENV TMP_PATH=/u/home/dump \
+    LD_LIBRARY_PATH="/usr/local/lib:/home/miniconda3/lib:$LD_LIBRARY_PATH" \
+    PATH=/home/miniconda3/bin:$PATH \
+    LANG=en_US.utf8 \
+    USER=researcher \
+    USER_ID=2018 \
+    USER_GID=2018
+
+#######################################
+# Adding a user which will be mapped to
+# real user running the docker
+#######################################
+
+RUN mkdir -p /u/home && \
+    groupadd --gid "${USER_GID}" "$USER" && \
+    useradd \
+    --uid ${USER_ID} \
+    --gid ${USER_GID} \
+    --home-dir /u/home \
+    --shell /usr/bin/zsh \
+    ${USER}
 
 ####################################
 # Set up oh my zsh
 ####################################
-RUN git clone https://github.com/robbyrussell/oh-my-zsh.git ${HOME}/.oh-my-zsh
-COPY zshrc ${HOME}/.zshrc
-RUN sed -i 's/❯/Docker❯/g' /home/.oh-my-zsh/themes/refined.zsh-theme
+WORKDIR /u/home
+COPY zshrc /u/home/.zshrc
+RUN git clone https://github.com/robbyrussell/oh-my-zsh.git /u/home/.oh-my-zsh &&\
+    sed -i 's/❯/Docker❯/g' /u/home/.oh-my-zsh/themes/refined.zsh-theme
+# Path to your oh-my-zsh installation.
+ENV ZSH=/u/home/.oh-my-zsh \
+    PATH_TO_COPY=${PATH}
 
+COPY entry_script.sh /
+RUN chmod u+x /entry_script.sh
+ENV HOME=/u/home
 
-ENTRYPOINT ["/bin/zsh"]
+ENTRYPOINT ["/bin/bash", "/entry_script.sh"]
diff --git a/env/Dockerfile.gpu10 b/env/Dockerfile.gpu10
@@ -0,0 +1,122 @@
+FROM nvidia/cuda:10.2-cudnn7-devel-ubuntu16.04
+
+####################################
+# Apt libraries
+####################################
+RUN apt-get update && apt-get install -y --no-install-recommends\
+        build-essential \
+        apt-utils \
+        cmake \
+        git \
+        less \
+        zsh \
+        curl \
+        vim \
+        graphviz \
+        libjpeg-dev \
+        libpng-dev \
+        gfortran \
+        zlib1g-dev \
+        automake \
+        autoconf \
+        git \
+        libtool \
+        subversion \
+        libatlas3-base \
+        ffmpeg \
+        python-pip \
+        python-dev \
+        wget \
+        libncurses5-dev \
+        libncursesw5-dev \
+        unzip \
+        sox \
+        locales &&\
+     rm -rf /var/lib/apt/lists/*
+
+WORKDIR /home
+ENV HOME /home
+
+###################################
+# Anaconda + python deps
+###################################
+#  https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+WORKDIR /home
+RUN curl -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh  && \
+     chmod +x ~/miniconda.sh && \
+     ~/miniconda.sh -b -p /home/miniconda3 && \
+     rm ~/miniconda.sh
+
+# RUN    /home/miniconda3/bin/conda install -c anaconda tensorflow-gpu
+RUN    /home/miniconda3/bin/conda install -y pytorch==0.4.1 torchvision -c pytorch
+RUN    /home/miniconda3/bin/pip install \
+     h5py \
+     matplotlib \
+     colorama \
+     tqdm \
+     scipy \
+     natsort \
+     pandas \
+     astropy \
+     ipdb \
+     scikit-learn \
+     pytest \
+     unidecode &&\
+    /home/miniconda3/bin/pip install sphinx sphinx-autobuild sphinxcontrib-napoleon sphinx_rtd_theme tabulate
+
+# Extra python packages
+RUN /home/miniconda3/bin/pip install seaborn pytest-sugar pytest-cov
+
+
+ENV PATH=$PATH:/u/home/.local/bin
+
+####################################
+# Set up locale to avoid zsh errors
+####################################
+RUN echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \
+    && locale-gen --purge --lang en_US \
+    && locale-gen
+
+################################################
+# Set up the environment variables
+# Used in installing following libs as well
+################################################
+
+ENV TMP_PATH=/u/home/dump \
+    LD_LIBRARY_PATH="/usr/local/lib:/home/miniconda3/lib:$LD_LIBRARY_PATH" \
+    PATH=/home/miniconda3/bin:$PATH \
+    LANG=en_US.utf8 \
+    USER=researcher \
+    USER_ID=2018 \
+    USER_GID=2018
+
+#######################################
+# Adding a user which will be mapped to
+# real user running the docker
+#######################################
+
+RUN mkdir -p /u/home && \
+    groupadd --gid "${USER_GID}" "$USER" && \
+    useradd \
+    --uid ${USER_ID} \
+    --gid ${USER_GID} \
+    --home-dir /u/home \
+    --shell /usr/bin/zsh \
+    ${USER}
+
+####################################
+# Set up oh my zsh
+####################################
+WORKDIR /u/home
+COPY zshrc /u/home/.zshrc
+RUN git clone https://github.com/robbyrussell/oh-my-zsh.git /u/home/.oh-my-zsh &&\
+    sed -i 's/❯/Docker❯/g' /u/home/.oh-my-zsh/themes/refined.zsh-theme
+# Path to your oh-my-zsh installation.
+ENV ZSH=/u/home/.oh-my-zsh \
+    PATH_TO_COPY=${PATH}
+
+COPY entry_script.sh /
+RUN chmod u+x /entry_script.sh
+ENV HOME=/u/home
+
+ENTRYPOINT ["/bin/bash", "/entry_script.sh"]
diff --git a/env/entry_script.sh b/env/entry_script.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+set -e
+
+
+if [ -z "${USER}" ]; then
+  echo "We need USER to be set!"; exit 100
+fi
+
+# if both not set we do not need to do anything
+if [ -z "${HOST_USER_ID}" -a -z "${HOST_USER_GID}" ]; then
+    echo "Nothing to do here." ; exit 0
+fi
+
+# reset user_?id to either new id or if empty old (still one of above
+# might not be set)
+USER_ID=${HOST_USER_ID:=$USER_ID}
+USER_GID=${HOST_USER_GID:=$USER_GID}
+
+LINE=$(grep -F "${USER}" /etc/passwd)
+# replace all ':' with a space and create array
+array=( ${LINE//:/ } )
+
+# home is 5th element
+USER_HOME=/u/home
+
+sed -i -e "s/^${USER}:\([^:]*\):[0-9]*:[0-9]*/${USER}:\1:${USER_ID}:${USER_GID}/"  /etc/passwd
+sed -i -e "s/^${USER}:\([^:]*\):[0-9]*/${USER}:\1:${USER_GID}/"  /etc/group
+
+chown $HOST_USER_ID:$HOST_USER_GID $USER_HOME
+chown $HOST_USER_ID:$HOST_USER_GID /u/home/.zshrc
+chown $HOST_USER_ID:$HOST_USER_GID /u/home/.oh-my-zsh
+
+_term() { 
+  echo "Caught SIGTERM signal!" 
+  kill -TERM "$child" 2>/dev/null
+}
+
+trap _term SIGTERM
+# PATH is reset on user switch but we need to preserve
+
+
+su -p "${USER}"
+
+
diff --git a/env/launch_docker.py b/env/launch_docker.py
@@ -7,44 +7,36 @@
 
 def launch_docker():
 
+    pwd = os.getcwd()
+    UID = os.getuid()
+    snn_dir = os.path.abspath(Path(os.path.dirname(os.path.realpath(__file__))).parent)
+    dump_dir = os.path.abspath(Path(snn_dir).parent)
+
     parser = argparse.ArgumentParser()
-    parser.add_argument("--dump_dir", default="../../snndump", help="Dump dir")
-    parser.add_argument("--use_cuda", action="store_true", help="Use gpu image")
+    parser.add_argument("--image", type=str, choices=['cpu','gpu','gpu10'],help="Use which image gpu or cpu")
+    parser.add_argument("--dump_dir", default=dump_dir, help='Dir to dump results')
 
     args = parser.parse_args()
 
-    pwd = os.getcwd()
-    UID = os.getuid()
-
-    if args.use_cuda:
-        cmd = (
-            f"docker run -it --gpus all --rm --user {UID}"
-            f" -v {pwd}/../../SuperNNova:/home/SuperNNova"
-            f" -v {pwd}/{args.dump_dir}:/home/snndump rnn-gpu:latest"
-        )
-        try:
-            subprocess.check_call(shlex.split(cmd))
-        except Exception as err:
-            print(err)
-            print("Possible errors:")
-            print("You may not have installed nvidia-docker.")
-            print("You may not have a GPU.")
-            print("You may not have built the images ==> call make gpu or make cpu")
-    else:
-
-        cmd = (
-            f"docker run -it --rm --user {UID}"
-            f" -v {pwd}/../../SuperNNova:/home/SuperNNova"
-            f" -v {pwd}/{args.dump_dir}:/home/{Path(args.dump_dir).name} rnn-cpu:latest"
-        )
-
-        try:
-            subprocess.check_call(shlex.split(cmd))
-        except Exception as err:
-            print(err)
-            print("Possible errors:")
-            print("You may not have installed docker.")
-            print("You may not have built the images ==> call make gpu or make cpu")
+    cmd = (
+        f"docker run -it --rm ")
+
+    cmd += f" --gpus all " if 'gpu' in args.image else ""
+
+    cmd += (
+        f" -v {snn_dir}:/u/home/SuperNNova"
+        f" -v {args.dump_dir}:/u/home/snndump"
+        f" -e HOST_USER_ID={os.getuid()} "
+        f" -e HOST_USER_GID={os.getgid()} "
+        f" rnn-{args.image}:latest"
+    )
+    try:
+        subprocess.check_call(shlex.split(cmd))
+    except Exception as err:
+        print(err)
+        print("Possible errors:")
+        print("You may not have a GPU.")
+        print(f"You may not have built the images ==> call make {args.image}")
 
 
 if __name__ == "__main__":

diff --git a/env/zshrc b/env/zshrc
@@ -1,11 +1,9 @@
-# Path to your oh-my-zsh installation.
-export ZSH=/home/.oh-my-zsh
-
 # Theme
 ZSH_THEME="refined"
+export PATH=$PATH_TO_COPY
 
 # Plugins
-plugins=(git autojump wd sm zsh_reload)
+plugins=(git zsh_reload)
 
 source "$ZSH/oh-my-zsh.sh"
 
@@ -14,7 +12,4 @@ if [[ -n $SSH_CONNECTION ]]; then
   export EDITOR='vim'
 else
   export EDITOR='vim'
-fi
-
-# Anaconda path
-export PATH="/home/miniconda3/bin:$PATH"
+fi
diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 
 setuptools.setup(
     name="supernnova",
-    version="0.0.5",
+    version="1.0",
     author="Anais Moller and Thibault de Boissiere",
     author_email = "anais.moller@clermont.in2p3.fr",
     description = "framework for Bayesian, Neural Network based supernova light-curve classification",

diff --git a/supernnova/data/make_dataset.py b/supernnova/data/make_dataset.py
@@ -697,7 +697,7 @@ def make_dataset(settings):
 
     # Aggregate the pivoted dataframe
     list_files = natsorted(
-        glob.glob(os.path.join(settings.preprocessed_dir, f"*pivot.pickle"))
+        glob.glob(os.path.join(settings.preprocessed_dir, f"*pivot.pickle*"))
     )
     logging_utils.print_green("Concatenating pivot")