Major Docker refactor

.rep_version

@@ -1,5 +1,4 @@
 # version of images that are created from current commit. 
 # 
 #
-REP_BASE_IMAGE=yandex/rep-base:0.6.1
 REP_IMAGE=yandex/rep:0.6.5

Makefile

@@ -1,103 +1,80 @@
-# Makefile for building & starting rep-containers
+# Makefile for building & starting REP containers
 # arguments can be supplied by -e definitions: 
 #
-#    ENV -- filename with environment variables passed to docker container
-#    NOTEBOOKS -- folder with notebooks that will be mounted into docker container
-#    PORT -- port to listen for incoming connection
+#	NOTEBOOKS -- folder to be mounted to container (default: ./notebooks)
+#	PORT -- port to listen for incoming connection (default: 8888)
 #
-#
-
 
-define read_cmd_args
-	# use the rest as arguments for "run" or "exec"
-	CMD_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
-endef
-
-ifeq (exec, $(firstword $(MAKECMDGOALS)))
-  $(eval $(call read_cmd_args))
-  ifeq ("", "$(CMD_ARGS)")
-    CMD_ARGS := bash
-  endif
-endif
-ifeq (run, $(firstword $(MAKECMDGOALS)))
-  $(eval $(call read_cmd_args))
-endif
-ifneq ("", "$(CMD_ARGS)")
-  # turn args into do-nothing targets  
-  $(eval $(CMD_ARGS):;@:)  
-endif
-
-CONTAINER_NAME := $(shell basename $(CURDIR) | tr - _ )
+CONTAINER_NAME := rep
 NOTEBOOKS ?= $(shell pwd)/notebooks
-VOLUMES := -v $(NOTEBOOKS):/notebooks
 PORT ?= 8888
-DOCKER_ARGS := $(VOLUMES) -p $(PORT):8888
-ifneq ("", "$(ENV)")
-  DOCKER_ARGS := $(DOCKER_ARGS) --env-file=$(ENV)
-endif
-
+DOCKER_ARGS := --volume $(NOTEBOOKS):/notebooks -p $(PORT):8888
 
-include .rep_version  # read REP_BASE_IMAGE, and REP_IMAGE
+HERE := $(shell pwd)
+include .rep_version  # read REP_IMAGE
 
 help:
 	@echo Usage: make [-e VARIABLE=VALUE] targets
 	@echo "variables:"
 	@grep -h "#\s\+\w\+ -- " $(MAKEFILE_LIST) |sed "s/#\s//"
 	@echo
-	@echo targets and corresponding dependencies:
+	@echo targets:
 	@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' -e 's/^/   /' | sed -e 's/##//'
 
-.PHONY: run run-daemon restart logs rep-image rep-base-image \
+.PHONY: run rep-image2 rep-image3 run-daemon restart logs  \
 	inspect exec help stop remove push push-base tag-latest push-latest
 
 version:
-	@echo $(REP_IMAGE), $(REP_BASE_IMAGE)
+	@echo $(REP_IMAGE)
 
-rep-base-image:	## build REP base image
-	source .version && docker build -t $(REP_BASE_IMAGE) -f ci/Dockerfile.base ci
+rep-image2:	## build REP image with python 2
+	docker build --build-arg REP_PYTHON_VERSION=2 -t $(REP_IMAGE) -f ci/Dockerfile.rep .
 
-rep-base-image3:
-	TRAVIS_PYTHON_VERSION=3.4 docker build -t $(REP_BASE_IMAGE) -f ci/Dockerfile.base ci
+rep-image3:	## build REP image with python 3
+	docker build --build-arg REP_PYTHON_VERSION=3 -t $(REP_IMAGE) -f ci/Dockerfile.rep .
 
-rep-image:	## build REP image
-	docker build -t $(REP_IMAGE) -f ci/Dockerfile.rep .
-
-local-dirs:
+local-dirs: # creates a local directory to be mounted to REP container
 	[ -d $(NOTEBOOKS) ] || mkdir -p $(NOTEBOOKS)
 
 run: local-dirs		## run REP interactively
-	docker run -ti --rm $(DOCKER_ARGS) --name $(CONTAINER_NAME) $(REP_IMAGE) $(CMD_ARGS) 
+	docker run --interactive --tty --rm $(DOCKER_ARGS) --name $(CONTAINER_NAME) $(REP_IMAGE)
 
 run-daemon: local-dirs	## run REP as a daemon
-	docker run -d $(DOCKER_ARGS) --name $(CONTAINER_NAME) $(REP_IMAGE) $(CMD_ARGS) 
+	docker run --detach $(DOCKER_ARGS) --name $(CONTAINER_NAME) $(REP_IMAGE)
+
+run-tests:  ## run tests inside a container, both notebooks and scripts
+	find tests -name '*.pyc' -delete
+	# for some reason nosetests fails if directly mounted to tests folder
+	mkdir -p $(HERE)/_docker_tests/
+	cp -r $(HERE)/tests $(HERE)/_docker_tests/
+	cp -r $(HERE)/howto $(HERE)/_docker_tests/
+	docker run  --interactive --tty --rm --volume $(HERE)/_docker_tests:/notebooks $(REP_IMAGE) \
+		/bin/bash -l -c "cd /notebooks/tests && nosetests -v --detailed-errors --nocapture . "
 
 restart:	## restart REP container
 	docker restart $(CONTAINER_NAME)
 
-exec:		## run command within REP container
-	docker exec -ti $(CONTAINER_NAME) $(CMD_ARGS)
+exec:       ## run command within REP container
+	docker exec -ti $(CONTAINER_NAME)
 
-logs:		## show container logs
+show-logs:  ## show container logs
 	docker logs $(CONTAINER_NAME)
 
-stop:		## stop REP container
+stop:       ## stop REP container
 	docker stop $(CONTAINER_NAME)
 
-remove: stop	## remove REP container
+remove: stop    ## remove REP container
 	docker rm $(CONTAINER_NAME)
 
-inspect:	## inspect REP image
-	docker inspect $(REP_IMAGE) 
+inspect:	# inspect REP image
+	docker inspect $(REP_IMAGE)
 
-push: rep-image	## build REP image & push to docker hub
+push: rep-image2	# build REP image & push to docker hub
 	@docker login -e="$(DOCKER_EMAIL)" -u="$(DOCKER_USERNAME)" -p="$(DOCKER_PASSWORD)"
 	docker push $(REP_IMAGE)
 
-push-base:	## push base image to docker hub
-	docker push $(REP_BASE_IMAGE)
-
-tag-latest: rep-image	## tag current REP image as latest
+tag-latest: rep-image2	# tag current REP image as latest
 	docker tag -f $(REP_IMAGE) yandex/rep:latest
 
-push-latest: tag-latest push	## tag current REP image as latest and push it to docker hub
+push-latest: tag-latest push	# tag current REP image as latest and push it to docker hub
 	docker push yandex/rep:latest

ci/Dockerfile.rep

@@ -1,49 +1,45 @@
-FROM yandex/rep-base:0.6.1
-MAINTAINER Andrey Ustyuzhanin <anaderi@yandex-team.ru>
+FROM ubuntu:14.04
+MAINTAINER Andrey Ustyuzhanin <anaderi@yandex-team.ru>, Alex Rogozhnikov <axelr@yandex-team.ru>
 
-ENV TEMP /tmp
-ENV SHELL /bin/bash
-ENV PORT_JUPYTER=8888
-ENV HOME /root
+# This variable is set during building docker container
+ARG REP_PYTHON_VERSION=2
 
-# copying howto
-COPY ./howto /REP_howto
+# Setting environment variables in the container
+ENV HOME=/root  \
+    TEMP=/tmp  \
+    PORT_JUPYTER=8888  \
+    SHELL=/bin/bash
+
+# Creating folders in container
+RUN mkdir -p $TEMP/build \
+ && mkdir -p /notebooks \
+ && mkdir -p /REP_howto
 
-# copying everything
-RUN mkdir $TEMP/build
+# Copy REP to build folder
 COPY ./ $TEMP/build/
-# copy scripts
-COPY ./ci/*.sh $HOME/
-RUN chmod +x $HOME/*.sh
-
-# Debugging TODO delete
-RUN ls $HOME
-RUN ls /REP_howto
-RUN ls $TEMP/build
-
-RUN /bin/bash --login -c " \
-  cd $TEMP/build && \
-  pip install . && \
-  cd $HOME && \
-  rm -rf $TEMP/build"
-
-# TODO MOVE
-# IPython setup
-RUN /bin/bash --login -c " \
-  jupyter notebook -y --generate-config &&\
-  /bin/echo -e \"\
-c.NotebookApp.ip = '*' \n\
-c.NotebookApp.open_browser = False \n\
-c.NotebookApp.port = $PORT_JUPYTER \n \" \
-  | tee -a $HOME/.jupyter/jupyter_notebook_config.py \
-  "
-
-# create folder for notebooks
-RUN mkdir /notebooks
+# Copy HowTos separately
+COPY ./howto /REP_howto
+
+# changing default shell to bash. See http://stackoverflow.com/questions/20635472/
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh
+
+# installing environment,
+# adding profiles to .bashrc,
+# pip-installing rep itself
+# deleting folder
+RUN source $TEMP/build/ci/install_repbase.sh $REP_PYTHON_VERSION \
+ && echo "source /etc/profile.d/rep_profile.sh" >> $HOME/.bashrc \
+ && echo "umask 0002" >> $HOME/.bashrc \
+ && pip install $TEMP/build \
+ && rm -rf $TEMP/build
 
 # registering mounting points
 VOLUME ["/notebooks"]
 # registering port
 EXPOSE $PORT_JUPYTER
-# starting IPython process
+
+# adding a file to run jupyter
+COPY ./ci/start_jupyter.sh $HOME/
+
+# starting IPython process when image is started
 CMD ["/bin/bash", "--login", "-c", "$HOME/start_jupyter.sh"]

ci/environment-jupyterhub_py3.yaml

@@ -3,13 +3,5 @@ name: jupyterhub_py3
 dependencies:
   - python=3.4
   - nomkl
-  # TODO do we need anything from below?
-  - ipython-notebook=4.0.4
-  - matplotlib=1.5.1
-  - numpy=1.10.4
-  - pandas=0.17.1
-  - scikit-learn=0.17
-  - scipy=0.17.0
-  - six=1.10.0
   - pip:
     - jupyterhub

ci/environment-rep-template.yaml

@@ -4,7 +4,6 @@ channels:
 dependencies:
   - python={PYTHON_VERSION}
   - nomkl
-  - ipython-notebook=4.0.4
   - matplotlib=1.5.1
   - numpy=1.10.4
   - pandas=0.17.1
@@ -15,10 +14,11 @@ dependencies:
   - rootpy
   - root=5*
   - pip:
+    - notebook==4.2.1
     - bokeh==0.11.1
     - neurolab==0.3.5
     - theano==0.8.2
-    - theanets==0.6.2
+    - theanets==0.7.3
     - pybrain==0.3
     - mpld3==0.2
     - hep_ml==0.4

ci/environment-rep2.yaml

@@ -1,10 +1,10 @@
+# This file is AUTOMATICALLY GENERATED BY SCRIPT, don't modify it! 
 name: rep_py2
 channels:
   - nlesc
 dependencies:
   - python=2.7
   - nomkl
-  - ipython-notebook=4.0.4
   - matplotlib=1.5.1
   - numpy=1.10.4
   - pandas=0.17.1
@@ -15,10 +15,11 @@ dependencies:
   - rootpy
   - root=5*
   - pip:
+    - notebook==4.2.1
     - bokeh==0.11.1
     - neurolab==0.3.5
     - theano==0.8.2
-    - theanets==0.6.2
+    - theanets==0.7.3
     - pybrain==0.3
     - mpld3==0.2
     - hep_ml==0.4

ci/environment-rep3.yaml

@@ -1,10 +1,10 @@
+# This file is AUTOMATICALLY GENERATED BY SCRIPT, don't modify it! 
 name: rep_py3
 channels:
   - nlesc
 dependencies:
   - python=3.4
   - nomkl
-  - ipython-notebook=4.0.4
   - matplotlib=1.5.1
   - numpy=1.10.4
   - pandas=0.17.1
@@ -15,10 +15,11 @@ dependencies:
   - rootpy
   - root=5*
   - pip:
+    - notebook==4.2.1
     - bokeh==0.11.1
     - neurolab==0.3.5
     - theano==0.8.2
-    - theanets==0.6.2
+    - theanets==0.7.3
     - pybrain==0.3
     - mpld3==0.2
     - hep_ml==0.4

ci/environments_generate.py

@@ -1,8 +1,8 @@
 """
 Generates environment files separately for python 2 and python 3. (rep_py2, rep_py3)
-SImply substitutes fields in the template
-
+Simply substitutes fields in the template. Usage:
 
+python environments_generate.py
 """
 from __future__ import division, print_function, absolute_import
 import os
@@ -18,5 +18,6 @@
     python_major_version = python_version[:1]
     new_content = content.replace('{PYTHON_MAJOR_VERSION}', python_major_version)\
                          .replace('{PYTHON_VERSION}', python_version)
+    new_content = "# This file is AUTOMATICALLY GENERATED BY SCRIPT, don't modify it! \n" + new_content
     with open(os.path.join(here, 'environment-rep' + python_major_version + '.yaml'), 'w') as new_file:
         new_file.write(new_content)