Skip to content

Commit

Permalink
feat: add gpu dockerfile and adapt makefile
Browse files Browse the repository at this point in the history
  • Loading branch information
eduardocarvp authored and Optimox committed Dec 3, 2019
1 parent 3b1c4d9 commit 8d14406
Show file tree
Hide file tree
Showing 2 changed files with 204 additions and 3 deletions.
191 changes: 191 additions & 0 deletions Dockerfile_gpu
@@ -0,0 +1,191 @@
# GENERATED FROM SCRIPTS
FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04@sha256:14175ed97d841be1a11c42ff9dcc5a8e50edacc00ce15dce4141a683986e7425
# Avoiding tzdata configuration
ENV DEBIAN_FRONTEND noninteractive
# Adding Python to image
# Dockerfile generated fragment to install Python and Pip
# Source: https://raw.githubusercontent.com/docker-library/python/master/3.7/buster/slim/Dockerfile
# Python: 3.7.5
# Pip: 19.3.1
# ensure local python is preferred over distribution python

ENV PATH /usr/local/bin:$PATH
# http://bugs.python.org/issue19846
# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
ENV LANG C.UTF-8
# runtime dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
netbase \
&& rm -rf /var/lib/apt/lists/*
ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D
ENV PYTHON_VERSION 3.7.5

RUN set -ex \
\
&& savedAptMark="$(apt-mark showmanual)" \
&& apt-get update && apt-get install -y --no-install-recommends \
dpkg-dev \
gcc \
libbz2-dev \
libc6-dev \
libexpat1-dev \
libffi-dev \
libgdbm-dev \
liblzma-dev \
libncursesw5-dev \
libreadline-dev \
libsqlite3-dev \
libssl-dev \
make \
tk-dev \
uuid-dev \
wget \
xz-utils \
zlib1g-dev \
# as of Stretch, "gpg" is no longer included by default
$(command -v gpg > /dev/null || echo 'gnupg dirmngr') \
\
&& wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
&& wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
&& export GNUPGHOME="$(mktemp -d)" \
&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \
&& gpg --batch --verify python.tar.xz.asc python.tar.xz \
&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \
&& rm -rf "$GNUPGHOME" python.tar.xz.asc \
&& mkdir -p /usr/src/python \
&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \
&& rm python.tar.xz \
\
&& cd /usr/src/python \
&& gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \
&& ./configure \
--build="$gnuArch" \
--enable-loadable-sqlite-extensions \
--enable-optimizations \
--enable-shared \
--with-system-expat \
--with-system-ffi \
--without-ensurepip \
&& make -j "$(nproc)" \
# setting PROFILE_TASK makes "--enable-optimizations" reasonable: https://bugs.python.org/issue36044 / https://github.com/docker-library/python/issues/160#issuecomment-509426916
PROFILE_TASK='-m test.regrtest --pgo \
test_array \
test_base64 \
test_binascii \
test_binhex \
test_binop \
test_bytes \
test_c_locale_coercion \
test_class \
test_cmath \
test_codecs \
test_compile \
test_complex \
test_csv \
test_decimal \
test_dict \
test_float \
test_fstring \
test_hashlib \
test_io \
test_iter \
test_json \
test_long \
test_math \
test_memoryview \
test_pickle \
test_re \
test_set \
test_slice \
test_struct \
test_threading \
test_time \
test_traceback \
test_unicode \
' \
&& make install \
&& ldconfig \
\
&& apt-mark auto '.*' > /dev/null \
&& apt-mark manual $savedAptMark \
&& find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec ldd '{}' ';' \
| awk '/=>/ { print $(NF-1) }' \
| sort -u \
| xargs -r dpkg-query --search \
| cut -d: -f1 \
| sort -u \
| xargs -r apt-mark manual \
&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
&& rm -rf /var/lib/apt/lists/* \
\
&& find /usr/local -depth \
\( \
\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
-o \
\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
\) -exec rm -rf '{}' + \
&& rm -rf /usr/src/python \
\
&& python3 --version

# make some useful symlinks that are expected to exist
RUN cd /usr/local/bin \
&& ln -s idle3 idle \
&& ln -s pydoc3 pydoc \
&& ln -s python3 python \
&& ln -s python3-config python-config
# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
ENV PYTHON_PIP_VERSION 19.3.1
# https://github.com/pypa/get-pip
ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/ffe826207a010164265d9cc807978e3604d18ca0/get-pip.py
ENV PYTHON_GET_PIP_SHA256 b86f36cc4345ae87bfd4f10ef6b2dbfa7a872fbff70608a1e43944d283fd0eee

RUN set -ex; \
\
savedAptMark="$(apt-mark showmanual)"; \
apt-get update; \
apt-get install -y --no-install-recommends wget; \
\
wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \
echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum --check --strict -; \
\
apt-mark auto '.*' > /dev/null; \
[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; \
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
rm -rf /var/lib/apt/lists/*; \
\
python get-pip.py \
--disable-pip-version-check \
--no-cache-dir \
"pip==$PYTHON_PIP_VERSION" \
; \
pip --version; \
\
find /usr/local -depth \
\( \
\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
-o \
\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
\) -exec rm -rf '{}' +; \
rm -f get-pip.py

RUN apt update && apt install curl make git libopenblas-base -y

RUN curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python

ENV SHELL /bin/bash -l

ENV POETRY_CACHE /work/.cache/poetry

ENV PIP_CACHE_DIR /work/.cache/pip

ENV JUPYTER_RUNTIME_DIR /work/.cache/jupyter/runtime

ENV JUPYTER_CONFIG_DIR /work/.cache/jupyter/config

RUN $HOME/.poetry/bin/poetry config settings.virtualenvs.path $POETRY_CACHE

ENV PATH /root/.poetry/bin:/bin:/usr/local/bin:/usr/bin

CMD ["bash", "-l"]
16 changes: 13 additions & 3 deletions Makefile
Expand Up @@ -10,11 +10,12 @@ WARN_COLOR=\\e[33m
PORT=8889
.SILENT: ;
default: help; # default target
DOCKER_RUN = docker run --rm -v ${FOLDER}:/work -w /work --entrypoint bash -lc python-poetry:latest -c

IMAGE_NAME=python-poetry:latest
IMAGE_NAME=tabnet:latest
IMAGE_RELEASER_NAME=release-changelog:latest

DOCKER_RUN = docker run --rm -v ${FOLDER}:/work -w /work --entrypoint bash -lc ${IMAGE_NAME} -c

prepare-release: build build-releaser ## Prepare release branch with changelog for given version
./release-script/prepare-release.sh
.PHONY: prepare-release
Expand All @@ -33,11 +34,21 @@ build: ## Build docker image
docker build -t ${IMAGE_NAME} .
.PHONY: build

build-gpu: ## Build docker image
echo "Building Dockerfile"
docker build -t ${IMAGE_NAME} . -f Dockerfile_gpu
.PHONY: build-gpu

start: build ## Start docker container
echo "Starting container ${IMAGE_NAME}"
docker run --rm -it -v ${FOLDER}:/work -w /work -p ${PORT}:${PORT} -e "JUPYTER_PORT=${PORT}" ${IMAGE_NAME}
.PHONY: start

start-gpu: build-gpu ## Start docker container
echo "Starting container ${IMAGE_NAME}"
docker run --runtime nvidia --rm -it -v ${FOLDER}:/work -w /work -p ${PORT}:${PORT} -e "JUPYTER_PORT=${PORT}" ${IMAGE_NAME}
.PHONY: start-gpu

install: build ## Install dependencies
$(DOCKER_RUN) 'poetry install'
.PHONY: install
Expand All @@ -57,4 +68,3 @@ root_bash: ## Start a root bash inside the container
help: ## Display help
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
.PHONY: help

0 comments on commit 8d14406

Please sign in to comment.