-
Notifications
You must be signed in to change notification settings - Fork 452
/
Dockerfile.gpu
224 lines (193 loc) · 8.03 KB
/
Dockerfile.gpu
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
FROM nvidia/cuda:10.2-base-ubuntu18.04
LABEL maintainer="Amazon AI"
LABEL dlc_major_version="2"
# prevent stopping by user interaction
ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN true
ENV SAGEMAKER_TRAINING_MODULE sagemaker_tensorflow_container.training:main
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
ENV PYTHONIOENCODING=UTF-8
ENV LANG=C.UTF-8
ENV LC_ALL=C.UTF-8
# Set environment variables for MKL
# For more about MKL with TensorFlow see:
# https://www.tensorflow.org/performance/performance_guide#tensorflow_with_intel%C2%AE_mkl_dnn
ENV KMP_AFFINITY=granularity=fine,compact,1,0
ENV KMP_BLOCKTIME=1
ENV KMP_SETTINGS=0
ARG PYTHON=python3.7
ARG PYTHON_PIP=python3-pip
ARG PIP=pip3
ARG PYTHON_VERSION=3.7.7
ARG OPENSSL_VERSION=1.1.1g
ARG TF_URL=https://aws-tensorflow-binaries.s3-us-west-2.amazonaws.com/tensorflow/r2.3_aws/20201002-134350/gpu/py37/tensorflow_gpu-2.3.1-cp37-cp37m-manylinux2010_x86_64.whl
ARG ESTIMATOR_URL=https://aws-tensorflow-binaries.s3-us-west-2.amazonaws.com/estimator/r2.3_aws/20201002-134350/tensorflow_estimator-2.3.0-py2.py3-none-any.whl
# The smdebug pipeline relies for following format to perform string replace and trigger DLC pipeline for validating
# the nightly builds. Therefore, while updating the smdebug version, please ensure that the format is not disturbed.
ARG SMDEBUG_VERSION=0.9.4
RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthenticated \
ca-certificates \
cuda-command-line-tools-10-2 \
cuda-cudart-dev-10-2 \
cuda-cufft-dev-10-2 \
cuda-curand-dev-10-2 \
cuda-cusolver-dev-10-2 \
cuda-cusparse-dev-10-2 \
curl \
emacs \
libcudnn7=7.6.5.32-1+cuda10.2 \
# TensorFlow doesn't require libnccl anymore but Open MPI still depends on it
libnccl2=2.7.6-1+cuda10.2 \
libgomp1 \
libnccl-dev=2.7.6-1+cuda10.2 \
libfreetype6-dev \
libhdf5-serial-dev \
liblzma-dev \
libpng-dev \
libtemplate-perl \
libzmq3-dev \
git \
wget \
vim \
build-essential \
openssh-client \
openssh-server \
zlib1g-dev \
# Install dependent library for OpenCV
libgtk2.0-dev \
#cuda-cublas-dev not available with 10-1, install libcublas instead
#it will downgrade the cublas from 10-2 to 10-1
#adding an extra flag --allow-downgrades for it
&& apt-get update \
&& apt-get install -y --no-install-recommends --allow-unauthenticated --allow-downgrades \
libcublas10=10.2.2.89-1 \
libcublas-dev=10.2.2.89-1 \
# The 'apt-get install' of nvinfer-runtime-trt-repo-ubuntu1804-5.0.2-ga-cuda10.0
# adds a new list which contains libnvinfer library, so it needs another
# 'apt-get update' to retrieve that list before it can actually install the
# library.
# We don't install libnvinfer-dev since we don't need to build against TensorRT,
# and libnvinfer4 doesn't contain libnvinfer.a static library.
# nvinfer-runtime-trt-repo doesn't have a 1804-cuda10.1 version yet. see:
# https://developer.download.nvidia.cn/compute/machine-learning/repos/ubuntu1804/x86_64/
&& apt-get update && apt-get install -y --no-install-recommends --allow-unauthenticated \
nvinfer-runtime-trt-repo-ubuntu1804-5.0.2-ga-cuda10.0 \
&& apt-get update && apt-get install -y --no-install-recommends --allow-unauthenticated \
libnvinfer6=6.0.1-1+cuda10.2 \
&& rm -rf /var/lib/apt/lists/* \
&& mkdir -p /var/run/sshd
###########################################################################
# Horovod & its dependencies
###########################################################################
# Install Open MPI
RUN mkdir /tmp/openmpi \
&& cd /tmp/openmpi \
&& curl -fSsL -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.4.tar.gz \
&& tar zxf openmpi-4.0.4.tar.gz \
&& cd openmpi-4.0.4 \
&& ./configure --enable-orterun-prefix-by-default \
&& make -j $(nproc) all \
&& make install \
&& ldconfig \
&& rm -rf /tmp/openmpi
# Create a wrapper for OpenMPI to allow running as root by default
RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real \
&& echo '#!/bin/bash' > /usr/local/bin/mpirun \
&& echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun \
&& chmod a+x /usr/local/bin/mpirun
# Configure OpenMPI to run good defaults:
# --bind-to none --map-by slot --mca btl_tcp_if_exclude lo,docker0
RUN echo "hwloc_base_binding_policy = none" >> /usr/local/etc/openmpi-mca-params.conf \
&& echo "rmaps_base_mapping_policy = slot" >> /usr/local/etc/openmpi-mca-params.conf
# Set default NCCL parameters
RUN echo NCCL_DEBUG=INFO >> /etc/nccl.conf
ENV LD_LIBRARY_PATH=/usr/local/openmpi/lib:$LD_LIBRARY_PATH
ENV PATH /usr/local/openmpi/bin/:$PATH
ENV PATH=/usr/local/nvidia/bin:$PATH
# SSH login fix. Otherwise user is kicked off after login
RUN mkdir -p /var/run/sshd \
&& sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
# Create SSH key.
RUN mkdir -p /root/.ssh/ \
&& ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa \
&& cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \
&& printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config
WORKDIR /
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
libbz2-dev \
libc6-dev \
libffi-dev \
libgdbm-dev \
libncursesw5-dev \
libreadline-gplv2-dev \
libsqlite3-dev \
libssl-dev \
tk-dev \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get clean
RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz \
&& tar -xvf Python-$PYTHON_VERSION.tgz \
&& cd Python-$PYTHON_VERSION \
&& ./configure && make && make install \
&& make && make install && rm -rf ../Python-$PYTHON_VERSION*
RUN ${PIP} --no-cache-dir install --upgrade \
pip \
setuptools
RUN wget -c https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \
&& apt-get update \
&& apt remove -y --purge openssl \
&& rm -rf /usr/include/openssl \
&& apt-get install -y \
ca-certificates \
&& tar -xzvf openssl-${OPENSSL_VERSION}.tar.gz \
&& cd openssl-${OPENSSL_VERSION} \
&& ./config && make && make test \
&& make install \
&& ldconfig \
&& cd .. && rm -rf openssl-*
# when we remove previous openssl, the ca-certificates pkgs and its symlinks gets deleted
# causing sslcertverificationerror the below steps are to fix that
RUN ln -s /etc/ssl/certs/*.* /usr/local/ssl/certs/
# Some TF tools expect a "python" binary
RUN ln -s $(which ${PYTHON}) /usr/local/bin/python \
&& ln -s $(which ${PIP}) /usr/bin/pip
# install PyYAML==5.1.2 to avoid conflict with latest awscli
# # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli
RUN ${PIP} install --no-cache-dir -U \
numpy==1.19.1 \
scipy==1.5.2 \
scikit-learn==0.23 \
pandas==1.1 \
Pillow==7.2.0 \
h5py==2.10.0 \
python-dateutil==2.8.1 \
pyYAML==5.3.1 \
requests==2.24.0 \
"awscli<2" \
mpi4py==3.0.3 \
opencv-python==4.3.0.36 \
"sagemaker<2" \
sagemaker-experiments==0.* \
"sagemaker-tensorflow>=2.3,<2.4" \
"sagemaker-tensorflow-training>=20" \
# Let's install TensorFlow separately in the end to avoid
# the library version to be overwritten
&& ${PIP} install --no-cache-dir -U \
${TF_URL} \
${ESTIMATOR_URL} \
werkzeug==1.0.1 \
smdebug==${SMDEBUG_VERSION}
# Install Horovod, temporarily using CUDA stubs
RUN ldconfig /usr/local/cuda-10.1/targets/x86_64-linux/lib/stubs \
&& HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_WITH_TENSORFLOW=1 ${PIP} install --no-cache-dir horovod==0.19.5 \
&& ldconfig
# Allow OpenSSH to talk to containers without asking for confirmation
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new \
&& echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \
&& mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
ADD https://raw.githubusercontent.com/aws/deep-learning-containers/master/src/deep_learning_container.py /usr/local/bin/deep_learning_container.py
RUN chmod +x /usr/local/bin/deep_learning_container.py
RUN curl https://aws-dlc-licenses.s3.amazonaws.com/tensorflow-2.3/license.txt -o /license.txt
CMD ["bin/bash"]