-
Notifications
You must be signed in to change notification settings - Fork 222
/
Dockerfile.nvidia
156 lines (136 loc) · 5.86 KB
/
Dockerfile.nvidia
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
##############################################################
# This Dockerfile contains the additional NVIDIA compilers,
# libraries, and plugins to enable OpenACC and NVIDIA GPU
# acceleration of Devito codes.
#
# BUILD:
# docker build --network=host --file docker/Dockerfile.nvidia --tag devito:nvidia .
#
# EXPERIMENTAL:
# Enabling and using MPI 4.0.5 works on R450 drivers, but is showing compatibility
# issues during testing on older R418 drivers.
#
# docker build --network=host --build-arg MPI_VER=4 --file docker/Dockerfile.nvidia --tag devito:nvidia .
#
# RUN:
# docker run --gpus all --rm -it -p 8888:8888 -p 8787:8787 -p 8786:8786 devito:nvidia
##############################################################
FROM python:3.6
ENV DEBIAN_FRONTEND noninteractive
# nodesource: nvdashboard requires nodejs>=10
RUN apt-get update -y && \
apt-get install -y -q \
apt-utils \
vim \
curl \
mpich libmpich-dev && \
curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
apt-get install -y -q \
nodejs \
liblapack-dev \
libblas-dev \
libibverbs-dev \
texlive-latex-extra texlive-fonts-recommended dvipng cm-super && \
wget -q -P /app/ \
https://developer.download.nvidia.com/hpc-sdk/21.2/nvhpc-21-2_21.2_amd64.deb \
https://developer.download.nvidia.com/hpc-sdk/21.2/nvhpc-21-2-cuda-multi_21.2_amd64.deb \
https://developer.download.nvidia.com/hpc-sdk/21.2/nvhpc-2021_21.2_amd64.deb && \
apt-get install -y -q \
/app/nvhpc-21-2_21.2_amd64.deb \
/app/nvhpc-21-2-cuda-multi_21.2_amd64.deb \
/app/nvhpc-2021_21.2_amd64.deb && \
apt-get update -y && \
rm -rf /app/nvhpc* && \
rm -rf /var/lib/apt/lists/*
ARG HPCSDK_HOME=/opt/nvidia/hpc_sdk/Linux_x86_64/2021
ARG HPCSDK_CUPTI=/opt/nvidia/hpc_sdk/Linux_x86_64/2021/cuda/11.2/extras/CUPTI
ARG MPI_VER=3
# nvidia-container-runtime
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
# required for nvidia-docker v1
RUN echo "$HPCSDK_HOME/cuda/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "$HPCSDK_HOME/cuda/lib64" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "$HPCSDK_HOME/compilers/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "$HPCSDK_HOME/comm_libs/mpi/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "$HPCSDK_CUPTI/lib64" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "$HPCSDK_HOME/math_libs/lib64" >> /etc/ld.so.conf.d/nvidia.conf
# Compiler, CUDA, and Library paths
ENV CUDA_HOME $HPCSDK_HOME/cuda
ENV CUDA_ROOT $HPCSDK_HOME/cuda/bin
ENV PATH $HPCSDK_HOME/compilers/bin:$HPCSDK_HOME/cuda/bin:$HPCSDK_HOME/comm_libs/mpi/bin:${PATH}
ENV LD_LIBRARY_PATH $HPCSDK_HOME/cuda/lib:$HPCSDK_HOME/cuda/lib64:$HPCSDK_HOME/compilers/lib:$HPCSDK_HOME/math_libs/lib64:$HPCSDK_HOME/comm_libs/mpi/lib:$HPCSDK_CUPTI/lib64:${LD_LIBRARY_PATH}
ADD ./requirements.txt /app/requirements.txt
ADD ./requirements-optional.txt /app/requirements-optional.txt
ADD ./requirements-nvidia.txt /app/requirements-nvidia.txt
ADD ./requirements-mpi.txt /app/requirements-mpi.txt
RUN python3 -m venv /venv && \
/venv/bin/pip install --no-cache-dir --upgrade pip && \
/venv/bin/pip install --no-cache-dir wheel && \
/venv/bin/pip install --no-cache-dir -r /app/requirements.txt && \
/venv/bin/pip install --no-cache-dir -r /app/requirements-optional.txt && \
/venv/bin/pip install --no-cache-dir -r /app/requirements-nvidia.txt && \
rm -rf ~/.cache/pip
# MPI ROOT USER DEFAULTS
ENV OMPI_ALLOW_RUN_AS_ROOT=1
ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
ENV UCX_MEMTYPE_CACHE=no
ENV UCX_TLS=self,shm,cuda_copy
ENV CPATH $HPCSDK_HOME/comm_libs/mpi/include:${CPATH}
ENV CFLAGS=-noswitcherror
RUN if [ "x$MPI_VER" = "x4" ]; then \
rm -f $HPCSDK_HOME/comm_libs/mpi && \
ln -sf $HPCSDK_HOME/comm_libs/openmpi4/openmpi-4.0.5 \
$HPCSDK_HOME/comm_libs/mpi ; \
fi; \
/venv/bin/pip install --no-cache-dir -r /app/requirements-mpi.txt && \
rm -rf ~/.cache/pip
ENV CFLAGS=
ADD ./devito /app/devito
ADD ./tests /app/tests
ADD ./scripts /app/scripts
ADD ./examples /app/examples
ADD ./benchmarks /app/benchmarks
COPY setup.cfg /app/
ADD docker/run-jupyterlab.sh /jupyter
ADD docker/run-tests.sh /tests
ADD docker/run-print-defaults.sh /print-defaults
ADD docker/entrypoint.sh /docker-entrypoint.sh
ADD docker/nvdashboard.json /app/nvdashboard.json
RUN chmod +x /print-defaults /jupyter /tests /docker-entrypoint.sh && \
/venv/bin/jupyter labextension install jupyterlab-nvdashboard && \
/venv/bin/jupyter labextension install dask-labextension && \
/venv/bin/jupyter serverextension enable dask_labextension && \
/venv/bin/jupyter lab workspaces import /app/nvdashboard.json
## Environment Variables for OpenACC Builds
# Reference: https://github.com/devitocodes/devito/wiki/FAQ#can-i-manually-modify-the-c-code-generated-by-devito-and-test-these-modifications
# Set arch to PGI (pgcc)
ENV DEVITO_ARCH="pgcc"
ENV DEVITO_LANGUAGE="openacc"
ENV DEVITO_PLATFORM=nvidiaX
# Options: [unset, 1] For PGI openacc; Should only be set after a first execution of the benchmark
# ENV DEVITO_JIT_BACKDOOR=1
# Enable logging, Options: [unset, PERF, DEBUG]
ENV DEVITO_LOGGING=DEBUG
#ENV PGI_ACC_TIME=1
## Create App user
# Set the home directory to our app user's home.
ENV HOME=/app
ENV APP_HOME=/app
# Create the home directory for the new app user.
# Create an app user so our program doesn't run as root.
# Chown all the files to the app user.
RUN mkdir -p /app && \
groupadd -r app && \
useradd -r -g app -d /app -s /sbin/nologin -c "Docker image user" app && \
chown -R app:app $APP_HOME && \
chown app:app /docker-entrypoint.sh && \
chown app:app /print-defaults && \
chown app:app /tests && \
chown app:app /jupyter
# Change to the app user.
USER app
WORKDIR /app
EXPOSE 8888
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD ["/jupyter"]