/
Dockerfile.amd
159 lines (134 loc) · 4.75 KB
/
Dockerfile.amd
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
##############################################################
# This Dockerfile contains AMD compilers
# Based on https://github.com/amd/InfinityHub-CI/tree/main/base-gpu-mpi-rocm-docker
##############################################################
ARG ROCM_VERSION=5.5.1
FROM rocm/dev-ubuntu-22.04:${ROCM_VERSION}-complete as sdk-base
ARG UCX_BRANCH="v1.13.1"
ARG OMPI_BRANCH="v4.1.4"
# Update and Install basic Linux development tools
RUN rm /etc/apt/sources.list.d/* \
&& apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
ca-certificates \
git \
ssh \
make \
vim \
nano \
libtinfo* \
initramfs-tools \
libelf-dev \
numactl \
wget \
tmux \
build-essential \
autoconf \
automake \
libtool \
pkg-config \
libnuma* \
gfortran \
flex \
hwloc \
&& apt-get clean
# Requires cmake > 3.22
RUN mkdir -p /opt/cmake \
&& wget --no-check-certificate --quiet -O - https://cmake.org/files/v3.22/cmake-3.22.2-linux-x86_64.tar.gz | tar --strip-components=1 -xz -C /opt/cmake
ENV ROCM_HOME=/opt/rocm \
UCX_HOME=/opt/ucx \
OMPI_HOME=/opt/ompi
# Until rocm base has it fixed
RUN ln -s /opt/rocm/llvm/bin/offload-arch /opt/rocm/bin/offload-arch
# Install tmpi
RUN curl https://raw.githubusercontent.com/Azrael3000/tmpi/master/tmpi -o /usr/local/bin/tmpi
# Adding rocm/cmake to the Environment
ENV PATH=$ROCM_HOME/bin:$ROCM_HOME/profiler/bin:$ROCM_HOME/opencl/bin:/opt/cmake/bin:$PATH \
LD_LIBRARY_PATH=$ROCM_HOME/lib:$ROCM_HOME/lib64:$ROCM_HOME/llvm/lib:$LD_LIBRARY_PATH \
C_INCLUDE_PATH=$ROCM_HOME/include:$C_INCLUDE_PATH \
CPLUS_INCLUDE_PATH=$ROCM_HOME/include:$CPLUS_INCLUDE_PATH \
CPATH=$ROCM_HOME/include:$CPATH \
INCLUDE=$ROCM_HOME/include:$INCLUDE
WORKDIR /tmp
# Install UCX
RUN cd /tmp/ \
&& git clone --depth 1 https://github.com/openucx/ucx.git -b ${UCX_BRANCH} \
&& cd ucx \
&& ./autogen.sh \
&& mkdir build \
&& cd build \
&& ../contrib/configure-release --prefix=$UCX_HOME \
--with-rocm=$ROCM_HOME \
--without-knem \
--without-xpmem \
--without-cuda \
--enable-optimizations \
--disable-logging \
--disable-debug \
--disable-examples \
&& make -j ${nproc} \
&& make install
# Install OpenMPI
RUN cd /tmp \
&& git clone --depth 1 --recursive https://github.com/open-mpi/ompi.git -b ${OMPI_BRANCH} \
&& cd ompi \
&& ./autogen.pl \
&& mkdir build \
&& cd build \
&& ../configure --prefix=$OMPI_HOME --with-ucx=$UCX_HOME \
CC=amdclang CXX=amdclang++ FC=amdflang F90=amdflang \
--enable-mca-no-build=btl-uct \
--without-verbs \
--with-pmix \
--enable-mpi \
--enable-mpi-fortran=yes \
--disable-debug \
&& make -j ${nproc} \
&& make install
# Cleanup
RUN rm -rf /tmp/ucx && rm -rf /tmp/ompi
# Adding OpenMPI and UCX to Environment
ENV PATH=$OMPI_HOME/bin:$UCX_HOME/bin:$PATH \
LD_LIBRARY_PATH=$OMPI_HOME/lib:$UCX_HOME/lib:$LD_LIBRARY_PATH \
C_INCLUDE_PATH=$OMPI_HOME/include:$UCX_HOME/include:$C_INCLUDE_PATH \
CPLUS_INCLUDE_PATH=$OMPI_HOME/include:$UCX_HOME/include:$CPLUS_INCLUDE_PATH \
CPATH=$OMPI_HOME/include:$UCX_HOME/include:$CPATH \
INCLUDE=$OMPI_HOME/include:$UCX_HOME/include:$INCLUDE \
PKG_CONFIG_PATH=$OMPI_HOME/lib/pkgconfig:$UCX_HOME/lib/pkgconfig:$PKG_CONFIG_PATH
# Adding environment variable for Running as ROOT
ENV OMPI_ALLOW_RUN_AS_ROOT=1
ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
##############################################################
# Additional Devito requirements
##############################################################
# Additional utils and python3
RUN apt-get update && \
apt-get install -y dh-autoreconf python3-venv python3-dev python3-pip
RUN apt-get clean && apt-get autoclean && apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/*
EXPOSE 8888
CMD ["/bin/bash"]
########################################################################
# AOMP for GPUs (OpenMP offloading)
########################################################################
# This will only trigger if arch is aomp since the final stage depends on it
FROM sdk-base as amdclang
# MPI env
ENV OMPI_CC="amdclang"
ENV OMPI_CXX="amdclang++"
# Devito env
ENV DEVITO_ARCH="aomp"
ENV DEVITO_PLATFORM="amdgpuX"
ENV DEVITO_LANGUAGE="openmp"
########################################################################
# HIPCC for GPUs (HIP)
########################################################################
# This will only trigger if arch is hip since the final stage depends on it
FROM sdk-base as hip
# MPI env
ENV OMPI_CC="hipcc"
ENV OMPI_CXX="hipcc"
# Devito env
ENV DEVITO_ARCH="hip"
ENV DEVITO_PLATFORM="amdgpuX"
ENV DEVITO_LANGUAGE="hip"