-
Notifications
You must be signed in to change notification settings - Fork 362
/
install_openmpi.sh
executable file
·235 lines (225 loc) · 8.09 KB
/
install_openmpi.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
#!/bin/bash -e
# TODO: Review and if possible fix shellcheck errors.
# shellcheck disable=all
[ "${BASH_SOURCE[0]}" ] && SCRIPT_NAME="${BASH_SOURCE[0]}" || SCRIPT_NAME=$0
SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_NAME")/.." && pwd -P)"
openmpi_ver="4.1.4"
openmpi_sha256="e166dbe876e13a50c2882e11193fecbc4362e89e6e7b6deeb69bf095c0f4fc4c"
openmpi_pkg="openmpi-${openmpi_ver}.tar.gz"
source "${SCRIPT_DIR}"/common_vars.sh
source "${SCRIPT_DIR}"/tool_kit.sh
source "${SCRIPT_DIR}"/signal_trap.sh
source "${INSTALLDIR}"/toolchain.conf
source "${INSTALLDIR}"/toolchain.env
[ ${MPI_MODE} != "openmpi" ] && exit 0
[ -f "${BUILDDIR}/setup_openmpi" ] && rm "${BUILDDIR}/setup_openmpi"
OPENMPI_CFLAGS=""
OPENMPI_LDFLAGS=""
OPENMPI_LIBS=""
! [ -d "${BUILDDIR}" ] && mkdir -p "${BUILDDIR}"
cd "${BUILDDIR}"
case "${with_openmpi}" in
__INSTALL__)
echo "==================== Installing OpenMPI ===================="
pkg_install_dir="${INSTALLDIR}/openmpi-${openmpi_ver}"
install_lock_file="$pkg_install_dir/install_successful"
if verify_checksums "${install_lock_file}"; then
echo "openmpi-${openmpi_ver} is already installed, skipping it."
else
if [ -f ${openmpi_pkg} ]; then
echo "${openmpi_pkg} is found"
else
download_pkg_from_cp2k_org "${openmpi_sha256}" "${openmpi_pkg}"
fi
echo "Installing from scratch into ${pkg_install_dir}"
[ -d openmpi-${openmpi_ver} ] && rm -rf openmpi-${openmpi_ver}
tar -xzf ${openmpi_pkg}
cd openmpi-${openmpi_ver}
if [ "${OPENBLAS_ARCH}" = "x86_64" ]; then
# can have issue with older glibc libraries, in which case
# we need to add the -fgnu89-inline to CFLAGS. We can check
# the version of glibc using ldd --version, as ldd is part of
# glibc package
glibc_version=$(ldd --version | awk '/ldd/{print $NF}')
glibc_major_ver=${glibc_version%%.*}
glibc_minor_ver=${glibc_version##*.}
if [ $glibc_major_ver -lt 2 ] ||
[ $glibc_major_ver -eq 2 -a $glibc_minor_ver -lt 12 ]; then
CFLAGS="${CFLAGS} -fgnu89-inline"
fi
fi
if [ $(command -v srun) ]; then
echo "Slurm installation found. OpenMPI will be configured with --with-pmi."
EXTRA_CONFIGURE_FLAGS="--with-pmi"
else
EXTRA_CONFIGURE_FLAGS=""
fi
# We still require MPI-1.0-compatability for PTSCOTCH
./configure CFLAGS="${CFLAGS}" \
--prefix=${pkg_install_dir} \
--libdir="${pkg_install_dir}/lib" \
--enable-mpi1-compatibility \
${EXTRA_CONFIGURE_FLAGS} \
> configure.log 2>&1 || tail -n ${LOG_LINES} configure.log
make -j $(get_nprocs) > make.log 2>&1 || tail -n ${LOG_LINES} make.log
make -j $(get_nprocs) install > install.log 2>&1 || tail -n ${LOG_LINES} install.log
cd ..
write_checksums "${install_lock_file}" "${SCRIPT_DIR}/stage1/$(basename ${SCRIPT_NAME})"
fi
check_dir "${pkg_install_dir}/bin"
check_dir "${pkg_install_dir}/lib"
check_dir "${pkg_install_dir}/include"
check_install ${pkg_install_dir}/bin/mpirun "openmpi" && MPIRUN="${pkg_install_dir}/bin/mpirun" || exit 1
check_install ${pkg_install_dir}/bin/mpicc "openmpi" && MPICC="${pkg_install_dir}/bin/mpicc" || exit 1
check_install ${pkg_install_dir}/bin/mpicxx "openmpi" && MPICXX="${pkg_install_dir}/bin/mpicxx" || exit 1
check_install ${pkg_install_dir}/bin/mpif90 "openmpi" && MPIFC="${pkg_install_dir}/bin/mpif90" || exit 1
MPIF90="${MPIFC}"
MPIF77="${MPIFC}"
OPENMPI_CFLAGS="-I'${pkg_install_dir}/include'"
OPENMPI_LDFLAGS="-L'${pkg_install_dir}/lib' -Wl,-rpath,'${pkg_install_dir}/lib'"
;;
__SYSTEM__)
echo "==================== Finding OpenMPI from system paths ===================="
check_command mpirun "openmpi" && MPIRUN="$(command -v mpirun)" || exit 1
check_command mpicc "openmpi" && MPICC="$(command -v mpicc)" || exit 1
check_command mpic++ "openmpi" && MPICXX="$(command -v mpic++)" || exit 1
check_command mpif90 "openmpi" && MPIFC="$(command -v mpif90)" || exit 1
MPIF90="${MPIFC}"
MPIF77="${MPIFC}"
# Fortran code in CP2K is built via the mpifort wrapper, but we may need additional
# libraries and linker flags for C/C++-based MPI codepaths, pull them in at this point.
OPENMPI_CFLAGS="$(mpicxx --showme:compile)"
OPENMPI_LDFLAGS="$(mpicxx --showme:link)"
;;
__DONTUSE__)
# Nothing to do
;;
*)
echo "==================== Linking OpenMPI to user paths ===================="
pkg_install_dir="${with_openmpi}"
check_dir "${pkg_install_dir}/bin"
check_dir "${pkg_install_dir}/lib"
check_dir "${pkg_install_dir}/include"
check_command ${pkg_install_dir}/bin/mpirun "openmpi" && MPIRUN="${pkg_install_dir}/bin/mpirun" || exit 1
check_command ${pkg_install_dir}/bin/mpicc "openmpi" && MPICC="${pkg_install_dir}/bin/mpicc" || exit 1
check_command ${pkg_install_dir}/bin/mpic++ "openmpi" && MPICXX="${pkg_install_dir}/bin/mpic++" || exit 1
check_command ${pkg_install_dir}/bin/mpif90 "openmpi" && MPIFC="${pkg_install_dir}/bin/mpif90" || exit 1
MPIF90="${MPIFC}"
MPIF77="${MPIFC}"
OPENMPI_CFLAGS="-I'${pkg_install_dir}/include'"
OPENMPI_LDFLAGS="-L'${pkg_install_dir}/lib' -Wl,-rpath,'${pkg_install_dir}/lib'"
;;
esac
if [ "${with_openmpi}" != "__DONTUSE__" ]; then
if [ "${with_openmpi}" != "__SYSTEM__" ]; then
mpi_bin="${pkg_install_dir}/bin/mpirun"
mpicxx_bin="${pkg_install_dir}/bin/mpicxx"
else
mpi_bin="mpirun"
mpicxx_bin="mpicxx"
fi
# check openmpi version as reported by mpirun
raw_version=$(${mpi_bin} --version 2>&1 |
grep "(Open MPI)" | awk '{print $4}')
major_version=$(echo ${raw_version} | cut -d '.' -f 1)
minor_version=$(echo ${raw_version} | cut -d '.' -f 2)
OPENMPI_LIBS=""
# grab additional runtime libs (for C/C++) from the mpicxx wrapper,
# and remove them from the LDFLAGS if present
for lib in $("${mpicxx_bin}" --showme:libs); do
OPENMPI_LIBS+=" -l${lib}"
OPENMPI_LDFLAGS="${OPENMPI_LDFLAGS//-l${lib}/}"
done
cat << EOF > "${BUILDDIR}/setup_openmpi"
export MPI_MODE="${MPI_MODE}"
export MPIRUN="${MPIRUN}"
export MPICC="${MPICC}"
export MPICXX="${MPICXX}"
export MPIFC="${MPIFC}"
export MPIF90="${MPIF90}"
export MPIF77="${MPIF77}"
export OPENMPI_CFLAGS="${OPENMPI_CFLAGS}"
export OPENMPI_LDFLAGS="${OPENMPI_LDFLAGS}"
export OPENMPI_LIBS="${OPENMPI_LIBS}"
export MPI_CFLAGS="${OPENMPI_CFLAGS}"
export MPI_LDFLAGS="${OPENMPI_LDFLAGS}"
export MPI_LIBS="${OPENMPI_LIBS}"
export CP_DFLAGS="\${CP_DFLAGS} IF_MPI(-D__parallel}|)"
export CP_CFLAGS="\${CP_CFLAGS} IF_MPI(${OPENMPI_CFLAGS}|)"
export CP_LDFLAGS="\${CP_LDFLAGS} IF_MPI(${OPENMPI_LDFLAGS}|)"
export CP_LIBS="\${CP_LIBS} IF_MPI(${OPENMPI_LIBS}|)"
EOF
if [ "${with_openmpi}" != "__SYSTEM__" ]; then
cat << EOF >> "${BUILDDIR}/setup_openmpi"
prepend_path PATH "${pkg_install_dir}/bin"
prepend_path LD_LIBRARY_PATH "${pkg_install_dir}/lib"
prepend_path LD_RUN_PATH "${pkg_install_dir}/lib"
prepend_path LIBRARY_PATH "${pkg_install_dir}/lib"
prepend_path CPATH "${pkg_install_dir}/include"
EOF
fi
cat "${BUILDDIR}/setup_openmpi" >> ${SETUPFILE}
fi
# ----------------------------------------------------------------------
# Suppress reporting of known leaks
# ----------------------------------------------------------------------
cat << EOF >> ${INSTALLDIR}/valgrind.supp
{
<BuggyOpenMPI_1>
Memcheck:Leak
...
fun:*alloc
...
fun:ompi_mpi_init
}
{
<BuggyOpenMPI_2>
Memcheck:Leak
...
fun:*alloc
...
fun:ompi_mpi_finalize
}
{
<BuggyOpenMPI_3>
Memcheck:Leak
...
fun:malloc
fun:opal_free_list_grow_st
...
fun:mpi_alloc_mem
}
{
<BuggyOpenMPI_4>
Memcheck:Leak
...
fun:malloc
...
fun:progress_engine
...
fun:clone
}
{
<BuggyOpenMPI_5>
Memcheck:Leak
...
fun:malloc
...
fun:query_2_0_0
...
fun:ompi_comm_activate
}
EOF
cat << EOF >> ${INSTALLDIR}/lsan.supp
# leaks related to OpenMPI
leak:query_2_0_0
leak:ompi_init_f
leak:ompi_finalize_f
leak:ompi_file_open_f
leak:progress_engine
leak:__GI___strdup
EOF
load "${BUILDDIR}/setup_openmpi"
write_toolchain_env "${INSTALLDIR}"
cd "${ROOTDIR}"
report_timing "openmpi"