From 5eccba17affa7e0e80e0adc3b637b84ef3aa5bf0 Mon Sep 17 00:00:00 2001 From: Howard Pritchard Date: Wed, 22 Jul 2015 12:55:02 -0600 Subject: [PATCH] lanl: help out lanl admins LANL admins want platform files and *.conf files so oblige them. Signed-off-by: Howard Pritchard --- .../platform/lanl/cray_xc_cle5.2/debug-common | 39 +++++++ .../platform/lanl/cray_xc_cle5.2/debug-lustre | 19 ++++ .../lanl/cray_xc_cle5.2/debug-lustre.conf | 104 +++++++++++++++++ .../lanl/cray_xc_cle5.2/optimized-common | 47 ++++++++ .../lanl/cray_xc_cle5.2/optimized-lustre | 17 +++ .../lanl/cray_xc_cle5.2/optimized-lustre.conf | 107 ++++++++++++++++++ 6 files changed, 333 insertions(+) create mode 100644 contrib/platform/lanl/cray_xc_cle5.2/debug-common create mode 100644 contrib/platform/lanl/cray_xc_cle5.2/debug-lustre create mode 100644 contrib/platform/lanl/cray_xc_cle5.2/debug-lustre.conf create mode 100644 contrib/platform/lanl/cray_xc_cle5.2/optimized-common create mode 100644 contrib/platform/lanl/cray_xc_cle5.2/optimized-lustre create mode 100644 contrib/platform/lanl/cray_xc_cle5.2/optimized-lustre.conf diff --git a/contrib/platform/lanl/cray_xc_cle5.2/debug-common b/contrib/platform/lanl/cray_xc_cle5.2/debug-common new file mode 100644 index 00000000000..d75d45a1f93 --- /dev/null +++ b/contrib/platform/lanl/cray_xc_cle5.2/debug-common @@ -0,0 +1,39 @@ +# do not use IB verbs +with_verbs=no + +enable_dlopen=no +enable_mem_profile=no +enable_binaries=yes +enable_heterogeneous=no +enable_shared=yes +enable_static=yes +enable_orte_static_ports=no +enable_ipv6=no +enable_mpi_fortran=yes + +# enable debugging +enable_mem_debug=yes +enable_debug_symbols=yes +enable_picky=yes +enable_debug=yes + +# disable c++ bindings (removed in MPI 3.0) +enable_mpi_cxx=no +enable_mpi_cxx_seek=no +enable_cxx_exceptions=no + +enable_per_user_config_files=no +enable_pty_support=no +enable_mca_no_build=crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,pml-cm,ess-cnos,grpcomm-cnos,plm-rsh,btl-tcp,oob-ud,ras-simulator + +# enable direct calling for ob1 +enable_mca_direct=pml-ob1 + +# enable development headers +with_devel_headers=yes + +# enable ptmalloc (enables lazy deregistration) +with_memory_manager=linux + +# disable valgrind +with_valgrind=no diff --git a/contrib/platform/lanl/cray_xc_cle5.2/debug-lustre b/contrib/platform/lanl/cray_xc_cle5.2/debug-lustre new file mode 100644 index 00000000000..2bedb1b7a33 --- /dev/null +++ b/contrib/platform/lanl/cray_xc_cle5.2/debug-lustre @@ -0,0 +1,19 @@ +# (c) 2012-2015 Los Alamos National Security, LLC. All rights reserved. +# Open MPI configuration for Cray XC v2.x GNU compiler, +# Lustre + +if test "$CC" = "cc" ; then + echo "ERROR: Open MPI should not be compiled with Cray's wrapper compilers (cc/CC/ftn)" + exit 1 +fi + +source debug-common + +if test "$CC" = "icc" ; then + source intel-common +else + source gnu-common +fi + +# enable and Lustre in romio +with_io_romio_flags="--with-file-system=ufs+nfs+lustre CFLAGS=-I/opt/cray/lustre-cray_gem_s/default/include" diff --git a/contrib/platform/lanl/cray_xc_cle5.2/debug-lustre.conf b/contrib/platform/lanl/cray_xc_cle5.2/debug-lustre.conf new file mode 100644 index 00000000000..657fc25a489 --- /dev/null +++ b/contrib/platform/lanl/cray_xc_cle5.2/debug-lustre.conf @@ -0,0 +1,104 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2015 Los Alamos National Security, LLC. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This is the default system-wide MCA parameters defaults file. +# Specifically, the MCA parameter "mca_param_files" defaults to a +# value of +# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" +# (this file is the latter of the two). So if the default value of +# mca_param_files is not changed, this file is used to set system-wide +# MCA parameters. This file can therefore be used to set system-wide +# default MCA parameters for all users. Of course, users can override +# these values if they want, but this file is an excellent location +# for setting system-specific MCA parameters for those users who don't +# know / care enough to investigate the proper values for them. + +# Note that this file is only applicable where it is visible (in a +# filesystem sense). Specifically, MPI processes each read this file +# during their startup to determine what default values for MCA +# parameters should be used. mpirun does not bundle up the values in +# this file from the node where it was run and send them to all nodes; +# the default value decisions are effectively distributed. Hence, +# these values are only applicable on nodes that "see" this file. If +# $sysconf is a directory on a local disk, it is likely that changes +# to this file will need to be propagated to other nodes. If $sysconf +# is a directory that is shared via a networked filesystem, changes to +# this file will be visible to all nodes that share this $sysconf. + +# The format is straightforward: one per line, mca_param_name = +# rvalue. Quoting is ignored (so if you use quotes or escape +# characters, they'll be included as part of the value). For example: + +# Disable run-time MPI parameter checking +# mpi_param_check = 0 + +# Note that the value "~/" will be expanded to the current user's home +# directory. For example: + +# Change component loading path +# component_path = /usr/local/lib/openmpi:~/my_openmpi_components + +# See "ompi_info --param all all" for a full listing of Open MPI MCA +# parameters available and their default values. +# + +# Basic behavior to smooth startup +mca_base_component_show_load_errors = 0 +orte_report_launch_progress = 1 + +# Set line buffering for stdout/stderr +ess_base_stream_buffering = 1 + +# Define timeout for daemons to report back during launch +orte_startup_timeout = 360 + +## Protect the shared file systems +orte_no_session_dirs = /lustre,/panfs,/scratch,/lscratch,/users,/usr/projects +orte_tmpdir_base = /var/tmp + +## Require an allocation to run - protects the frontend +## from inadvertent job executions +orte_allocation_required = 1 + +## Deal with the allocator +orte_strip_prefix = nid +orte_retain_aliases = 1 +# 1st alias entry is the stripped node name, +# 2nd is the unstripped one +orte_hostname_alias_index = 2 + +## Add the interface for out-of-band communication +## and set it up +oob_tcp_if_include=ipogif0 +oob_tcp_peer_retries = 1000 +oob_tcp_sndbuf = 32768 +oob_tcp_rcvbuf = 32768 + +## Define the MPI interconnects +btl = self,vader,ugni + +## Enable cpu affinity +hwloc_base_binding_policy = core + +## Setup MPI options +mpi_show_handle_leaks = 1 +mpi_warn_on_fork = 1 +#mpi_abort_print_stack = 1 diff --git a/contrib/platform/lanl/cray_xc_cle5.2/optimized-common b/contrib/platform/lanl/cray_xc_cle5.2/optimized-common new file mode 100644 index 00000000000..04e556ff84e --- /dev/null +++ b/contrib/platform/lanl/cray_xc_cle5.2/optimized-common @@ -0,0 +1,47 @@ +# do not use IB verbs +with_verbs=no + +enable_dlopen=no + +enable_mem_profile=no +enable_binaries=yes + +enable_heterogeneous=no +enable_shared=yes +enable_static=yes + +enable_orte_static_ports=no + +enable_ipv6=no + +# enable fortran bindings +enable_mpi_fortran=yes + +# disable debugging +enable_mem_debug=no +enable_debug_symbols=no +enable_picky=no +enable_debug=no + +# disable c++ bindings (removed in MPI 3.0) +enable_mpi_cxx=no +enable_mpi_cxx_seek=no +enable_cxx_exceptions=no + +enable_per_user_config_files=no +enable_pty_support=no +enable_mca_no_build=crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,pml-cm,ess-cnos,grpcomm-cnos,plm-rsh,btl-tcp,oob-ud,ras-simulator,mpool-fake +enable_mca_static=btl:ugni,btl:self,btl:vader,pml:ob1 +#enable_contrib_no_build=libnbc + +# enable direct calling for ob1 +enable_mca_direct=pml-ob1 + +# enable development headers +with_devel_headers=yes + +# enable ptmalloc (enables lazy deregistration) +with_memory_manager=linux + +# disable valgrind +with_valgrind=no diff --git a/contrib/platform/lanl/cray_xc_cle5.2/optimized-lustre b/contrib/platform/lanl/cray_xc_cle5.2/optimized-lustre new file mode 100644 index 00000000000..8c2d47ab59e --- /dev/null +++ b/contrib/platform/lanl/cray_xc_cle5.2/optimized-lustre @@ -0,0 +1,17 @@ +# (c) 2012-2015 Los Alamos National Security, LLC. All rights reserved. +# Open MPI configuration for Cray XC v2.x GNU compiler, +# Lustre + +if test "$CC" = "cc" ; then + echo "ERROR: Open MPI should not be compiled with Cray's wrapper compilers (cc/CC/ftn)" + exit 1 +fi + +if test "$enable_debug" = "yes"; then + source debug-common +else + source optimized-common +fi + +# enable and Lustre in romio +with_io_romio_flags="--with-file-system=ufs+nfs+lustre CFLAGS=-I/opt/cray/lustre-cray_gem_s/default/include" diff --git a/contrib/platform/lanl/cray_xc_cle5.2/optimized-lustre.conf b/contrib/platform/lanl/cray_xc_cle5.2/optimized-lustre.conf new file mode 100644 index 00000000000..d545d6891ef --- /dev/null +++ b/contrib/platform/lanl/cray_xc_cle5.2/optimized-lustre.conf @@ -0,0 +1,107 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2015 Los Alamos National Security, LLC. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This is the default system-wide MCA parameters defaults file. +# Specifically, the MCA parameter "mca_param_files" defaults to a +# value of +# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" +# (this file is the latter of the two). So if the default value of +# mca_param_files is not changed, this file is used to set system-wide +# MCA parameters. This file can therefore be used to set system-wide +# default MCA parameters for all users. Of course, users can override +# these values if they want, but this file is an excellent location +# for setting system-specific MCA parameters for those users who don't +# know / care enough to investigate the proper values for them. + +# Note that this file is only applicable where it is visible (in a +# filesystem sense). Specifically, MPI processes each read this file +# during their startup to determine what default values for MCA +# parameters should be used. mpirun does not bundle up the values in +# this file from the node where it was run and send them to all nodes; +# the default value decisions are effectively distributed. Hence, +# these values are only applicable on nodes that "see" this file. If +# $sysconf is a directory on a local disk, it is likely that changes +# to this file will need to be propagated to other nodes. If $sysconf +# is a directory that is shared via a networked filesystem, changes to +# this file will be visible to all nodes that share this $sysconf. + +# The format is straightforward: one per line, mca_param_name = +# rvalue. Quoting is ignored (so if you use quotes or escape +# characters, they'll be included as part of the value). For example: + +# Disable run-time MPI parameter checking +# mpi_param_check = 0 + +# Note that the value "~/" will be expanded to the current user's home +# directory. For example: + +# Change component loading path +# component_path = /usr/local/lib/openmpi:~/my_openmpi_components + +# See "ompi_info --param all all" for a full listing of Open MPI MCA +# parameters available and their default values. +# + +# Basic behavior to smooth startup +mca_base_component_show_load_errors = 0 +#orte_report_launch_progress = 1 + +# Set line buffering for stdout/stderr +ess_base_stream_buffering = 1 + +# Define timeout for daemons to report back during launch +orte_startup_timeout = 360 + +## Protect the shared file systems +orte_no_session_dirs = /lustre,/panfs,/scratch,/lscratch,/users,/usr/projects +orte_tmpdir_base = /var/tmp + +## Require an allocation to run - protects the frontend +## from inadvertent job executions +orte_allocation_required = 1 + +## Deal with the allocator +orte_strip_prefix = nid +orte_retain_aliases = 1 +# 1st alias entry is the stripped node name, +# 2nd is the unstripped one +orte_hostname_alias_index = 2 + +## Add the interface for out-of-band communication +## and set it up +oob_tcp_if_include=ipogif0 +oob_tcp_peer_retries = 1000 +oob_tcp_sndbuf = 32768 +oob_tcp_rcvbuf = 32768 + +## Define the MPI interconnects +btl = self,vader,ugni + +## Setup Gemini +# TODO LANL + +## Enable cpu affinity +hwloc_base_binding_policy = core + +## Setup MPI options +mpi_show_handle_leaks = 1 +mpi_warn_on_fork = 1 +#mpi_abort_print_stack = 1