Merge pull request ESMCI#3441 from jedwards4b/thunder_port

Thunder port
ekluzek · Mar 13, 2020 · 38c2b9f · 38c2b9f
2 parents 5d9ff80 + 6c85f73
commit 38c2b9f
Show file tree

Hide file tree

Showing 7 changed files with 171 additions and 9 deletions.
diff --git a/config/cesm/machines/config_batch.xml b/config/cesm/machines/config_batch.xml
@@ -602,4 +602,19 @@
       <queue walltimemax="00:60:00" nodemin="1" nodemax="50" default="true">default</queue>
     </queues>
   </batch_system>
+  <batch_system MACH="thunder" type="slurm">
+    <batch_submit>sbatch</batch_submit>
+    <submit_args>
+      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
+      <arg flag="-q" name="$JOB_QUEUE"/>
+    </submit_args>
+    <directives>
+      <directive>--ntasks-per-node=64</directive>
+      <directive>--hint=nomultithread</directive>
+    </directives>
+    <queues>
+      <queue walltimemax="02:00:00" nodemin="1" nodemax="4" default="true">regular</queue>
+    </queues>
+  </batch_system>      
+
 </config_batch>
diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml
@@ -85,6 +85,73 @@ using a fortran linker.
   <SUPPORTS_CXX>FALSE</SUPPORTS_CXX>
 </compiler>
 
+<compiler COMPILER="arm">
+  <!-- LLVM compiler support is a work in progess -->
+  <MPICC> mpicc  </MPICC>
+  <MPICXX> mpicxx </MPICXX>
+  <MPIFC> mpif90 </MPIFC>
+  <SCC> clang </SCC>
+  <SCXX> clang++ </SCXX>
+  <SFC> flang </SFC>
+  <CPPDEFS>
+    <append> -DCPRLLVM</append>
+    <append MODEL="gptl"> -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY  </append>
+  </CPPDEFS>
+</compiler>
+
+<compiler COMPILER="armgcc">
+  <CFLAGS>
+    <base> -std=gnu99 </base>
+    <append compile_threaded="TRUE"> -fopenmp </append>
+    <append DEBUG="TRUE"> -g -Wall -Og -fbacktrace -ffpe-trap=invalid,zero,overflow -fcheck=bounds </append>
+    <append DEBUG="FALSE"> -O </append>
+  </CFLAGS>
+  <CPPDEFS>
+    <!-- http://gcc.gnu.org/onlinedocs/gfortran/ -->
+    <append> -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU</append>
+  </CPPDEFS>
+  <CXX_LINKER>FORTRAN</CXX_LINKER>
+  <FC_AUTO_R8>
+    <base> -fdefault-real-8 </base>
+  </FC_AUTO_R8>
+  <FFLAGS>
+    <!-- -ffree-line-length-none and -ffixed-line-length-none need to be in FFLAGS rather than in FIXEDFLAGS/FREEFLAGS
+       so that these are passed to cmake builds (cmake builds don't use FIXEDFLAGS and FREEFLAGS). -->
+    <base>  -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none </base>
+    <append compile_threaded="TRUE"> -fopenmp </append>
+    <!-- Ideally, we would also have 'invalid' in the ffpe-trap list. But at
+         least with some versions of gfortran (confirmed with 5.4.0, 6.3.0 and
+         7.1.0), gfortran's isnan (which is called in cime via the
+         CPRGNU-specific shr_infnan_isnan) causes a floating point exception
+         when called on a signaling NaN. -->
+    <append DEBUG="TRUE"> -g -Wall -Og -fbacktrace -ffpe-trap=zero,overflow -fcheck=bounds </append>
+    <append DEBUG="FALSE"> -O </append>
+  </FFLAGS>
+  <FFLAGS_NOOPT>
+    <base> -O0 </base>
+  </FFLAGS_NOOPT>
+  <FIXEDFLAGS>
+    <base>  -ffixed-form </base>
+  </FIXEDFLAGS>
+  <FREEFLAGS>
+    <base> -ffree-form </base>
+  </FREEFLAGS>
+  <HAS_F2008_CONTIGUOUS>FALSE</HAS_F2008_CONTIGUOUS>
+  <LDFLAGS>
+    <append compile_threaded="TRUE"> -fopenmp </append>
+  </LDFLAGS>
+  <MPICC> mpicc  </MPICC>
+  <MPICXX> mpicxx </MPICXX>
+  <MPIFC> mpif90 </MPIFC>
+  <SCC> gcc </SCC>
+  <SCXX> g++ </SCXX>
+  <SFC> gfortran </SFC>
+  <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
+</compiler>
+
+
+
+
 <compiler COMPILER="cray">
   <CFLAGS>
     <append compile_threaded="FALSE"> -h noomp </append>

diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml
@@ -2607,6 +2607,85 @@ This allows using a different mpirun command to launch unit tests
     </module_system>
   </machine>
 
+  <machine MACH="thunder">
+    <DESC>NCAR ARM platform, os is Linux, 64/128 pes/node, batch system is SLURM</DESC>
+    <NODENAME_REGEX>.*.thunder.ucar.edu</NODENAME_REGEX>
+    <OS>LINUX</OS>
+    <!-- arm LLVM compiler support is a work in progress -->
+    <COMPILERS>armgcc,gnu,arm</COMPILERS>
+    <MPILIBS>openmpi</MPILIBS>
+    <CIME_OUTPUT_ROOT>/glade/scratch/$USER</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>$ENV{CESMDATAROOT}/inputdata</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>$DIN_LOC_ROOT/CTSM_datm_forcing_data</DIN_LOC_ROOT_CLMFORC>
+    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
+    <BASELINE_ROOT>$ENV{CESMDATAROOT}/cesm_baselines</BASELINE_ROOT>
+    <CCSM_CPRNC>$ENV{CESMDATAROOT}/tools/cprnc/cprnc</CCSM_CPRNC>
+    <GMAKE_J>16</GMAKE_J>
+    <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
+    <SUPPORTED_BY>cseg</SUPPORTED_BY>
+    <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
+    <mpirun mpilib="default">
+      <executable>mpiexec</executable>
+      <arguments>
+	<arg name="labelstdout">--tag-output</arg>
+	<arg name="num_tasks"> -np {{ total_tasks }}</arg>
+      </arguments>
+    </mpirun>
+    <module_system type="module">
+      <init_path lang="perl">/glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/perl</init_path>
+      <init_path lang="python">/glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/env_modules_python.py</init_path>
+      <init_path lang="csh">/glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/csh</init_path>
+      <init_path lang="sh">/glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/sh</init_path>
+      <cmd_path lang="perl">/glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/libexec/lmod perl</cmd_path>
+      <cmd_path lang="python">/glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/libexec/lmod python</cmd_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <modules>
+	<command name="purge"/>
+	<command name="load">ncarenv/1.3</command>
+	<command name="load">cmake/3.14.4</command>
+      </modules>
+      <modules compiler="arm">
+	<command name="load">arm/19.3</command>
+      </modules>
+      <modules compiler="armgcc">
+	<command name="load">armgcc/8.2.0</command>
+      </modules>
+      <modules compiler="gnu">
+        <command name="load">gnu/9.1.0</command>
+        <command name="load">openblas/0.3.6</command>
+	<command name="load">esmf_libs/8.0.0</command>
+      </modules>
+      <!-- must load after compiler -->
+      <modules>
+	<command name="load">ncarcompilers/0.5.0</command>
+      </modules>	
+      <modules mpilib="openmpi">
+	<command name="load">openmpi/4.0.3</command>
+	<command name="load">netcdf-mpi/4.7.1</command>
+	<command name="load">pnetcdf/1.12.1</command>
+      </modules>
+      <modules mpilib="mpi-serial">
+	<command name="load">netcdf/4.7.1</command>
+      </modules>
+      <modules compiler="gnu" mpilib="mpi-serial" DEBUG="TRUE">
+	<command name="load">esmf-8.0.0-ncdfio-uni-g</command>
+      </modules>
+      <modules compiler="gnu" mpilib="mpi-serial" DEBUG="FALSE">
+	<command name="load">esmf-8.0.0-ncdfio-uni-O</command>
+      </modules>
+    </module_system>
+    <environment_variables>
+      <env name="OMP_STACKSIZE">256M</env>
+      <env name="NETCDF_PATH">$ENV{NETCDF}</env>
+    </environment_variables>
+    <environment_variables comp_interface="nuopc">
+      <env name="ESMF_RUNTIME_PROFILE">ON</env>
+      <env name="ESMF_RUNTIME_PROFILE_OUTPUT">SUMMARY</env>
+    </environment_variables>
+  </machine>
+
   <default_run_suffix>
     <default_run_exe>${EXEROOT}/cesm.exe </default_run_exe>
     <default_run_misc_suffix> >> cesm.log.$LID 2>&amp;1 </default_run_misc_suffix>

diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py
@@ -260,7 +260,7 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid
             case.set_value("CCSM_CPRNC", os.path.join(full_lib_path, "cprnc"))
             if not os.path.isdir(full_lib_path):
                 os.makedirs(full_lib_path)
-                libs.append("cprnc")
+                libs.insert(0,"cprnc")
 
     logs = []
 

diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py
@@ -2390,10 +2390,11 @@ def test_xml_caching(self):
     ###########################################################################
     def test_configure(self):
     ###########################################################################
-        self._create_test(["SMS.f09_g16.X", "--no-build"], test_id=self._baseline_name)
+        testname = "SMS.f09_g16.X"
+        self._create_test([testname, "--no-build"], test_id=self._baseline_name)
 
         casedir = os.path.join(self._testroot,
-                               "{}.{}".format(CIME.utils.get_full_test_name("SMS.f09_g16.X", machine=self._machine, compiler=self._compiler), self._baseline_name))
+                               "{}.{}".format(CIME.utils.get_full_test_name(testname, machine=self._machine, compiler=self._compiler), self._baseline_name))
 
         manual_config_dir = os.path.join(casedir, "manual_config")
         os.mkdir(manual_config_dir)
@@ -2411,10 +2412,11 @@ def test_configure(self):
     ###########################################################################
     def test_self_build_cprnc(self):
     ###########################################################################
-        self._create_test(["ERS.f19_g16_rx1.A", "--no-build"], test_id=self._baseline_name)
+        testname = "ERS_Ln7.f19_g16.X"
+        self._create_test([testname, "--no-build"], test_id=self._baseline_name)
 
         casedir = os.path.join(self._testroot,
-                               "{}.{}".format(CIME.utils.get_full_test_name("ERS.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), self._baseline_name))
+                               "{}.{}".format(CIME.utils.get_full_test_name(testname, machine=self._machine, compiler=self._compiler), self._baseline_name))
 
         run_cmd_assert_result(self, "./xmlchange CCSM_CPRNC=this_is_a_broken_cprnc",
                               from_dir=casedir)

diff --git a/src/build_scripts/buildlib.cprnc b/src/build_scripts/buildlib.cprnc
@@ -51,15 +51,15 @@ def buildlib(bldroot, installpath, case):
     run_bld_cmd_ensure_logging("{}/tools/configure --mpilib=mpi-serial --macros-format=CMake --machine={} --compiler={}".format(cimeroot, case.get_value("MACH"), compiler), logger, from_dir=bldroot)
 
     cmake_args = get_standard_cmake_args(case, shared_lib=True)
-
-    cmake_cmd = "CIMEROOT={cimeroot} . ./.env_mach_specific.sh && NETCDF=$(dirname $(dirname $(which nf-config))) CIMEROOT={cimeroot} cmake {cmake_args} -DMPILIB=mpi-serial -DDEBUG=FALSE -C Macros.cmake {cimeroot}/tools/cprnc -DCMAKE_PREFIX_PATH={dest_path} -DBLDROOT={bldroot}".\
+    os.environ["CIMEROOT"] = cimeroot
+    cmake_cmd = ". ./.env_mach_specific.sh && NETCDF=$(dirname $(dirname $(which nf-config))) cmake {cmake_args} -DMPILIB=mpi-serial -DDEBUG=FALSE -C Macros.cmake {cimeroot}/tools/cprnc -DCMAKE_PREFIX_PATH={dest_path} -DBLDROOT={bldroot}".\
         format(cimeroot=cimeroot, dest_path=installpath, cmake_args=cmake_args, bldroot=bldroot)
     run_bld_cmd_ensure_logging(cmake_cmd, logger, from_dir=bldroot)
 
     gmake_cmd = case.get_value("GMAKE")
     gmake_j = case.get_value("GMAKE_J")
 
-    run_bld_cmd_ensure_logging("{} VERBOSE=1 -j {}".format(gmake_cmd, gmake_j), logger, from_dir=bldroot)
+    run_bld_cmd_ensure_logging(". ./.env_mach_specific.sh && {} VERBOSE=1 -j {}".format(gmake_cmd, gmake_j), logger, from_dir=bldroot)
 
 def _main(argv, documentation):
     bldroot, installpath, caseroot = parse_command_line(argv, documentation)

diff --git a/src/share/util/shr_infnan_mod.F90.in b/src/share/util/shr_infnan_mod.F90.in
@@ -1,4 +1,3 @@
-
 ! Flag representing compiler support of Fortran 2003's
 ! ieee_arithmetic intrinsic module.
 #if defined CPRIBM || defined CPRPGI || defined CPRINTEL ||  defined CPRCRAY || defined CPRNAG