From 35720da848b879d5183525a85c1331955f10a231 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 9 Mar 2020 16:13:17 -0600 Subject: [PATCH 1/6] port to ncar system thunder --- config/cesm/machines/config_batch.xml | 15 +++++ config/cesm/machines/config_compilers.xml | 67 +++++++++++++++++++ config/cesm/machines/config_machines.xml | 79 +++++++++++++++++++++++ scripts/Tools/Makefile | 15 +++-- scripts/lib/CIME/build.py | 2 +- scripts/tests/scripts_regression_tests.py | 10 +-- src/share/util/shr_infnan_mod.F90.in | 1 - 7 files changed, 176 insertions(+), 13 deletions(-) diff --git a/config/cesm/machines/config_batch.xml b/config/cesm/machines/config_batch.xml index a0212cc5b8c..c02864900c9 100644 --- a/config/cesm/machines/config_batch.xml +++ b/config/cesm/machines/config_batch.xml @@ -602,4 +602,19 @@ default + + sbatch + + + + + + --ntasks-per-node=64 + --hint=nomultithread + + + regular + + + diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml index 1e601b90ea8..78fb1c96403 100644 --- a/config/cesm/machines/config_compilers.xml +++ b/config/cesm/machines/config_compilers.xml @@ -85,6 +85,73 @@ using a fortran linker. FALSE + + + mpicc + mpicxx + mpif90 + clang + clang++ + flang + + -DCPRLLVM + -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY + + + + + + -std=gnu99 + -fopenmp + -g -Wall -Og -fbacktrace -ffpe-trap=invalid,zero,overflow -fcheck=bounds + -O + + + + -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU + + FORTRAN + + -fdefault-real-8 + + + + -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none + -fopenmp + + -g -Wall -Og -fbacktrace -ffpe-trap=zero,overflow -fcheck=bounds + -O + + + -O0 + + + -ffixed-form + + + -ffree-form + + FALSE + + -fopenmp + + mpicc + mpicxx + mpif90 + gcc + g++ + gfortran + TRUE + + + + + -h noomp diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml index 84ab79d1e91..1a439b13eda 100644 --- a/config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -2607,6 +2607,85 @@ This allows using a different mpirun command to launch unit tests + + NCAR ARM platform, os is Linux, 64/128 pes/node, batch system is SLURM + .*.thunder.ucar.edu + LINUX + + armgcc,gnu,arm + openmpi + /glade/scratch/$USER + $ENV{CESMDATAROOT}/inputdata + $DIN_LOC_ROOT/CTSM_datm_forcing_data + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{CESMDATAROOT}/cesm_baselines + $ENV{CESMDATAROOT}/tools/cprnc/cprnc + 16 + slurm + cseg + 64 + 128 + + mpiexec + + --tag-output + -np {{ total_tasks }} + + + + /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/perl + /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/env_modules_python.py + /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/csh + /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/sh + /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/libexec/lmod perl + /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/libexec/lmod python + module + module + + + ncarenv/1.3 + cmake/3.14.4 + + + arm/19.3 + + + armgcc/8.2.0 + + + gnu/9.1.0 + openblas/0.3.6 + esmf_libs/8.0.0 + + + + ncarcompilers/0.5.0 + + + openmpi/4.0.3 + netcdf-mpi/4.7.1 + pnetcdf/1.12.1 + + + netcdf/4.7.1 + + + esmf-8.0.0-ncdfio-uni-g + + + esmf-8.0.0-ncdfio-uni-O + + + + 256M + $ENV{NETCDF} + + + ON + SUMMARY + + + ${EXEROOT}/cesm.exe >> cesm.log.$LID 2>&1 diff --git a/scripts/Tools/Makefile b/scripts/Tools/Makefile index f7823dadc5f..33bfeaeea57 100644 --- a/scripts/Tools/Makefile +++ b/scripts/Tools/Makefile @@ -573,16 +573,17 @@ ifeq ($(findstring -cosp,$(CAM_CONFIG_OPTS)),-cosp) endif endif -CAM_DYCORE ?= $(shell $(CASEROOT)/xmlquery --caseroot $(CASEROOT) CAM_DYCORE --value) +ifeq ($(MODEL),cam) + CAM_DYCORE ?= $(shell $(CASEROOT)/xmlquery --caseroot $(CASEROOT) CAM_DYCORE --value) -ifeq ($(CAM_DYCORE),fv3) -FV3CORE_LIBDIR:=$(abspath $(EXEROOT)/atm/obj/atmos_cubed_sphere) -INCLDIR+=-I$(FV3CORE_LIBDIR) -I$(FV3CORE_LIBDIR)/../ -I../$(INSTALL_SHAREDPATH)/include -I../$(CSM_SHR_INCLUDE) -I$(abspath $(EXEROOT)/FMS) -I$(CIMEROOT)/../libraries/FMS/src/include -endif + ifeq ($(CAM_DYCORE),fv3) + FV3CORE_LIBDIR:=$(abspath $(EXEROOT)/atm/obj/atmos_cubed_sphere) + INCLDIR+=-I$(FV3CORE_LIBDIR) -I$(FV3CORE_LIBDIR)/../ -I../$(INSTALL_SHAREDPATH)/include -I../$(CSM_SHR_INCLUDE) -I$(abspath $(EXEROOT)/FMS) -I$(CIMEROOT)/../libraries/FMS/src/include + endif -ifeq ($(MODEL),cam) - # These RRTMG files take an extraordinarily long time to compile with optimization. + +# These RRTMG files take an extraordinarily long time to compile with optimization. # Until mods are made to read the data from files, just remove optimization from # their compilation. rrtmg_lw_k_g.o: rrtmg_lw_k_g.f90 diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py index 7720cb209ee..d77699f04f9 100644 --- a/scripts/lib/CIME/build.py +++ b/scripts/lib/CIME/build.py @@ -260,7 +260,7 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid case.set_value("CCSM_CPRNC", os.path.join(full_lib_path, "cprnc")) if not os.path.isdir(full_lib_path): os.makedirs(full_lib_path) - libs.append("cprnc") + libs.insert(0,"cprnc") logs = [] diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index 19049a4cd26..88773f20ef0 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -2390,10 +2390,11 @@ def test_xml_caching(self): ########################################################################### def test_configure(self): ########################################################################### - self._create_test(["SMS.f09_g16.X", "--no-build"], test_id=self._baseline_name) + testname = "SMS.f09_g16.X" + self._create_test([testname, "--no-build"], test_id=self._baseline_name) casedir = os.path.join(self._testroot, - "{}.{}".format(CIME.utils.get_full_test_name("SMS.f09_g16.X", machine=self._machine, compiler=self._compiler), self._baseline_name)) + "{}.{}".format(CIME.utils.get_full_test_name(testname, machine=self._machine, compiler=self._compiler), self._baseline_name)) manual_config_dir = os.path.join(casedir, "manual_config") os.mkdir(manual_config_dir) @@ -2411,10 +2412,11 @@ def test_configure(self): ########################################################################### def test_self_build_cprnc(self): ########################################################################### - self._create_test(["ERS.f19_g16_rx1.A", "--no-build"], test_id=self._baseline_name) + testname = "ERS_Ln7.f19_g16.X" + self._create_test([testname, "--no-build"], test_id=self._baseline_name) casedir = os.path.join(self._testroot, - "{}.{}".format(CIME.utils.get_full_test_name("ERS.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), self._baseline_name)) + "{}.{}".format(CIME.utils.get_full_test_name(testname, machine=self._machine, compiler=self._compiler), self._baseline_name)) run_cmd_assert_result(self, "./xmlchange CCSM_CPRNC=this_is_a_broken_cprnc", from_dir=casedir) diff --git a/src/share/util/shr_infnan_mod.F90.in b/src/share/util/shr_infnan_mod.F90.in index 74f37ba4ea3..992c46fc9b8 100644 --- a/src/share/util/shr_infnan_mod.F90.in +++ b/src/share/util/shr_infnan_mod.F90.in @@ -1,4 +1,3 @@ - ! Flag representing compiler support of Fortran 2003's ! ieee_arithmetic intrinsic module. #if defined CPRIBM || defined CPRPGI || defined CPRINTEL || defined CPRCRAY || defined CPRNAG From 3e358cab27d8f1673f17973c9ea03b5af2c3a873 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 10 Mar 2020 13:11:04 -0600 Subject: [PATCH 2/6] need to be in mpi-serial environment for build --- src/build_scripts/buildlib.cprnc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/build_scripts/buildlib.cprnc b/src/build_scripts/buildlib.cprnc index 53be9bd6219..d1e52653540 100755 --- a/src/build_scripts/buildlib.cprnc +++ b/src/build_scripts/buildlib.cprnc @@ -51,15 +51,15 @@ def buildlib(bldroot, installpath, case): run_bld_cmd_ensure_logging("{}/tools/configure --mpilib=mpi-serial --macros-format=CMake --machine={} --compiler={}".format(cimeroot, case.get_value("MACH"), compiler), logger, from_dir=bldroot) cmake_args = get_standard_cmake_args(case, shared_lib=True) - - cmake_cmd = "CIMEROOT={cimeroot} . ./.env_mach_specific.sh && NETCDF=$(dirname $(dirname $(which nf-config))) CIMEROOT={cimeroot} cmake {cmake_args} -DMPILIB=mpi-serial -DDEBUG=FALSE -C Macros.cmake {cimeroot}/tools/cprnc -DCMAKE_PREFIX_PATH={dest_path} -DBLDROOT={bldroot}".\ + os.environ["CIMEROOOT"] = cimeroot + cmake_cmd = ". ./.env_mach_specific.sh && NETCDF=$(dirname $(dirname $(which nf-config))) cmake {cmake_args} -DMPILIB=mpi-serial -DDEBUG=FALSE -C Macros.cmake {cimeroot}/tools/cprnc -DCMAKE_PREFIX_PATH={dest_path} -DBLDROOT={bldroot}".\ format(cimeroot=cimeroot, dest_path=installpath, cmake_args=cmake_args, bldroot=bldroot) run_bld_cmd_ensure_logging(cmake_cmd, logger, from_dir=bldroot) gmake_cmd = case.get_value("GMAKE") gmake_j = case.get_value("GMAKE_J") - run_bld_cmd_ensure_logging("{} VERBOSE=1 -j {}".format(gmake_cmd, gmake_j), logger, from_dir=bldroot) + run_bld_cmd_ensure_logging(". ./.env_mach_specific.sh {} VERBOSE=1 -j {}".format(gmake_cmd, gmake_j), logger, from_dir=bldroot) def _main(argv, documentation): bldroot, installpath, caseroot = parse_command_line(argv, documentation) From 4c2a91d4ff42eb4e1c0f9ffc5b76f19b92d9cd51 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 10 Mar 2020 14:00:15 -0600 Subject: [PATCH 3/6] improve fix --- src/build_scripts/buildlib.cprnc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build_scripts/buildlib.cprnc b/src/build_scripts/buildlib.cprnc index d1e52653540..9ea5838153e 100755 --- a/src/build_scripts/buildlib.cprnc +++ b/src/build_scripts/buildlib.cprnc @@ -59,7 +59,7 @@ def buildlib(bldroot, installpath, case): gmake_cmd = case.get_value("GMAKE") gmake_j = case.get_value("GMAKE_J") - run_bld_cmd_ensure_logging(". ./.env_mach_specific.sh {} VERBOSE=1 -j {}".format(gmake_cmd, gmake_j), logger, from_dir=bldroot) + run_bld_cmd_ensure_logging(". ./.env_mach_specific.sh && {} VERBOSE=1 -j {}".format(gmake_cmd, gmake_j), logger, from_dir=bldroot) def _main(argv, documentation): bldroot, installpath, caseroot = parse_command_line(argv, documentation) From 3adb64ff94bbd7fb0a35c43294f8e3a2c8a0eab9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 13 Mar 2020 09:59:33 -0600 Subject: [PATCH 4/6] two Os not three --- src/build_scripts/buildlib.cprnc | 2 +- src/externals/pio1/pio/piolib_mod.F90 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build_scripts/buildlib.cprnc b/src/build_scripts/buildlib.cprnc index 9ea5838153e..9bf21b7fc60 100755 --- a/src/build_scripts/buildlib.cprnc +++ b/src/build_scripts/buildlib.cprnc @@ -51,7 +51,7 @@ def buildlib(bldroot, installpath, case): run_bld_cmd_ensure_logging("{}/tools/configure --mpilib=mpi-serial --macros-format=CMake --machine={} --compiler={}".format(cimeroot, case.get_value("MACH"), compiler), logger, from_dir=bldroot) cmake_args = get_standard_cmake_args(case, shared_lib=True) - os.environ["CIMEROOOT"] = cimeroot + os.environ["CIMEROOT"] = cimeroot cmake_cmd = ". ./.env_mach_specific.sh && NETCDF=$(dirname $(dirname $(which nf-config))) cmake {cmake_args} -DMPILIB=mpi-serial -DDEBUG=FALSE -C Macros.cmake {cimeroot}/tools/cprnc -DCMAKE_PREFIX_PATH={dest_path} -DBLDROOT={bldroot}".\ format(cimeroot=cimeroot, dest_path=installpath, cmake_args=cmake_args, bldroot=bldroot) run_bld_cmd_ensure_logging(cmake_cmd, logger, from_dir=bldroot) diff --git a/src/externals/pio1/pio/piolib_mod.F90 b/src/externals/pio1/pio/piolib_mod.F90 index b45e552dbd0..b169a8dba2b 100644 --- a/src/externals/pio1/pio/piolib_mod.F90 +++ b/src/externals/pio1/pio/piolib_mod.F90 @@ -1231,7 +1231,7 @@ subroutine PIO_initdecomp_dof_i8(iosystem,basepiotype,dims,compdof, iodesc, iost if (associated(displace)) then call dealloc_check(displace) endif - + print *,__FILE__,__LINE__,compdof #ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then From b353b9f1dc9e8fd1de9b593b34290cb8be251812 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 13 Mar 2020 10:00:14 -0600 Subject: [PATCH 5/6] revert unintended commit --- src/externals/pio1/pio/piolib_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/externals/pio1/pio/piolib_mod.F90 b/src/externals/pio1/pio/piolib_mod.F90 index b169a8dba2b..b45e552dbd0 100644 --- a/src/externals/pio1/pio/piolib_mod.F90 +++ b/src/externals/pio1/pio/piolib_mod.F90 @@ -1231,7 +1231,7 @@ subroutine PIO_initdecomp_dof_i8(iosystem,basepiotype,dims,compdof, iodesc, iost if (associated(displace)) then call dealloc_check(displace) endif - print *,__FILE__,__LINE__,compdof + #ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then From 6c85f7311f7adcb1aa7a8afb31e5b52c53a27f26 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 13 Mar 2020 15:40:51 -0600 Subject: [PATCH 6/6] remove Makefile change from this PR --- scripts/Tools/Makefile | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/Tools/Makefile b/scripts/Tools/Makefile index 33bfeaeea57..f7823dadc5f 100644 --- a/scripts/Tools/Makefile +++ b/scripts/Tools/Makefile @@ -573,17 +573,16 @@ ifeq ($(findstring -cosp,$(CAM_CONFIG_OPTS)),-cosp) endif endif -ifeq ($(MODEL),cam) - CAM_DYCORE ?= $(shell $(CASEROOT)/xmlquery --caseroot $(CASEROOT) CAM_DYCORE --value) - - ifeq ($(CAM_DYCORE),fv3) - FV3CORE_LIBDIR:=$(abspath $(EXEROOT)/atm/obj/atmos_cubed_sphere) - INCLDIR+=-I$(FV3CORE_LIBDIR) -I$(FV3CORE_LIBDIR)/../ -I../$(INSTALL_SHAREDPATH)/include -I../$(CSM_SHR_INCLUDE) -I$(abspath $(EXEROOT)/FMS) -I$(CIMEROOT)/../libraries/FMS/src/include - endif +CAM_DYCORE ?= $(shell $(CASEROOT)/xmlquery --caseroot $(CASEROOT) CAM_DYCORE --value) +ifeq ($(CAM_DYCORE),fv3) +FV3CORE_LIBDIR:=$(abspath $(EXEROOT)/atm/obj/atmos_cubed_sphere) +INCLDIR+=-I$(FV3CORE_LIBDIR) -I$(FV3CORE_LIBDIR)/../ -I../$(INSTALL_SHAREDPATH)/include -I../$(CSM_SHR_INCLUDE) -I$(abspath $(EXEROOT)/FMS) -I$(CIMEROOT)/../libraries/FMS/src/include +endif -# These RRTMG files take an extraordinarily long time to compile with optimization. +ifeq ($(MODEL),cam) + # These RRTMG files take an extraordinarily long time to compile with optimization. # Until mods are made to read the data from files, just remove optimization from # their compilation. rrtmg_lw_k_g.o: rrtmg_lw_k_g.f90