Merge pull request NCAR#264 from climbfuji/cmakelists_separate_precis…

…ion_flags_includes_mynnpbl_bugfix

Cmakelists separate precision flags includes mynnpbl bugfix

CMakeLists.txt

@@ -100,9 +100,12 @@ list(APPEND LIBS "ccpp")
 include(./CCPP_SCHEMES.cmake)
 # Set the sources: physics scheme caps
 include(./CCPP_CAPS.cmake)
-# Create empty lists for schemes with special compiler flags
-set(SCHEMES_SFX "")
-
+# Create empty lists for schemes with special compiler optimization flags
+set(SCHEMES_SFX_OPT "")
+# Create empty lists for schemes with special floating point precision flags
+set(SCHEMES_SFX_PREC "")
+# Create a duplicate of the SCHEMES list for handling floating point precision flags
+set(SCHEMES2 ${SCHEMES})
 #------------------------------------------------------------------------------
 if (${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ffree-line-length-none")
@@ -112,11 +115,33 @@ if (${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU")
   SET_SOURCE_FILES_PROPERTIES(./physics/module_nst_water_prop.f90 PROPERTIES COMPILE_FLAGS "-ffree-line-length-none -fdefault-real-8 -ffree-form")
   SET_SOURCE_FILES_PROPERTIES(./physics/aer_cloud.F ./physics/wv_saturation.F ./physics/cldwat2m_micro.F ./physics/surface_perturbation.F90 PROPERTIES COMPILE_FLAGS "-fdefault-real-8 -fdefault-double-8")
   SET_SOURCE_FILES_PROPERTIES(./physics/module_mp_thompson_make_number_concentrations.F90 PROPERTIES COMPILE_FLAGS "-fdefault-real-8 -fdefault-double-8")
+
   if (PROJECT STREQUAL "CCPP-FV3")
+    # Set 32-bit floating point precision flags for certain files
+    # that are executed in the dynamics (fast physics part)
     if (DYN32)
-      message (FATAL_ERROR "The current build system does not allow building fast physics with 32-bit precision when the GNU compilers are used")
+      # Reduce floating point precision from 64-bit to 32-bit, if necessary
+      set(CMAKE_Fortran_FLAGS_PREC32 ${CMAKE_Fortran_FLAGS_DEFAULT_PREC})
+      string(REPLACE "-fdefault-real-8" ""
+             CMAKE_Fortran_FLAGS_PREC32 "${CMAKE_Fortran_FLAGS_PREC32}")
+      string(REPLACE "-fdefault-double-8" ""
+             CMAKE_Fortran_FLAGS_PREC32 "${CMAKE_Fortran_FLAGS_PREC32}")
+      SET_PROPERTY(SOURCE ./physics/gfdl_fv_sat_adj.F90
+                   APPEND_STRING PROPERTY COMPILE_FLAGS " ${CMAKE_Fortran_FLAGS_PREC32} ")
+      # Add all of the above files to the list of schemes with special floating point precision flags
+      list(APPEND SCHEMES_SFX_PREC ./physics/gfdl_fv_sat_adj.F90)
     endif (DYN32)
+
+    # Remove files with special floating point precision flags from list
+    # of files with standard floating point precision flags flags
+    if (SCHEMES_SFX_PREC)
+      list(REMOVE_ITEM SCHEMES2 ${SCHEMES_SFX_PREC})
+    endif (SCHEMES_SFX_PREC)
+    # Assign standard floating point precision flags to all remaining schemes and caps
+    SET_PROPERTY(SOURCE ${SCHEMES2} ${CAPS}
+                 APPEND_STRING PROPERTY COMPILE_FLAGS " ${CMAKE_Fortran_FLAGS_DEFAULT_PREC} ")
   endif (PROJECT STREQUAL "CCPP-FV3")
+
 elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
   # Adjust settings for bit-for-bit reproducibility of NEMSfv3gfs
   if (PROJECT STREQUAL "CCPP-FV3")
@@ -158,7 +183,7 @@ elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
     SET_SOURCE_FILES_PROPERTIES(./physics/radiation_aerosols.f
                                 PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_LOPT1}")
     # Add all of the above files to the list of schemes with special compiler flags
-    list(APPEND SCHEMES_SFX ./physics/radiation_aerosols.f)
+    list(APPEND SCHEMES_SFX_OPT ./physics/radiation_aerosols.f)
 
     # Force consistent results of math calculations for MG microphysics;
     # in Debug/Bitforbit mode; without this flag, the results of the
@@ -194,7 +219,7 @@ elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
                                   ./physics/gcm_shoc.F90
                                   PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_LOPT2}")
       # Add all of the above files to the list of schemes with special compiler flags
-      list(APPEND SCHEMES_SFX ./physics/micro_mg2_0.F90
+      list(APPEND SCHEMES_SFX_OPT ./physics/micro_mg2_0.F90
                               ./physics/micro_mg3_0.F90
                               ./physics/aer_cloud.F
                               ./physics/cldmacro.F
@@ -207,17 +232,33 @@ elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
     endif (TRANSITION)
 
     # Remove files with special compiler flags from list of files with standard compiler flags
-    list(REMOVE_ITEM SCHEMES ${SCHEMES_SFX})
+    list(REMOVE_ITEM SCHEMES ${SCHEMES_SFX_OPT})
     # Assign standard compiler flags to all remaining schemes and caps
     SET_SOURCE_FILES_PROPERTIES(${SCHEMES} ${CAPS}
                                 PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_OPT}")
 
-    # This has to come last: append 32-bit dynamics flags to certain files that are executed
-    # in the dynamics (fast physics part); this will overwrite any preceding -real-size 64
+    # Set 32-bit floating point precision flags for certain files
+    # that are executed in the dynamics (fast physics part)
     if (DYN32)
-      SET_PROPERTY(SOURCE ./physics/gfdl_fv_sat_adj.F90 APPEND_STRING PROPERTY COMPILE_FLAGS " -real-size 32 ")
+      # Reduce floating point precision from 64-bit to 32-bit, if necessary
+      set(CMAKE_Fortran_FLAGS_PREC32 ${CMAKE_Fortran_FLAGS_DEFAULT_PREC})
+      string(REPLACE "-real-size 64" "-real-size 32"
+             CMAKE_Fortran_FLAGS_PREC32 "${CMAKE_Fortran_FLAGS_PREC32}")
+      SET_PROPERTY(SOURCE ./physics/gfdl_fv_sat_adj.F90
+                   APPEND_STRING PROPERTY COMPILE_FLAGS " ${CMAKE_Fortran_FLAGS_PREC32} ")
+      # Add all of the above files to the list of schemes with special floating point precision flags
+      list(APPEND SCHEMES_SFX_PREC ./physics/gfdl_fv_sat_adj.F90)
     endif (DYN32)
 
+    # Remove files with special floating point precision flags from list
+    # of files with standard floating point precision flags flags
+    if (SCHEMES_SFX_PREC)
+      list(REMOVE_ITEM SCHEMES2 ${SCHEMES_SFX_PREC})
+    endif (SCHEMES_SFX_PREC)
+    # Assign standard floating point precision flags to all remaining schemes and caps
+    SET_PROPERTY(SOURCE ${SCHEMES2} ${CAPS}
+                 APPEND_STRING PROPERTY COMPILE_FLAGS " ${CMAKE_Fortran_FLAGS_DEFAULT_PREC} ")
+
   else (PROJECT STREQUAL "CCPP-FV3")
     SET_SOURCE_FILES_PROPERTIES(./physics/module_bfmicrophysics.f ./physics/sflx.f ./physics/sfc_diff.f ./physics/sfc_diag.f PROPERTIES COMPILE_FLAGS -r8)
     SET_SOURCE_FILES_PROPERTIES(./physics/module_nst_model.f90 ./physics/calpreciptype.f90 PROPERTIES COMPILE_FLAGS "-r8 -free")
@@ -233,9 +274,28 @@ elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "PGI")
   SET_SOURCE_FILES_PROPERTIES(./physics/aer_cloud.F ./physics/wv_saturation.F ./physics/cldwat2m_micro.F ./physics/surface_perturbation.F90 PROPERTIES COMPILE_FLAGS "-r8")
   SET_SOURCE_FILES_PROPERTIES(./physics/module_mp_thompson_make_number_concentrations.F90 PROPERTIES COMPILE_FLAGS "-r8")
   if (PROJECT STREQUAL "CCPP-FV3")
+    # Set 32-bit floating point precision flags for certain files
+    # that are executed in the dynamics (fast physics part)
     if (DYN32)
-      SET_PROPERTY(SOURCE ./physics/gfdl_fv_sat_adj.F90 APPEND_STRING PROPERTY COMPILE_FLAGS " -r4 ")
+      # Reduce floating point precision from 64-bit to 32-bit, if necessary
+      set(CMAKE_Fortran_FLAGS_PREC32 ${CMAKE_Fortran_FLAGS_DEFAULT_PREC})
+      string(REPLACE "-r8" "-r4"
+             CMAKE_Fortran_FLAGS_PREC32 "${CMAKE_Fortran_FLAGS_PREC32}")
+      SET_PROPERTY(SOURCE ./physics/gfdl_fv_sat_adj.F90
+                   APPEND_STRING PROPERTY COMPILE_FLAGS " ${CMAKE_Fortran_FLAGS_PREC32} ")
+      # Add all of the above files to the list of schemes with special floating point precision flags
+      list(APPEND SCHEMES_SFX_PREC ./physics/gfdl_fv_sat_adj.F90)
     endif (DYN32)
+
+    # Remove files with special floating point precision flags from list
+    # of files with standard floating point precision flags flags
+    if (SCHEMES_SFX_PREC)
+      list(REMOVE_ITEM SCHEMES2 ${SCHEMES_SFX_PREC})
+    endif (SCHEMES_SFX_PREC)
+    # Assign standard floating point precision flags to all remaining schemes and caps
+    SET_PROPERTY(SOURCE ${SCHEMES2} ${CAPS}
+                 APPEND_STRING PROPERTY COMPILE_FLAGS " ${CMAKE_Fortran_FLAGS_DEFAULT_PREC} ")
+
   endif (PROJECT STREQUAL "CCPP-FV3")
 else (${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU")
   message ("CMAKE_Fortran_COMPILER full path: " ${CMAKE_Fortran_COMPILER})
@@ -252,14 +312,14 @@ endif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU")
 # physics schemes, these checks can and should remain enabled. Overwriting
 # the pointer check flags explicitly works for Intel and GNU, but not for PGI.
 if (${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU")
-  set_property(SOURCE ${CAPS} PROPERTY COMPILE_FLAGS "-fcheck=no-pointer,no-bounds")
+  set_property(SOURCE ${CAPS} APPEND_STRING PROPERTY COMPILE_FLAGS " -fcheck=no-pointer,no-bounds ")
 elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
-  set_property(SOURCE ${CAPS} PROPERTY COMPILE_FLAGS "-check nopointers,nobounds")
+  set_property(SOURCE ${CAPS} APPEND_STRING PROPERTY COMPILE_FLAGS " -check nopointers,nobounds ")
 elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "PGI")
   if (CMAKE_Fortran_FLAGS MATCHES ".*chkptr.*")
     message (FATAL_ERROR "PGI compiler option chkptr cannot be used for CCPP physics")
   endif (CMAKE_Fortran_FLAGS MATCHES ".*chkptr.*")
-  set_property(SOURCE ${CAPS} PROPERTY COMPILE_FLAGS "-Mnobounds")
+  set_property(SOURCE ${CAPS} APPEND_STRING PROPERTY COMPILE_FLAGS " -Mnobounds ")
 endif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU")
 
 if (PROJECT STREQUAL "CCPP-SCM")
@@ -269,15 +329,15 @@ endif (PROJECT STREQUAL "CCPP-SCM")
 #------------------------------------------------------------------------------
 
 if(STATIC)
-  add_library(ccppphys STATIC ${SCHEMES} ${SCHEMES_SFX} ${CAPS})
+  add_library(ccppphys STATIC ${SCHEMES} ${SCHEMES_SFX_OPT} ${CAPS})
   # Generate list of Fortran modules from defined sources
   foreach(source_f90 ${CAPS})
       string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${source_f90})
       string(TOLOWER ${tmp_module_f90} module_f90)
       list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/../${module_f90})
   endforeach()
 else(STATIC)
-  add_library(ccppphys SHARED ${SCHEMES} ${SCHEMES_SFX} ${CAPS})
+  add_library(ccppphys SHARED ${SCHEMES} ${SCHEMES_SFX_OPT} ${CAPS})
 endif(STATIC)
 
 if (NOT STATIC)

physics/module_bl_mynn.F90

@@ -5735,8 +5735,13 @@ SUBROUTINE DMP_mf(                       &
     !     d(k)=thl(k) + dtz(k)*flt + tcd(k)*delt &
     !        & -dtz(k)*s_awthl(kts+1) + diss_heat(k)*delt*dheat_opt
     ! So, s_awthl(kts+1) must be less than flt
-    THVk = (THL(kts)*DZ(kts+1)+THL(kts+1)*DZ(kts))/(DZ(kts+1)+DZ(kts))
-    flx1 = MAX(s_aw(kts+1)*(s_awthl(kts+1)/s_aw(kts+1) - THVk),0.0)
+    !GJF: check if s_aw(kts+1) /= 0 before using it; if KTOP=0, s_aw(kts+1) = 0; caught using -fpe0 with intel compiler
+    IF (s_aw(kts+1) /= 0.) THEN
+      THVk = (THL(kts)*DZ(kts+1)+THL(kts+1)*DZ(kts))/(DZ(kts+1)+DZ(kts))
+      flx1 = MAX(s_aw(kts+1)*(s_awthl(kts+1)/s_aw(kts+1) - THVk),0.0)
+    ELSE
+      flx1 = 0.0
+    ENDIF
     !flx1 = -dt/dz(kts)*s_awthl(kts+1)
     !flx1 = (s_awthl(kts+1)-s_awthl(kts))!/(0.5*(dz(k)+dz(k-1)))
     adjustment=1.0