Merge pull request #2590 from stfc/2589_fix_lfric_integration_tests

(closes #2589) Update LFRic integration test
stfc · May 15, 2024 · 8ccdebe · 8ccdebe
2 parents 1482f5c + 5c6d70d
commit 8ccdebe
Show file tree

Hide file tree

Showing 9 changed files with 864 additions and 859 deletions.
diff --git a/.github/workflows/lfric_test.yml b/.github/workflows/lfric_test.yml
@@ -46,7 +46,7 @@ jobs:
     if: ${{ github.repository == 'stfc/PSyclone-mirror' }}
     runs-on: self-hosted
     env:
-      LFRIC_REV: 45376
+      LFRIC_APPS_REV: 1192
       PYTHON_VERSION: 3.12.3
 
     steps:
@@ -79,34 +79,34 @@ jobs:
     - name: LFRic passthrough (with DistributedMemory)
       run: |
         # Set up environment
-        source /apps/spack/spack-repo/share/spack/setup-env.sh
-        spack env activate lfric-gnu
-        export FC="gfortran"
-        export CC="gcc"
-        export CXX="g++"
-        export FPP="cpp -traditional-cpp"
-        export LDMPI="mpif90"
-        export FFLAGS="-Wall -Werror=conversion -Werror=unused-variable -Werror=character-truncation -Werror=unused-value -Werror=tabs -I$(spack location -e lfric-gnu)/.spack-env/view/include"
-        export LDFLAGS="-L$(spack location -e lfric-gnu)/.spack-env/view/lib"
-        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(spack location -e lfric-gnu)/.spack-env/view/lib"
-        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(spack location -e lfric-gnu)/.spack-env/view/lib64"
+        source /apps/spack/spack-upstream/share/spack/setup-env.sh
+        spack load lfric-buildenv%gcc
         source .runner_venv/bin/activate
         export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
-        export MINIAPP_DIR=${HOME}/LFRic_${LFRIC_REV}/miniapps/gungho_model
-        export PSYCLONE_CONFIG=${HOME}/lfric_psyclone_config.cfg
-        cd ${MINIAPP_DIR}
-        # Compile
-        make clean
-        make -j 6 build
+        export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
+        export LFRIC_DIR=${HOME}/LFRic/lfric_apps
+        cd ${LFRIC_DIR}
+        # Clean previous version and compile again
+        rm -rf appplications/gungho_model/working
+        ./build/local_build.py -a gungho_model
         # Run
-        cd example
+        cd applications/gungho_model/example
         cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its.nml configuration.nml
         mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt
         python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its_output.txt output.txt
         cat timer.txt
         export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
         export VAR_HALOS=$(grep "gungho_model" halo_calls_counter.txt | cut -d'|' -f5)
         echo $GITHUB_REF_NAME $GITHUB_SHA $VAR_TIME $VAR_HALOS >> ${HOME}/store_results/lfric_passthrough_performance_history
+        ${HOME}/mongosh-2.1.1-linux-x64/bin/mongosh \
+          "mongodb+srv://cluster0.x8ncpxi.mongodb.net/PerformanceMonitoring" \
+          --quiet --apiVersion 1 --username ${{ secrets.MONGODB_USERNAME }} \
+          --password ${{ secrets.MONGODB_PASSWORD }} \
+          --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'",
+          github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'",
+          ci_test: "LFRic Passthrough with DM", lfric_apps_version: '"$LFRIC_APPS_REV"', system: "GlaDos",
+          compiler:"spack-gfortran-11", date: new Date(), elapsed_time: '"$VAR_TIME"',
+          num_of_halo_exchanges: '"$VAR_HALOS"'})'
 
     - name: Upload LFRic passthrough results
       uses: exuanbo/actions-deploy-gist@v1
@@ -119,30 +119,22 @@ jobs:
     - name: LFRic with all transformations
       run: |
         # Set up environment
-        source /apps/spack/spack-repo/share/spack/setup-env.sh
-        spack env activate lfric-gnu
-        export FC="gfortran"
-        export CC="gcc"
-        export CXX="g++"
-        export FPP="cpp -traditional-cpp"
-        export LDMPI="mpif90"
-        export FFLAGS="-I$(spack location -e lfric-gnu)/.spack-env/view/include"
-        export LDFLAGS="-L$(spack location -e lfric-gnu)/.spack-env/view/lib"
-        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(spack location -e lfric-gnu)/.spack-env/view/lib"
-        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH::$(spack location -e lfric-gnu)/.spack-env/view/lib64"
+        source /apps/spack/spack-upstream/share/spack/setup-env.sh
+        spack load lfric-buildenv%gcc
         source .runner_venv/bin/activate
         export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
-        export MINIAPP_DIR=${HOME}/LFRic_${LFRIC_REV}/miniapps/gungho_model
-        export PSYCLONE_CONFIG=${HOME}/lfric_psyclone_config.cfg
-        cd ${MINIAPP_DIR}
-        # Prepare script (LFRics expects a folder with a global.py)
-        mkdir -p psyclone-test
-        cp ${PSYCLONE_LFRIC_DIR}/everything_everywhere_all_at_once.py psyclone-test/global.py
-        # Compile
-        make clean
-        make OPTIMISATION_PATH=${MINIAPP_DIR}/psyclone-test -j 6 build |& tee compiler_output.txt
+        export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
+        export LFRIC_DIR=${HOME}/LFRic/lfric_apps
+        export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test
+        cd ${LFRIC_DIR}
+        # Psyclone scripts must now be under 'optimisation' and be called 'global.py'
+        mkdir -p applications/gungho_model/optimisation/psyclone-test
+        cp ${PSYCLONE_LFRIC_DIR}/everything_everywhere_all_at_once.py ${OPT_DIR}/global.py
+        # Clean previous version and compile again
+        rm -rf appplications/gungho_model/working
+        ./build/local_build.py -a gungho_model -p psyclone-test
         # Run
-        cd example
+        cd applications/gungho_model/example
         cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its.nml configuration.nml
         export OMP_NUM_THREADS=6
         mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt
@@ -151,6 +143,15 @@ jobs:
         export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
         export VAR_HALOS=$(grep "gungho_model" halo_calls_counter.txt | cut -d'|' -f5)
         echo $GITHUB_REF_NAME $GITHUB_SHA $VAR_TIME $VAR_HALOS >> ${HOME}/store_results/lfric_optimised_performance_history
+        ${HOME}/mongosh-2.1.1-linux-x64/bin/mongosh \
+          "mongodb+srv://cluster0.x8ncpxi.mongodb.net/PerformanceMonitoring" \
+          --quiet --apiVersion 1 --username ${{ secrets.MONGODB_USERNAME }} \
+          --password ${{ secrets.MONGODB_PASSWORD }} \
+          --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'",
+          github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'",
+          ci_test: "LFRic all transformations", lfric_version: '"$LFRIC_APPS_REV"', omp_threads: '"$OMP_NUM_THREADS"',
+          system: "GlaDos", compiler:"spack-gfortran-11", date: new Date(), elapsed_time: '"$VAR_TIME"',
+          num_of_halo_exchanges: '"$VAR_HALOS"'})'
 
     - name: Upload LFRic optimised results
       uses: exuanbo/actions-deploy-gist@v1

diff --git a/.github/workflows/nemo_tests.yml b/.github/workflows/nemo_tests.yml
@@ -115,8 +115,16 @@ jobs:
         tail -n 1 output.txt | grep -q "|U|_max:  0.458515"
         tail -n 1 output.txt | grep -q "S_min:  0.482686"
         tail -n 1 output.txt | grep -q "S_max:  0.407622"
-        export VAR_TIME=$(grep -A 1 "Elapsed Time" output.txt | head -n 2 | tail -n 1)
+        export VAR_TIME=$(grep -A 1 "Elapsed Time" output.txt | head -n 2 | tail -n 1 | awk '{print $1}')
         echo $GITHUB_REF_NAME $GITHUB_SHA $VAR_TIME >> ${HOME}/store_results/performance_history_openmp_gpu
+        ${HOME}/mongosh-2.1.1-linux-x64/bin/mongosh \
+          "mongodb+srv://cluster0.x8ncpxi.mongodb.net/PerformanceMonitoring" \
+          --quiet --apiVersion 1 --username ${{ secrets.MONGODB_USERNAME }} \
+          --password ${{ secrets.MONGODB_PASSWORD }} \
+          --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'",
+          github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'",
+          ci_test: "NEMO OpenMP for GPU", nemo_version: "NEMO MO 4.0.2", system: "GlaDos",
+          compiler:"nvhpc-'"$NVFORTRAN_VERSION"'", date: new Date(), elapsed_time: '"$VAR_TIME"'})'
 
     - name: Upload NEMO OpenMP GPU results
       uses: exuanbo/actions-deploy-gist@v1
@@ -145,9 +153,16 @@ jobs:
         tail -n 1 output.txt | grep -q "|U|_max:  0.458515"
         tail -n 1 output.txt | grep -q "S_min:  0.482686"
         tail -n 1 output.txt | grep -q "S_max:  0.407622"
-        export VAR_TIME=$(grep -A 1 "Elapsed Time" output.txt | head -n 2 | tail -n 1)
+        export VAR_TIME=$(grep -A 1 "Elapsed Time" output.txt | head -n 2 | tail -n 1 | awk '{print $1}')
         echo $GITHUB_REF_NAME $GITHUB_SHA $VAR_TIME >> ${HOME}/store_results/performance_history_openacc_kernels_gpu
-        # Upload the results into a Github Gist
+        ${HOME}/mongosh-2.1.1-linux-x64/bin/mongosh \
+          "mongodb+srv://cluster0.x8ncpxi.mongodb.net/PerformanceMonitoring" \
+          --quiet --apiVersion 1 --username ${{ secrets.MONGODB_USERNAME }} \
+          --password ${{ secrets.MONGODB_PASSWORD }} \
+          --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'",
+          github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'",
+          ci_test: "NEMO OpenACC kernels for GPU", nemo_version: "MEMO MO 4.0.2", system: "GlaDos",
+          compiler:"nvhpc-'"$NVFORTRAN_VERSION"'", date: new Date(), elapsed_time: '"$VAR_TIME"'})'
 
     - name: Upload NEMO OpenACC kernels GPU results
       uses: exuanbo/actions-deploy-gist@v1
@@ -176,9 +191,16 @@ jobs:
         tail -n 1 output.txt | grep -q "|U|_max:  0.458515"
         tail -n 1 output.txt | grep -q "S_min:  0.482686"
         tail -n 1 output.txt | grep -q "S_max:  0.407622"
-        export VAR_TIME=$(grep -A 1 "Elapsed Time" output.txt | head -n 2 | tail -n 1)
+        export VAR_TIME=$(grep -A 1 "Elapsed Time" output.txt | head -n 2 | tail -n 1 | awk '{print $1}')
         echo $GITHUB_REF_NAME $GITHUB_SHA $VAR_TIME >> ${HOME}/store_results/performance_history_openacc_loops_gpu
-        # Upload the results into a Github Gist
+        ${HOME}/mongosh-2.1.1-linux-x64/bin/mongosh \
+          "mongodb+srv://cluster0.x8ncpxi.mongodb.net/PerformanceMonitoring" \
+          --quiet --apiVersion 1 --username ${{ secrets.MONGODB_USERNAME }} \
+          --password ${{ secrets.MONGODB_PASSWORD }} \
+          --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'",
+          github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'",
+          ci_test: "NEMO OpenACC loops for GPU", nemo_version: "MEMO MO 4.0.2", system: "GlaDos",
+          compiler:"nvhpc-'"$NVFORTRAN_VERSION"'", date: new Date(), elapsed_time: '"$VAR_TIME"'})'
 
     - name: Upload NEMO OpenACC loops GPU results
       uses: exuanbo/actions-deploy-gist@v1
@@ -215,8 +237,17 @@ jobs:
         tail -n 1 output.txt | grep -q "|U|_max:  0.148409"
         tail -n 1 output.txt | grep -q "S_min:  0.108530"
         tail -n 1 output.txt | grep -q "S_max:  0.404045"
-        export VAR_TIME=$(grep -A 1 "Elapsed Time" output.txt | head -n 2 | tail -n 1)
+        export VAR_TIME=$(grep -A 1 "Elapsed Time" output.txt | head -n 2 | tail -n 1 | awk '{print $1}')
         echo  $GITHUB_REF_NAME $GITHUB_SHA $VAR_TIME >> ${HOME}/store_results/performance_history_openmp_cpu
+        ${HOME}/mongosh-2.1.1-linux-x64/bin/mongosh \
+          "mongodb+srv://cluster0.x8ncpxi.mongodb.net/PerformanceMonitoring" \
+          --quiet --apiVersion 1 --username ${{ secrets.MONGODB_USERNAME }} \
+          --password ${{ secrets.MONGODB_PASSWORD }} \
+          --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'",
+          github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'",
+          ci_test: "NEMO OpenMP for CPU", nemo_version: "NEMO ECMWF 4.0 V40 with mpp", system: "GlaDos",
+          compiler:"intel-'"$ONEAPI_VERSION"'" , date: new Date(),
+          num_omp_threads: '"$OMP_NUM_THREADS"', elapsed_time: '"$VAR_TIME"'})'
 
     - name: Upload NEMO OpenMP CPU results
       uses: exuanbo/actions-deploy-gist@v1

diff --git a/changelog b/changelog
@@ -83,7 +83,10 @@
 	variables from other modules.
 
 	31) PR #2547 towards #1779. Removes the NemoLoop node and makes
-  	the loop_type inference rules part of generic PSyIR.
+	the loop_type inference rules part of generic PSyIR.
+
+	32) PR #2590 fpr #2589. Moves to the latest LFRic (split repo)
+	and fixes the LFRic integration tests.
 
 release 2.5.0 14th of February 2024
 

diff --git a/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its.nml b/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its.nml
@@ -84,7 +84,7 @@ stretching_method='linear',
   v0             = 0.0
   sbr_angle_lat  = 0.0
   sbr_angle_lon  = 0.0
-  smp_init_wind=.false.
+  smp_init_wind  = .false.
 /
 
 &initialization
@@ -200,6 +200,7 @@ stretching_method='linear',
   dep_pt_stencil_extent = 3
   inner_order = 2
   outer_order = 2
+  vertical_order = 2
   calculate_detj = 'upwind'
 /
 
@@ -219,12 +220,12 @@ stretching_method='linear',
 /
 
 &departure_points
-  departure_grid      = 'uniform'
   horizontal_limit    = 'none'
-  method              = 'euler'
+  horizontal_method   = 'euler'
   n_dep_pt_iterations = 1
   vertical_limit      = 'exponential'
   vertical_method     = 'euler'
+  vertical_sorting    = .false.
 /
 
 &transport
@@ -245,7 +246,9 @@ stretching_method='linear',
   log_space = .false., .false., .false.
   enforce_min_value=.false.,.false.,.false.
   max_vert_cfl_calc='dep_point'
+  min_val_method='iterative'
   min_value=0.0,0.0,-99999999.0,0.0
+  consistent_ffsl_splitting=3*1
   cfl_mol_1d_stab = 1.0
   cfl_mol_2d_stab = 1.0
   cfl_mol_3d_stab = 1.0
@@ -262,6 +265,7 @@ stretching_method='linear',
   runge_kutta_method = 'ssp5'
   min_val_abs_tol=-1.0e-12
   min_val_max_iterations=10
+  theta_dispersion_correction=.false.
   theta_variable='dry'
   use_density_predictor=.true.
   transport_ageofair=.false.