Merge pull request #953 from xylar/add_dev_suite

Add a developer test suite that uses an existing conda env

suite/run_dev_suite.bash

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+set -e
+
+env_name=mpas_dev
+
+conda_base=$(dirname $(dirname $CONDA_EXE))
+source $conda_base/etc/profile.d/conda.sh
+source $conda_base/etc/profile.d/mamba.sh
+
+export HDF5_USE_FILE_LOCKING=FALSE
+
+branch=$(git symbolic-ref --short HEAD)
+
+# test building the docs
+mamba activate ${env_name}
+cd docs
+make clean
+make html
+cd ..
+
+machine=$(python -c "from mache import discover_machine; print(discover_machine())")
+
+py=3.11
+./suite/setup.py -p ${py} -r main_py${py} -b ${branch} --copy_docs --clean -e ${env_name}
+./suite/setup.py -p ${py} -r wc_defaults -b ${branch} --no_polar_regions -e ${env_name}
+./suite/setup.py -p ${py} -r no_ncclimo -b ${branch} -e ${env_name}
+./suite/setup.py -p ${py} -r ctrl -b ${branch} -e ${env_name}
+./suite/setup.py -p ${py} -r main_vs_ctrl -b ${branch} -e ${env_name}
+./suite/setup.py -p ${py} -r no_polar_regions -b ${branch} --no_polar_regions -e ${env_name}
+./suite/setup.py -p ${py} -r mesh_rename -b ${branch} -e ${env_name}
+./suite/setup.py -p ${py} -r QU480 -b ${branch} -e ${env_name}
+
+# submit the jobs
+cd ${machine}_test_suite
+
+main_py=3.11
+cd main_py${main_py}
+echo main_py${main_py}
+RES=$(sbatch job_script.bash)
+cd ..
+
+cd main_vs_ctrl
+echo main_vs_ctrl
+sbatch --dependency=afterok:${RES##* } --kill-on-invalid-dep=yes job_script.bash
+cd ..
+
+for run in wc_defaults no_ncclimo no_polar_regions \
+    mesh_rename QU480
+do
+    cd ${run}
+    echo ${run}
+    sbatch job_script.bash
+    cd ..
+done
+
+cd ..
+

suite/run_e3sm_unified_suite.bash

@@ -27,7 +27,7 @@ cd ..
 
 cd main_vs_ctrl
 echo main_vs_ctrl
-sbatch --dependency=afterok:${RES##* } job_script.bash --kill-on-invalid-dep=yes
+sbatch --dependency=afterok:${RES##* } --kill-on-invalid-dep=yes job_script.bash
 cd ..
 
 for run in wc_defaults no_ncclimo no_polar_regions mesh_rename

suite/run_suite.bash

@@ -72,11 +72,11 @@ cd ..
 
 cd main_vs_ctrl
 echo main_vs_ctrl
-sbatch --dependency=afterok:${RES##* } job_script.bash --kill-on-invalid-dep=yes
+sbatch --dependency=afterok:${RES##* } --kill-on-invalid-dep=yes job_script.bash
 cd ..
 
 for run in main_py${alt_py} wc_defaults no_ncclimo no_polar_regions \
-    mesh_rename xarray_main
+    mesh_rename xarray_main QU480
 do
     cd ${run}
     echo ${run}
@@ -85,15 +85,3 @@ do
 done
 
 cd ..
-
-# only LCRC machines have a separate QU480 run
-if [[ "$machine" == "anvil" || "$machine" == "chrysalis" ]]
-then
-   py=${main_py}
-   conda activate test_mpas_analysis_py${py}
-  ./suite/setup.py -p ${py} -r QU480 -b ${branch}
-  cd ${machine}_test_suite/QU480
-  echo QU480
-  sbatch job_script.bash
-  cd ../..
-fi

suite/setup.py

@@ -50,32 +50,22 @@ def main():
     web_section = machine_info.config['web_portal']
     web_base = os.path.join(web_section['base_path'], web_section['username'])
     html_base = f'{web_base}/analysis_testing'
-    if machine in ['anvil', 'chrysalis']:
-        input_base = '/lcrc/group/e3sm/ac.xylar/acme_scratch/anvil'
-        output_base = f'/lcrc/group/e3sm/{username}/analysis_testing'
-        if args.run == 'QU480':
-            simulation = '20200305.A_WCYCL1850.ne4_oQU480.anvil'
-            mesh = 'QU480'
-        else:
-            simulation = '20201025.GMPAS-IAF.T62_oQU240wLI.anvil'
-            mesh = 'oQU240wLI'
-    elif machine == 'cori-haswell':
-        input_base = '/global/cfs/cdirs/e3sm/xylar'
-        scratch = os.environ['CSCRATCH']
-        output_base = f'{scratch}/analysis_testing'
+    if args.run == 'QU480':
         simulation = '20200305.A_WCYCL1850.ne4_oQU480.anvil'
         mesh = 'QU480'
+    else:
+        simulation = '20230406.GMPAS-IAF-ISMF.T62_oQU240wLI.chrysalis'
+        mesh = 'oQU240wLI'
+    if machine in ['anvil', 'chrysalis']:
+        input_base = '/lcrc/group/e3sm/public_html/diagnostics/mpas_analysis/example_simulations'
+        output_base = f'/lcrc/group/e3sm/{username}/analysis_testing'
     elif machine == 'pm-cpu':
-        input_base = '/global/cfs/cdirs/e3sm/xylar'
+        input_base = '/global/cfs/cdirs/e3sm/diagnostics/mpas_analysis/example_simulations'
         scratch = os.environ['SCRATCH']
         output_base = f'{scratch}/analysis_testing'
-        simulation = '20200305.A_WCYCL1850.ne4_oQU480.anvil'
-        mesh = 'QU480'
     elif machine == 'compy':
-        input_base = '/compyfs/asay932/analysis_testing/test_output'
+        input_base = '/compyfs/diagnostics/mpas_analysis/example_simulations'
         output_base = f'/compyfs/{username}/analysis_testing'
-        simulation = '20200305.A_WCYCL1850.ne4_oQU480.anvil'
-        mesh = 'QU480'
     else:
         raise ValueError(f'Machine {machine} is not set up for the test suite '
                          f'yet.')
@@ -111,7 +101,7 @@ def main():
     elif mesh == 'oQU240wLI':
         generate = "['all', 'no_BGC', 'no_icebergs', 'no_index', 'no_eke', " \
                    "'no_waves']"
-        end_year = '8'
+        end_year = '10'
     else:
         raise ValueError(f'Unexpected mesh: {mesh}')