WWM for deterministic & use separate compiled schism for WWM and no WWM

docker/pyschism/docker/setup_ensemble.py

@@ -26,7 +26,6 @@
 from pyschism.forcing import NWM
 from ensembleperturbation.perturbation.atcf import perturb_tracks
 
-
 import wwm
 
 
@@ -175,7 +174,7 @@ def main(args):
         'forcings': forcing_configurations,
         'perturbations': perturbations,
         'platform': platform,
-        'schism_executable': 'pschism_PAHM_TVD-VL'
+#        'schism_executable': 'pschism_PAHM_TVD-VL'
     }
 
     run_configuration = SCHISMRunConfiguration(
@@ -197,7 +196,7 @@ def main(args):
     })
 
     if use_wwm:
-        wwm.setup_wwm(mesh_file, workdir)
+        wwm.setup_wwm(mesh_file, workdir, ensemble=True)
 
 
 def parse_arguments():

docker/pyschism/docker/setup_model.py

@@ -28,6 +28,8 @@
 from pyschism.mesh.fgrid import ManningsN
 from pyschism.stations import Stations
 
+import wwm
+
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
@@ -404,6 +406,8 @@ def setup_schism_model(
         pass
     ## end of workaround
 
+    if use_wwm:
+        wwm.setup_wwm(mesh_file, schism_dir, ensemble=False)
 
     logger.info("Setup done")
 
@@ -424,6 +428,7 @@ def main(args):
     )
     tpxo_dir = EFS_MOUNT_POINT / args.tpxo_dir
     nwm_dir = EFS_MOUNT_POINT / args.nwm_dir
+    use_wwm = args.use_wwm
 
     if TPXO_LINK_PATH.is_dir():
         shutil.rmtree(TPXO_LINK_PATH)
@@ -508,6 +513,10 @@ def main(args):
         type=pathlib.Path
     )
 
+    argument_parser.add_argument(
+        "--use-wwm", action="store_true"
+    )
+
     parser.add_argument(
         "name", help="name of the storm", type=str)
 

docker/pyschism/docker/wwm.py

@@ -13,16 +13,19 @@
 
 REFS = Path('~').expanduser() / 'app/refs'
 
-def setup_wwm(mesh_file: Path, ensemble_dir: Path):
+def setup_wwm(mesh_file: Path, setup_dir: Path, ensemble: bool):
     '''Output is
         - hgrid_WWM.gr3
         - param.nml
         - wwmbnd.gr3
         - wwminput.nml
     '''
 
-    spinup_dir = ensemble_dir/'spinup'
-    runs_dir = ensemble_dir.glob('runs/*')
+
+    runs_dir = [setup_dir]
+    if ensemble:
+        spinup_dir = setup_dir/'spinup'
+        runs_dir = setup_dir.glob('runs/*')
 
     schism_grid = Gr3.open(mesh_file, crs=4326)
     wwm_grid = break_quads(schism_grid)

docker/schism/docker/Dockerfile

@@ -39,6 +39,17 @@ RUN \
     mkdir -p schism/build && \
     PREV_PWD=$PWD && \
     cd schism/build && \
+    cmake ../src/ \
+        -DCMAKE_Fortran_COMPILER=mpifort \
+        -DCMAKE_C_COMPILER=mpicc \
+        -DNetCDF_Fortran_LIBRARY=$(nc-config --libdir)/libnetcdff.so \
+        -DNetCDF_C_LIBRARY=$(nc-config --libdir)/libnetcdf.so \
+        -DNetCDF_INCLUDE_DIR=$(nc-config --includedir) \
+        -DUSE_PAHM=TRUE \
+        -DCMAKE_Fortran_FLAGS_RELEASE="-O2 -ffree-line-length-none -fallow-argument-mismatch" && \
+    make -j8 && \
+    mv bin/* -t /usr/bin/ && \
+    rm -rf * && \
     cmake ../src/ \ 
         -DCMAKE_Fortran_COMPILER=mpifort \
         -DCMAKE_C_COMPILER=mpicc \

docker/schism/docker/entrypoint.sh

@@ -13,7 +13,7 @@ date
 set -ex
 
 mkdir -p outputs
-mpirun -np $SCHISM_NPROCS pschism_WWM_PAHM_TVD-VL 4
+mpirun -np $SCHISM_NPROCS $2 4
 
 
 echo "Combining outputs..."

prefect/workflow/flows/jobs/ecs.py

@@ -26,6 +26,7 @@
     param_ensemble_sample_rule,
     param_past_forecast,
     param_wind_coupling,
+    param_schism_exec,
 )
 from tasks.infra import ContainerInstance, task_add_ecs_attribute_for_ec2
 from tasks.jobs import (
@@ -38,16 +39,18 @@
     task_format_kill_timedout,
     task_check_docker_success)
 from tasks.utils import (
-        ECSTaskDetail,
-        task_check_param_true,
-        task_pylist_from_jsonlist,
-        task_get_run_tag,
-        task_get_flow_run_id,
-        task_bundle_params,
-        task_replace_tag_in_template,
-        task_convert_str_to_path,
-        task_return_value_if_param_true,
-        task_return_value_if_param_false)
+    ECSTaskDetail,
+    task_check_param_true,
+    task_pylist_from_jsonlist,
+    task_get_run_tag,
+    task_get_flow_run_id,
+    task_bundle_params,
+    task_replace_tag_in_template,
+    task_convert_str_to_path,
+    task_return_value_if_param_true,
+    task_return_value_if_param_false,
+    task_return_this_if_param_true_else_that
+)
 from flows.utils import LocalAWSFlow, flow_dependency, task_create_ecsrun_config
 
 
@@ -262,22 +265,19 @@ def _task_recurse():
                 True,
                 "nwm/NWM_v2.0_channel_hydrofabric/nwm_v2_0_hydrofabric.gdb"
             ),
-            _use_if_and(
-                param_ensemble, True,
-                param_wind_coupling, True,
-                value="--use-wwm"
-            ),
             # Common arguments
             "--date-range-file",
             _tag('hurricanes/{tag}/setup/dates.csv'),
             "--tpxo-dir", 'tpxo',
+            _use_if(param_wind_coupling, True, "--use-wwm"),
             param_storm_name, param_storm_year],
         "setup",
         60, 180, ["CDSAPI_URL", "CDSAPI_KEY"]),
     "schism-run-aws-single": ECSTaskDetail(
         SCHISM_CLUSTER, SCHISM_TEMPLATE_ID, "odssm-solve", "solve", [
-            param_schism_dir
-        ],
+            param_schism_dir,
+            param_schism_exec
+            ],
         "SCHISM",
         60, 240, []),
     "viz-sta-html-aws": ECSTaskDetail(
@@ -432,7 +432,12 @@ def make_flow_solve_ecs_task(child_flow):
                         name=param_storm_name,
                         year=param_storm_year,
                         run_id=param_run_id,
-                        schism_dir=rundir
+                        schism_dir=rundir,
+                        schism_exec=task_return_this_if_param_true_else_that(
+                            param_wind_coupling,
+                            'pschism_WWM_PAHM_TVD-VL',
+                            'pschism_PAHM_TVD-VL',
+                        )
                     )
                 )
             )
@@ -462,7 +467,8 @@ def make_flow_solve_ecs_task(child_flow):
                         name=param_storm_name,
                         year=param_storm_year,
                         run_id=param_run_id,
-                        schism_dir=result_ensemble_dir + '/spinup'
+                        schism_dir=result_ensemble_dir + '/spinup',
+                        schism_exec='pschism_PAHM_TVD-VL',
                     ),
                     run_config=ecs_config,
                 )
@@ -478,6 +484,13 @@ def make_flow_solve_ecs_task(child_flow):
                         name=unmapped(param_storm_name),
                         year=unmapped(param_storm_year),
                         run_id=unmapped(param_run_id),
+                        schism_exec=unmapped(
+                            task_return_this_if_param_true_else_that(
+                                param_wind_coupling,
+                                'pschism_WWM_PAHM_TVD-VL',
+                                'pschism_PAHM_TVD-VL',
+                            )
+                        ),
                         schism_dir=_task_pathlist_to_strlist(
                             hotstart_dirs, rel_to='/efs'
                         )

prefect/workflow/flows/jobs/pw.py

@@ -5,13 +5,15 @@
 
 from conf import PW_S3, PW_S3_PREFIX
 from tasks.params import (
-        param_storm_name, param_storm_year, param_run_id,
-        param_subset_mesh, param_ensemble,
-        param_mesh_hmax,
-        param_mesh_hmin_low, param_mesh_rate_low,
-        param_mesh_trans_elev,
-        param_mesh_hmin_high, param_mesh_rate_high,
-        param_use_rdhpcs_post)
+    param_storm_name, param_storm_year, param_run_id,
+    param_subset_mesh, param_ensemble,
+    param_mesh_hmax,
+    param_mesh_hmin_low, param_mesh_rate_low,
+    param_mesh_trans_elev,
+    param_mesh_hmin_high, param_mesh_rate_high,
+    param_use_rdhpcs_post,
+    param_wind_coupling,
+)
 from tasks.jobs import (
     task_submit_slurm,
     task_format_mesh_slurm,
@@ -33,12 +35,14 @@
     task_start_rdhpcs_cluster,
     task_stop_rdhpcs_cluster)
 from tasks.utils import (
-        task_check_param_true,
-        task_bundle_params, task_get_run_tag,
-        task_replace_tag_in_template,
-        task_convert_str_to_path,
-        task_return_value_if_param_true,
-        task_return_value_if_param_false)
+    task_check_param_true,
+    task_bundle_params, task_get_run_tag,
+    task_replace_tag_in_template,
+    task_convert_str_to_path,
+    task_return_value_if_param_true,
+    task_return_value_if_param_false,
+    task_return_this_if_param_true_else_that,
+)
 from flows.utils import (
         LocalPWFlow, RDHPCSMeshFlow, RDHPCSSolveFlow, flow_dependency)
 
@@ -164,7 +168,7 @@ def make_flow_mesh_rdhpcs(mesh_pw_task_flow):
                         mesh_cutoff=param_mesh_trans_elev,
                         mesh_hmin_high=param_mesh_hmin_high,
                         mesh_rate_high=param_mesh_rate_high,
-                        subset_mesh=param_subset_mesh
+                        subset_mesh=param_subset_mesh,
                 )
         )
 
@@ -216,6 +220,11 @@ def make_flow_solve_rdhpcs_pw_task():
             result_after_single_run = task_submit_slurm(
                 command=task_format_schism_slurm(
                     run_path=result_rundir,
+                    schism_exec=task_return_this_if_param_true_else_that(
+                        param_wind_coupling,
+                        'pschism_WWM_PAHM_TVD-VL',
+                        'pschism_PAHM_TVD-VL',
+                    ),
                     upstream_tasks=[result_s3_to_lustre]))
 
             result_wait_slurm_done_1 = task_wait_slurm_done(
@@ -231,6 +240,7 @@ def make_flow_solve_rdhpcs_pw_task():
             result_after_coldstart = task_submit_slurm(
                 command=task_format_schism_slurm(
                     run_path=result_ensemble_dir + '/spinup',
+                    schism_exec='pschism_PAHM_TVD-VL',
                     upstream_tasks=[result_s3_to_lustre]))
             result_wait_slurm_done_spinup = task_wait_slurm_done(
                 job_id=result_after_coldstart)
@@ -246,6 +256,11 @@ def make_flow_solve_rdhpcs_pw_task():
                 command=task_format_schism_slurm.map(
                     run_path=_task_pathlist_to_strlist(
                         hotstart_dirs, rel_to='/lustre'),
+                    schism_exec=unmapped(task_return_this_if_param_true_else_that(
+                        param_wind_coupling,
+                        'pschism_WWM_PAHM_TVD-VL',
+                        'pschism_PAHM_TVD-VL',
+                    )),
                     upstream_tasks=[unmapped(result_wait_slurm_done_spinup)]))
             result_wait_slurm_done_2 = task_wait_slurm_done.map(
                 job_id=result_after_hotstart)
@@ -313,7 +328,8 @@ def make_flow_solve_rdhpcs(solve_pw_task_flow):
                         name=param_storm_name,
                         year=param_storm_year,
                         run_id=param_run_id,
-                        ensemble=param_ensemble
+                        ensemble=param_ensemble,
+                        couple_wind=param_wind_coupling,
                     )
                 )
 

prefect/workflow/main.py

@@ -119,6 +119,7 @@ def _make_workflow():
                 ensemble=param_ensemble,
                 hr_before_landfall=param_hr_prelandfall,
                 past_forecast=param_past_forecast,
+                couple_wind=param_wind_coupling,
                 )
 
         result_bundle_params_2 = task_bundle_params(

prefect/workflow/tasks/jobs.py

@@ -270,7 +270,7 @@ def task_wait_slurm_done(job_id):
     name="Prepare Slurm script to submit the batch job",
     template=" ".join(
         ["sbatch",
-         "--export=ALL,STORM_PATH=\"{run_path}\"",
+         "--export=ALL,STORM_PATH=\"{run_path}\",SCHISM_EXEC=\"{schism_exec}\"",
          "~/schism.sbatch"]
     )
 )

prefect/workflow/tasks/params.py

@@ -8,6 +8,7 @@
 param_use_parametric_wind = Parameter('parametric_wind', default=False)
 param_run_id = Parameter('run_id')
 param_schism_dir = Parameter('schism_dir')
+param_schism_exec = Parameter('schism_exec')
 param_subset_mesh = Parameter('subset_mesh', default=False)
 param_past_forecast = Parameter('past_forecast', default=False)
 param_hr_prelandfall = Parameter('hr_before_landfall', default=-1)

rdhpcs/scripts/compile_schism.sh

@@ -63,7 +63,6 @@ function _compile {
         -DNetCDF_FORTRAN_DIR=$NetCDF_FORTRAN_DIR \
         -DTVD_LIM=$TVD_LIM \
         -DUSE_PAHM=TRUE \
-        -DUSE_WWM=TRUE \
         -DCMAKE_C_FLAGS="-no-multibyte-chars" \
         -DCMAKE_CXX_FLAGS="-no-multibyte-chars"
 
@@ -73,6 +72,27 @@ function _compile {
     mkdir -p $install_dir
     cp -L -r bin/ $install_dir
 
+    rm -rf *
+    cmake ../src \
+        -DCMAKE_Fortran_COMPILER=$CMAKE_Fortran_COMPILER \
+        -DCMAKE_CXX_COMPILER=$CMAKE_CXX_COMPILER \
+        -DCMAKE_C_COMPILER=$CMAKE_C_COMPILER \
+        -DMPI_HEADER_PATH=$MPI_HEADER_PATH \
+        -DNetCDF_C_DIR=$NetCDF_C_DIR \
+        -DNetCDF_INCLUDE_DIR=$NetCDF_INCLUDE_DIR \
+        -DNetCDF_LIBRARIES=$NetCDF_LIBRARIES \
+        -DNetCDF_FORTRAN_DIR=$NetCDF_FORTRAN_DIR \
+        -DTVD_LIM=$TVD_LIM \
+        -DUSE_PAHM=TRUE \
+        -DUSE_WWM=TRUE \
+        -DCMAKE_C_FLAGS="-no-multibyte-chars" \
+        -DCMAKE_CXX_FLAGS="-no-multibyte-chars"
+
+    #gnu make
+    make -j 6
+
+    cp -L -r bin/ $install_dir
+
     if [ -f $link_path ]; then
         rm $link_path
     fi

rdhpcs/scripts/schism.sbatch

@@ -33,7 +33,7 @@ set -ex
 pushd /lustre/${STORM_PATH}
 mkdir -p outputs
 #srun --mpi=pmi2 pschism_TVD-VL 4
-mpirun --ppn ${SLURM_TASKS_PER_NODE} pschism_WWM_PAHM_TVD-VL 4
+mpirun --ppn ${SLURM_TASKS_PER_NODE} ${SCHISM_EXEC} 4
 
 if [ $? -eq 0 ]; then
     echo "Combining outputs..."