bring back energy conservation

format amip plots debug amip instability addit rebase clean rm tmp output buildkite spec stop extra ts update display amip plot flip axis ncep plots format ncep sfc data fix automate ncep plot date generalize plot function format reinit conservation check last Conflicts: experiments/AMIP/moist_mpi_earth/coupler_driver.jl experiments/AMIP/moist_mpi_earth/coupler_utils/amip_visualizer.jl experiments/AMIP/moist_mpi_earth/coupler_utils/ncep_visualizer.jl experiments/AMIP/moist_mpi_earth/coupler_utils/variable_definer.jl mpi-enabled fix Conflicts: experiments/AMIP/moist_mpi_earth/Manifest.toml experiments/AMIP/moist_mpi_earth/cli_options.jl bring back energy conservation format amip plots debug amip instability addit rebase clean rm tmp output buildkite spec stop extra ts update display amip plot flip axis ncep plots format ncep sfc data fix automate ncep plot date generalize plot function format reinit conservation check last Conflicts: experiments/AMIP/moist_mpi_earth/coupler_driver.jl experiments/AMIP/moist_mpi_earth/coupler_utils/amip_visualizer.jl experiments/AMIP/moist_mpi_earth/coupler_utils/ncep_visualizer.jl experiments/AMIP/moist_mpi_earth/coupler_utils/variable_definer.jl mpi-enabled fix Conflicts: experiments/AMIP/moist_mpi_earth/Manifest.toml experiments/AMIP/moist_mpi_earth/cli_options.jl bring back energy conservation format amip plots debug amip instability addit rebase clean rm tmp output buildkite spec stop extra ts update display amip plot flip axis ncep plots format ncep sfc data fix automate ncep plot date generalize plot function format reinit conservation check last Conflicts: experiments/AMIP/moist_mpi_earth/coupler_driver.jl experiments/AMIP/moist_mpi_earth/coupler_utils/amip_visualizer.jl experiments/AMIP/moist_mpi_earth/coupler_utils/ncep_visualizer.jl experiments/AMIP/moist_mpi_earth/coupler_utils/variable_definer.jl mpi-enabled fix name fix mpi race - wip wip Co-authored-by: Sriharsha Kandala <sriharsha.kvs@gmail.com> Co-authored-by: Simone Byrne masks ok but NaNs (wip) Cd Ch vals rebased and MPI tested (pending physical validation) REGRID_DIR for times unstable at 4d - unrelared to mpi (wip) mesh fix re-add regridding for postprocessor (AMIP runs) Co-authored-by: Zhaoyi Shen <szy21@users.noreply.github.com> Co-authored-by: Valeria Barra <valeriabarra@users.noreply.github.com> re-add regridding for postprocessor (AMIP runs) Co-authored-by: Zhaoyi Shen <szy21@users.noreply.github.com> Co-authored-by: Valeria Barra <valeriabarra@users.noreply.github.com> format buildkite race fix format format mpi fix unit test unit test MPI race MPI race in bc reader format format delete file format save output from single core amip update unit test call buildkite artifacts long high res run pipelines format format
CliMA · Oct 25, 2022 · 5007428 · 5007428
1 parent 26395cb
commit 5007428
Show file tree

Hide file tree

Showing 12 changed files with 56 additions and 56 deletions.
diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml
@@ -35,8 +35,8 @@ steps:
     steps:
 
       - label: "AMIP"
-        command: "mpiexec julia --project=experiments/AMIP/moist_mpi_earth/ experiments/AMIP/moist_mpi_earth/coupler_driver.jl --energy_check=false --mode_name=amip --anim=true --t_end=100days --dt_save_to_sol=1days --dt_cpl=200 --mono_surface=false --z_elem 45 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --dt 400secs --t_end 100days--z_elem 45 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --dt 400secs --t_end 100days --dt_save_restart=10days --job_id longrun_bw_rhoe_highres"
-        artifact_paths: "longrun_bw_rhoe_highres/*"
+        command: "mpiexec julia --project=experiments/AMIP/moist_mpi_earth/ experiments/AMIP/moist_mpi_earth/coupler_driver.jl --surface_scheme monin_obukhov --moist equil --rad clearsky --microphy 0M --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt_rad 1hours --idealized_insolation true --FLOAT_TYPE Float64 --vert_diff true --energy_check=false --mode_name=amip --t_end=100days --dt_save_to_sol=10days --mono_surface=false  --run_name=current_amip_n32_longrun"
+        artifact_paths: "experiments/AMIP/moist_mpi_earth/output/amip/current_amip_n32_longrun_artifacts/*"
         env:
           CLIMACORE_DISTRIBUTED: "MPI"
         agents:

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -21,7 +21,7 @@ steps:
     command:
       # - "echo $$JULIA_DEPOT_PATH"
       - echo "--- Instantiate package env"
-      - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project -e 'sing Pkg; Pkg.instantiate(;verbose=true)'"
       - "julia --project -e 'using Pkg; Pkg.precompile()'"
       - "julia --project -e 'using Pkg; Pkg.status()'"
 
@@ -65,3 +65,12 @@ steps:
           CLIMACORE_DISTRIBUTED: "MPI"
         agents:
           slurm_ntasks: 2
+
+      - label: "MPI AMIP FINE"
+        key: "mpi_amip_fine"
+        command: "mpiexec julia --project=experiments/AMIP/moist_mpi_earth/ experiments/AMIP/moist_mpi_earth/coupler_driver.jl --surface_scheme monin_obukhov --moist equil --rad allskywithclear --microphy 0M --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt_rad 1hours --idealized_insolation false --FLOAT_TYPE Float64 --vert_diff true --energy_check=false --mode_name=amip --t_end=5days --dt_save_to_sol=10days --mono_surface=false  --run_name=target_amip_n32_shortrun"
+        artifact_paths: "experiments/AMIP/moist_mpi_earth/output/amip/target_amip_n32_shortrun_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+        agents:
+          slurm_ntasks: 32
diff --git a/.gitignore b/.gitignore
@@ -38,6 +38,8 @@ docs/src/generated/
 *.pvtu
 *.nc
 *.jld2
+*.g
+*.hdf5
 
 # Julia System Images
 *.so

diff --git a/experiments/AMIP/moist_mpi_earth/bucket/bucket_init.jl b/experiments/AMIP/moist_mpi_earth/bucket/bucket_init.jl
@@ -152,7 +152,7 @@ function bucket_init(
     α_snow = FT(0.8) # snow albedo
     albedo = BulkAlbedo{FT}(α_snow, α_soil)
     σS_c = FT(0.2)
-    W_f = FT(0.15)
+    W_f = FT(0.5)
     d_soil = FT(3.5) # soil depth
     z_0m = FT(1e-2)
     z_0b = FT(1e-3)

diff --git a/experiments/AMIP/moist_mpi_earth/cli_options.jl b/experiments/AMIP/moist_mpi_earth/cli_options.jl
@@ -240,7 +240,7 @@ function parse_commandline()
         "--start_date"
         help = "Start date of the simulation"
         arg_type = String
-        default = "19790101"
+        default = "19790321"
         "--topography"
         help = "Define the surface elevation profile [`NoWarp`,`Earth`,`DCMIP200`]"
         arg_type = String
@@ -374,3 +374,6 @@ function time_to_seconds(s::String)
     end
     error("Uncaught case in computing time from given string.")
 end
+
+import Base.joinpath
+Base.joinpath(s::String, ::Nothing) = s
diff --git a/experiments/AMIP/moist_mpi_earth/coupler_driver.jl b/experiments/AMIP/moist_mpi_earth/coupler_driver.jl
@@ -27,28 +27,23 @@ t_end = FT(time_to_seconds(parsed_args["t_end"]))
 tspan = (0, t_end)
 Δt_cpl = FT(parsed_args["dt_cpl"])
 saveat = time_to_seconds(parsed_args["dt_save_to_sol"])
-date0 = date = DateTime(1979, 01, 01)
+date0 = date = DateTime(parsed_args["start_date"], dateformat"yyyymmdd")
 mono_surface = parsed_args["mono_surface"]
 
 # overwrite some parsed args
 parsed_args["coupled"] = true
 parsed_args["dt"] = string(Δt_cpl) * "secs"
 parsed_args["enable_threading"] = true
-parsed_args["microphy"] = "0M"
-parsed_args["forcing"] = nothing
-parsed_args["idealized_h2o"] = false
-parsed_args["vert_diff"] = true
-parsed_args["rad"] = "gray"
-parsed_args["hyperdiff"] = true
 parsed_args["config"] = "sphere"
-parsed_args["moist"] = "equil"
 
 import ClimaCoupler
 pkg_dir = pkgdir(ClimaCoupler)
 COUPLER_OUTPUT_DIR = joinpath(pkg_dir, "experiments/AMIP/moist_mpi_earth/output", joinpath(mode_name, run_name))
 !isdir(COUPLER_OUTPUT_DIR) && mkpath(COUPLER_OUTPUT_DIR)
 REGRID_DIR = joinpath(COUPLER_OUTPUT_DIR, "regrid_tmp/")
 mkpath(REGRID_DIR)
+@info COUPLER_OUTPUT_DIR
+@info parsed_args
 
 # get the paths to the necessary data files - land sea mask, sst map, sea ice concentration
 include("artifacts.jl")
@@ -88,7 +83,7 @@ land_sim =
 
 @info mode_name
 if mode_name == "amip"
-    println("AMIP boundary conditions - do not expect energy conservation")
+    @info "AMIP boundary conditions - do not expect energy conservation"
 
     # 2. ocean
     SST_info = bcfile_info_init(
@@ -129,7 +124,7 @@ if mode_name == "amip"
     )
     update_midmonth_data!(date0, SIC_info)
     SIC_init = interpolate_midmonth_to_daily(date0, SIC_info)
-    ice_mask = get_ice_mask.(SIC_init, mono_surface)
+    ice_mask = get_ice_mask.(SIC_init, mono_surface) .* FT(0)
     ice_sim = ice_init(FT; tspan = tspan, dt = Δt_cpl, space = boundary_space, saveat = saveat, ice_mask = ice_mask)
     mode_specifics = (; name = mode_name, SST_info = SST_info, SIC_info = SIC_info)
 

diff --git a/experiments/AMIP/moist_mpi_earth/coupler_utils/masker.jl b/experiments/AMIP/moist_mpi_earth/coupler_utils/masker.jl
@@ -71,29 +71,6 @@ function combine_surfaces!(combined_field::Fields.Field, masks::NamedTuple, fiel
 end
 nans_to_zero(v) = isnan(v) ? FT(0) : v
 
-"""
-    update_masks(cs)
-
-Updates dynamically changing masks. 
-"""
-function update_masks(cs)
-
-    # dynamic masks
-    ice_d = cs.model_sims.ice_sim.integrator.p.ice_mask
-    FT = eltype(ice_d)
-
-    # static mask
-    land_s = cs.surface_masks.land
-
-    cs.surface_masks.ice .= min.(ice_d .+ land_s, FT(1)) .- land_s
-    cs.surface_masks.ocean .= (FT(1) .- cs.surface_masks.ice .- land_s)
-
-    @assert minimum(cs.surface_masks.ice) >= FT(0)
-    @assert minimum(cs.surface_masks.land) >= FT(0)
-    @assert minimum(cs.surface_masks.ocean) >= FT(0)
-
-end
-
 """
     time_slice_ncfile(sic_data, time_idx = 1)
 - slices a dataset at time index `time_idx` and saves it under `sic_data_slice`. Used for more efficient regridding of mask, SST and SIC files. 

diff --git a/experiments/AMIP/moist_mpi_earth/coupler_utils/unit_tester.jl b/experiments/AMIP/moist_mpi_earth/coupler_utils/unit_tester.jl
@@ -119,6 +119,7 @@ function test_monthly_accumulation()
     # read Feb data
     hdfreader = InputOutput.HDF5Reader(joinpath(TEST_DATA_DIR, "test_data.monthly_1979-02-01T00:00:00.hdf5"))
     hdf5_data = InputOutput.read_field(hdfreader, "test_data")
+    close(hdfreader)
     @test parent(hdf5_data)[1] ≈ 1 # no hours in March 
 
 end

diff --git a/experiments/AMIP/moist_mpi_earth/push_pull.jl b/experiments/AMIP/moist_mpi_earth/push_pull.jl
@@ -9,8 +9,7 @@ function atmos_push!(cs)
     dummmy_remap!(csf.F_A, .-atmos_sim.integrator.p.dif_flux_energy_bc)
     dummmy_remap!(csf.F_E, .-atmos_sim.integrator.p.dif_flux_ρq_tot_bc)
     dummmy_remap!(csf.P_liq, atmos_sim.integrator.p.col_integrated_rain .+ atmos_sim.integrator.p.col_integrated_snow)
-    cs.parsed_args["rad"] == "gray" ? dummmy_remap!(csf.F_R, level(atmos_sim.integrator.p.ᶠradiation_flux, half)) :
-    nothing
+    dummmy_remap!(csf.F_R, level(atmos_sim.integrator.p.ᶠradiation_flux, half))
 end
 
 """
@@ -25,11 +24,15 @@ function land_pull!(cs)
     land_sim = cs.model_sims.land_sim
     csf = cs.fields
     FT = cs.FT
+    land_mask = cs.surface_masks.land
     parent(land_sim.integrator.p.bucket.ρ_sfc) .= parent(csf.ρ_sfc)
-    parent(land_sim.integrator.p.bucket.turbulent_energy_flux) .= parent(csf.F_A)
+    parent(land_sim.integrator.p.bucket.turbulent_energy_flux) .=
+        apply_mask.(parent(land_mask), >, parent(csf.F_A), parent(csf.F_A) .* FT(0), FT(0))
     ρ_liq = (LSMP.ρ_cloud_liq(land_sim.params.earth_param_set))
-    parent(land_sim.integrator.p.bucket.evaporation) .= parent(csf.F_E) ./ ρ_liq
-    parent(land_sim.integrator.p.bucket.R_n) .= parent(csf.F_R)
+    parent(land_sim.integrator.p.bucket.evaporation) .=
+        apply_mask.(parent(land_mask), >, parent(csf.F_E) ./ ρ_liq, parent(csf.F_E) .* FT(0), FT(0))
+    parent(land_sim.integrator.p.bucket.R_n) .=
+        apply_mask.(parent(land_mask), >, parent(csf.F_R), parent(csf.F_R) .* FT(0), FT(0))
     parent(land_sim.integrator.p.bucket.P_liq) .= FT(-1.0) .* parent(csf.P_liq) # land expects this to be positive
     parent(land_sim.integrator.p.bucket.P_snow) .= FT(0.0) .* parent(csf.P_snow)
 
@@ -58,8 +61,11 @@ sublimating. That contribution has been zeroed out in the atmos fluxes.
 function ice_pull!(cs)
     ice_sim = cs.model_sims.ice_sim
     csf = cs.fields
-    @. ice_sim.integrator.p.F_aero = csf.F_A
-    @. ice_sim.integrator.p.F_rad = csf.F_R
+    ice_mask = cs.surface_masks.ice
+    parent(ice_sim.integrator.p.F_rad) .=
+        apply_mask.(parent(ice_mask), >, parent(csf.F_R), parent(csf.F_R) .* FT(0), FT(0))
+    parent(ice_sim.integrator.p.F_aero) .=
+        apply_mask.(parent(ice_mask), >, parent(csf.F_A), parent(csf.F_A) .* FT(0), FT(0))
 end
 
 """
@@ -118,7 +124,8 @@ function atmos_pull!(cs)
 
     # surface specific humidity
     ocean_q_sfc = TD.q_vap_saturation_generic.(thermo_params, T_ocean, ρ_sfc_cpl, TD.Liquid())
-    sea_ice_q_sfc = TD.q_vap_saturation_generic.(thermo_params, T_ice, ρ_sfc_cpl, TD.Ice())
+    sea_ice_q_sfc =
+        swap_space!(Spaces.level(atmos_sim.integrator.u.c.ρq_tot ./ atmos_sim.integrator.u.c.ρ, 1), boundary_space) #ρ_sfc_cpl .* FT(0) #D.q_vap_saturation_generic.(thermo_params, T_ice, ρ_sfc_cpl, TD.Ice())
     land_q_sfc = get_land_q(land_sim, atmos_sim, T_land, ρ_sfc_cpl)
     combine_surfaces!(combined_field, cs.surface_masks, (; land = land_q_sfc, ocean = ocean_q_sfc, ice = sea_ice_q_sfc))
     dummmy_remap!(q_sfc_cpl, combined_field)
@@ -144,10 +151,10 @@ function atmos_pull!(cs)
     parent(atmos_sim.integrator.p.T_sfc) .= parent(T_sfc_cpl)
     parent(atmos_sim.integrator.p.ρ_sfc) .= parent(ρ_sfc_cpl)
     parent(atmos_sim.integrator.p.q_sfc) .= parent(q_sfc_cpl)
-    if :z0b in propertynames(integrator.p.surface_scheme)
-        parent(atmos_sim.integrator.p.z0b) .= parent(z0b_cpl)
-        parent(atmos_sim.integrator.p.z0m) .= parent(z0m_cpl)
-    end
+    # if :z0b in propertynames(integrator.p.surface_scheme)
+    #     parent(atmos_sim.integrator.p.z0b) .= parent(z0b_cpl)
+    #     parent(atmos_sim.integrator.p.z0m) .= parent(z0m_cpl)
+    # end
 
     calculate_surface_fluxes_atmos_grid!(atmos_sim.integrator, info_sfc)
 

diff --git a/experiments/AMIP/moist_mpi_earth/sbatch_job.sh b/experiments/AMIP/moist_mpi_earth/sbatch_job.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
-#SBATCH --ntasks=2          
-#SBATCH --time=10:00:00     # walltime
+#SBATCH --ntasks=32          
+#SBATCH --time=30:00:00     # walltime
 
 set -euo pipefail # kill the job if anything fails
 set -x # echo script
@@ -16,6 +16,6 @@ export JULIA_HDF5_PATH=""
 julia --project -e 'using Pkg; Pkg.instantiate(); Pkg.build()'
 julia --project -e 'using Pkg; Pkg.build("MPI"); Pkg.build("HDF5")'
 
-mpiexec julia --project coupler_driver.jl --energy_check=false --mode_name=amip --anim=true --t_end=32days --dt_save_to_sol=1days --dt_cpl=200 --mono_surface=false --h_elem=6 --dt_save_restart=5days
+mpiexec julia --project coupler_driver.jl --surface_scheme monin_obukhov --moist equil --rad clearsky --microphy 0M --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt_rad 1hours --idealized_insolation true --FLOAT_TYPE Float64 --vert_diff true --energy_check false --mode_name amip --t_end 360days --dt_save_to_sol 10days --mono_surface false --dt_save_to_disk 1days --run_name monin_spg10_fixmsk_csky_iinsol_vereq3_dztop_Wf0p5
 
 # goal: mpiexec julia --project=examples examples/hybrid/driver.jl --surface_scheme monin_obukhov --moist equil --rad allskywithclear --microphy 0M --z_elem 45 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --rayleigh_sponge true --alpha_rayleigh_uh 0 --dt 150secs --dt_rad 1hours --idealized_insolation false --t_end 365days --job_id longrun_aquaplanet_rhoe_equil_highres_allsky --dt_save_to_disk 10days --FLOAT_TYPE Float64
diff --git a/experiments/AMIP/moist_mpi_earth/slab_ice/slab_init.jl b/experiments/AMIP/moist_mpi_earth/slab_ice/slab_init.jl
@@ -32,7 +32,7 @@ function ice_rhs!(du, u, p, t)
 
     F_conductive = @. params.k_ice / (params.h) * (params.T_base - Y.T_sfc)
     rhs = @. (-F_aero - F_rad + F_conductive) / (params.h * params.ρ * params.c)
-    parent(dY.T_sfc) .= parent(rhs) # apply_mask.(FT, parent(ice_mask), parent(rhs))
+    parent(dY.T_sfc) .= parent(rhs) # apply_mask.(parent(ice_mask), >, parent(rhs), parent(rhs) .* FT(0), FT(0) )
 end
 
 """
@@ -50,7 +50,7 @@ function ice_init(::Type{FT}; tspan, saveat, dt, space, ice_mask, stepper = Eule
         FT(1e-3),
         FT(1e-5),
         FT(273.15),
-        FT(0.0),# k_ice
+        FT(2.0),# k_ice
         FT(0.38), # albedo
     )
 
@@ -74,3 +74,9 @@ clean_sic(SIC, _info) = swap_space!(SIC, axes(_info.land_mask)) ./ _info.FT(100.
 
 # setting that SIC < 0.5 os counted as ocean if binary remapping of landsea mask. 
 get_ice_mask(h_ice::FT, mono, threshold = 0.5) where {FT} = mono ? h_ice : binary_mask.(h_ice, threshold = threshold)
+
+"""
+apply_mask(mask, condition, yes, no, value = 0.5) 
+- apply mask mased on a threshold value in the mask
+"""
+apply_mask(mask, condition, yes, no, value) = condition(mask, value) ? yes : no
diff --git a/experiments/AMIP/moist_mpi_earth/slab_ocean/slab_init.jl b/experiments/AMIP/moist_mpi_earth/slab_ocean/slab_init.jl
@@ -41,7 +41,7 @@ function slab_ocean_rhs!(dY, Y, cache, t)
     p, F_aero, F_rad, ocean_mask = cache
     FT = eltype(Y.T_sfc)
     rhs = @. -(F_aero + F_rad) / (p.h * p.ρ * p.c)
-    parent(dY.T_sfc) .= parent(rhs) # apply_mask.(FT, parent(ocean_mask), parent(rhs))
+    parent(dY.T_sfc) .= parent(rhs) .* parent(ocean_mask)# apply_mask.(FT, parent(ocean_mask), parent(rhs))
 end