Skip to content

Commit

Permalink
Added the new WWM outputs in scribe; re-ordered them to make coding
Browse files Browse the repository at this point in the history 
easier.
Added a new routine in scribe_io to facilitate recv & write for 3D
variables.
  • Loading branch information
@josephzhang8
josephzhang8 committed Apr 28, 2022
1 parent 3906b0c commit 567b612
Show file tree
Hide file tree
Showing 6 changed files with 272 additions and 67 deletions.
4 changes: 2 additions & 2 deletions cmake/SCHISM.local.build
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ set (TVD_LIM VL CACHE STRING "Flux limiter")
#Turn OLDIO off to use the new scribe based I/O
set (OLDIO OFF CACHE BOOLEAN "Old nc output (each rank dumps its own data)")

set (PREC_EVAP ON CACHE BOOLEAN "Include precipitation and evaporation calculation")
set (PREC_EVAP OFF CACHE BOOLEAN "Include precipitation and evaporation calculation")
set (USE_BULK_FAIRALL OFF CACHE BOOLEAN "Enable Fairall bulk scheme for air-sea exchange")
##Older versions of GOTM (3.*) have issues with netcdf v4, so are not maintained
set (USE_GOTM OFF CACHE BOOLEAN "Use GOTM turbulence model. This just enables the build -- GOTM must still be selected in param.nml")
Expand All @@ -29,7 +29,7 @@ set( USE_PAHM OFF CACHE BOOLEAN "Use PaHM module")

# Enable/Disable Modules
set( USE_SED2D OFF CACHE BOOLEAN "Use 2D sediment module")
set( USE_WWM OFF CACHE BOOLEAN "Use wind-wave module")
set( USE_WWM ON CACHE BOOLEAN "Use wind-wave module")
##Coupling to WW3, either via ESMF or hard coupling (in dev)
set( USE_WW3 OFF CACHE BOOLEAN "Use Wave Watch III")
set( USE_ICE OFF CACHE BOOLEAN "Use 1-class ICE module")
Expand Down
28 changes: 15 additions & 13 deletions sample_inputs/param.nml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -798,8 +798,8 @@
iof_hydro(22) = 0 !eddy viscosity [m^2/s] {viscosity} 3D
iof_hydro(23) = 0 !turbulent kinetic energy {turbulentKineticEner} 3D
iof_hydro(24) = 0 !turbulent mixing length [m] {mixingLength} 3D
! iof_hydro(25) = 1 !z-coord {zCoordinates} 3D - this flag will be reset by code to be always on
iof_hydro(26) = 0 !horizontal vel vector [m/s] {horizontalVelX,Y} 3D vector
iof_hydro(25) = 1 !z-coord {zCoordinates} 3D - this flag should be on for visIT etc
iof_hydro(26) = 1 !horizontal vel vector [m/s] {horizontalVelX,Y} 3D vector
iof_hydro(27) = 0 !horizontal vel vector defined @side [m/s] {horizontalSideVelX,Y} 3D vector
iof_hydro(28) = 0 !vertical vel. @elem [m/s] {verticalVelAtElement} 3D
iof_hydro(29) = 0 !T @prism centers [C] {temperatureAtElement} 3D
Expand Down Expand Up @@ -843,17 +843,19 @@
! iof_wwm(24) = 0 !Friction velocity (m/s?) {frictionalVelocity} 2D
! iof_wwm(25) = 0 !Charnock coefficient {CharnockCoeff} 2D
! iof_wwm(26) = 0 !Rougness length {rougnessLength} 2D
! iof_wwm(27) = 0 !WWM_energy vector {waveEnergyDirX,Y} 2D vector
! iof_wwm(28) = 0 !Wave force vector (m.s-2) computed by wwm @side centers and whole levels {waveForceX,Y} 2D vector

! iof_wwm(29) = 0 !Horizontal Stokes velocity (m.s-1) @nodes and whole levels {stokes_hvel}
! iof_wwm(30) = 0 !Vertical Stokes velocity (m.s-1) @nodes and whole levels {stokes_wvel}
! iof_wwm(31) = 0 !Roller contribution to horizontal Stokes velocity (m.s-1) @nodes and whole levels {roller_stokes_hvel}
! iof_wwm(32) = 0 !Roller energy dissipation rate (W/m²) @nodes {Drol}
! iof_wwm(33) = 0 !Total wave energy dissipation rate by depth-induced breaking (W/m²) @nodes {wave_sbrtot}
! iof_wwm(34) = 0 !Total wave energy dissipation rate by bottom friction (W/m²) @nodes {wave_sbftot}
! iof_wwm(35) = 0 !Total wave energy dissipation rate by whitecapping (W/m²) @nodes {wave_sdstot}
! iof_wwm(36) = 0 !Total wave energy input rate from atmospheric forcing (W/m²) @nodes {wave_sintot}

! iof_wwm(27) = 0 !Roller energy dissipation rate (W/m²) @nodes {Drol} 2D
! iof_wwm(28) = 0 !Total wave energy dissipation rate by depth-induced breaking (W/m²) @nodes {wave_sbrtot} 2D
! iof_wwm(29) = 0 !Total wave energy dissipation rate by bottom friction (W/m²) @nodes {wave_sbftot} 2D
! iof_wwm(30) = 0 !Total wave energy dissipation rate by whitecapping (W/m²) @nodes {wave_sdstot} 2D
! iof_wwm(31) = 0 !Total wave energy input rate from atmospheric forcing (W/m²) @nodes {wave_sintot} 2D
! iof_wwm(32) = 0 !WWM_energy vector {waveEnergyDirX,Y} 2D vector

! iof_wwm(33) = 0 !Vertical Stokes velocity (m.s-1) @sides and whole levels {stokes_wvel} 3D
! iof_wwm(34) = 0 !Wave force vector (m.s-2) computed by wwm @side centers and whole levels {waveForceX,Y} 3D vector

! iof_wwm(35) = 0 !Horizontal Stokes velocity (m.s-1) @nodes and whole levels {stokes_hvel} 3D vector
! iof_wwm(36) = 0 !Roller contribution to horizontal Stokes velocity (m.s-1) @nodes and whole levels {roller_stokes_hvel} 3D vector

!-----------------------------------------------------------------------
! Tracer module outputs. In most cases, actual # of outputs depends on # of tracers used
Expand Down
148 changes: 124 additions & 24 deletions src/Core/scribe_io.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ module scribe_io
integer,save :: ntrs(natrm),iof_hydro(40),iof_wwm(40),iof_icm(210),iof_cos(20),iof_fib(5), &
&iof_sed2d(14),iof_ice(10),iof_ana(20),iof_marsh(2),counter_out_name
real(rkind), save :: dt,h0
character(len=20), save :: varname3,out_name(500)
character(len=20), save :: out_name(500)
integer, save :: iout_23d(500)
character(len=1000),save :: out_dir

Expand Down Expand Up @@ -328,6 +328,7 @@ subroutine scribe_step(it)
integer,intent(in) :: it

integer :: i,j,k,m,rrqst,ierr,irank,itotal,icount_out_name,itmp5
character(len=20) :: varname3

! Return if not output step
if(nc_out==0.or.mod(it,nspool)/=0) return
Expand Down Expand Up @@ -385,29 +386,56 @@ subroutine scribe_step(it)
!------------------
!3D node: hydro
do j=17,25
if(iof_hydro(j)/=0) then
itotal=itotal+1
icount_out_name=icount_out_name+1 !index into out_name
irank=nproc_schism-itotal
if(myrank_schism==irank) then
!OK to fill partial arrays as long as respect column major
do i=1,nproc_compute
call mpi_irecv(var3dnode(:,:,i),np(i)*nvrt,MPI_REAL4,i-1,200+itotal,comm_schism,rrqst2(i),ierr)
enddo !i
call mpi_waitall(nproc_compute,rrqst2,MPI_STATUSES_IGNORE,ierr)

do i=1,nproc_compute
var3dnode_gb(1:nvrt,iplg(1:np(i),i))=var3dnode(1:nvrt,1:np(i),i)
enddo !i
varname3=out_name(icount_out_name)
call nc_writeout3D(1,it,nvrt,np_global,var3dnode_gb,varname3,iout_23d(icount_out_name))
endif !myrank_schism
endif !iof_hydro
if(iof_hydro(j)/=0) call scribe_recv_write(it,1,1,itotal,icount_out_name)

! itotal=itotal+1
! icount_out_name=icount_out_name+1 !index into out_name
! irank=nproc_schism-itotal
! if(myrank_schism==irank) then
! !OK to fill partial arrays as long as respect column major
! do i=1,nproc_compute
! call mpi_irecv(var3dnode(:,:,i),np(i)*nvrt,MPI_REAL4,i-1,200+itotal,comm_schism,rrqst2(i),ierr)
! enddo !i
! call mpi_waitall(nproc_compute,rrqst2,MPI_STATUSES_IGNORE,ierr)
!
! do i=1,nproc_compute
! var3dnode_gb(1:nvrt,iplg(1:np(i),i))=var3dnode(1:nvrt,1:np(i),i)
! enddo !i
! varname3=out_name(icount_out_name)
! call nc_writeout3D(1,it,nvrt,np_global,var3dnode_gb,varname3,iout_23d(icount_out_name))
! endif !myrank_schism
! endif !iof_hydro
enddo !j

!3D node vectors
do j=26,26
if(iof_hydro(j)/=0) then
if(iof_hydro(j)/=0) call scribe_recv_write(it,1,2,itotal,icount_out_name)
! do m=1,2 !components
! itotal=itotal+1
! icount_out_name=icount_out_name+1
! irank=nproc_schism-itotal
! if(myrank_schism==irank) then
! do i=1,nproc_compute
! call mpi_irecv(var3dnode(:,:,i),np(i)*nvrt,MPI_REAL4,i-1,200+itotal,comm_schism,rrqst2(i),ierr)
! enddo !i
! call mpi_waitall(nproc_compute,rrqst2,MPI_STATUSES_IGNORE,ierr)
!
! do i=1,nproc_compute
! var3dnode_gb(1:nvrt,iplg(1:np(i),i))=var3dnode(1:nvrt,1:np(i),i)
! enddo !i
!
! varname3=out_name(icount_out_name)
! call nc_writeout3D(1,it,nvrt,np_global,var3dnode_gb,varname3,iout_23d(icount_out_name))
! endif !myrank_schism
! enddo !m
! endif !iof_hydro
enddo !j

!Add modules
#ifdef USE_WWM
!Vectors
do j=35,36
if(iof_wwm(j)/=0) then
do m=1,2 !components
itotal=itotal+1
icount_out_name=icount_out_name+1
Expand All @@ -417,7 +445,7 @@ subroutine scribe_step(it)
call mpi_irecv(var3dnode(:,:,i),np(i)*nvrt,MPI_REAL4,i-1,200+itotal,comm_schism,rrqst2(i),ierr)
enddo !i
call mpi_waitall(nproc_compute,rrqst2,MPI_STATUSES_IGNORE,ierr)

do i=1,nproc_compute
var3dnode_gb(1:nvrt,iplg(1:np(i),i))=var3dnode(1:nvrt,1:np(i),i)
enddo !i
Expand All @@ -426,10 +454,10 @@ subroutine scribe_step(it)
call nc_writeout3D(1,it,nvrt,np_global,var3dnode_gb,varname3,iout_23d(icount_out_name))
endif !myrank_schism
enddo !m
endif !iof_hydro
endif !iof_wwm
enddo !j
#endif /*USE_WWM*/

!Add modules
#ifdef USE_GEN
do j=1,ntrs(3)
if(iof_gen(j)==1) then
Expand Down Expand Up @@ -648,7 +676,26 @@ subroutine scribe_step(it)

!Add modules
#ifdef USE_WWM
if(iof_wwm(28)/=0) then
if(iof_wwm(33)/=0) then
itotal=itotal+1
icount_out_name=icount_out_name+1
irank=nproc_schism-itotal
if(myrank_schism==irank) then
do i=1,nproc_compute
call mpi_irecv(var3dside(:,:,i),ns(i)*nvrt,MPI_REAL4,i-1,200+itotal,comm_schism,rrqst2(i),ierr)
enddo !i
call mpi_waitall(nproc_compute,rrqst2,MPI_STATUSES_IGNORE,ierr)
do i=1,nproc_compute
var3dside_gb(1:nvrt,islg(1:ns(i),i))=var3dside(1:nvrt,1:ns(i),i)
enddo !i

varname3=out_name(icount_out_name)
call nc_writeout3D(3,it,nvrt,ns_global,var3dside_gb,varname3,iout_23d(icount_out_name))
endif !myrank_schism
endif !iof_wwm

!Vector
if(iof_wwm(34)/=0) then
do m=1,2 !vector components
itotal=itotal+1
icount_out_name=icount_out_name+1 !index into out_name
Expand Down Expand Up @@ -971,6 +1018,59 @@ subroutine nc_writeout3D(imode,it,idim1,idim2,var3d_gb2,vname,i23d)

end subroutine nc_writeout3D

subroutine scribe_recv_write(it,imode,ivs,itotal,icount_out_name)
implicit none
!imode: 1/2/3 for node/elem/side; ivs: 1 (scalar) or 2 (vector)
integer, intent(in) :: it,imode,ivs
integer, intent(inout) :: itotal,icount_out_name !global counters

character(len=20) :: varname3
integer :: i,j,m,irank,ierr

if(imode<1.or.imode>3) call parallel_abort('scribe_recv_write: imode')
if(ivs/=1.and.ivs/=2) call parallel_abort('scribe_recv_write: ivs')

do m=1,ivs !components
itotal=itotal+1
icount_out_name=icount_out_name+1
irank=nproc_schism-itotal
if(myrank_schism==irank) then
!OK to fill partial arrays as long as respect column major
do i=1,nproc_compute
if(imode==1) then !node
call mpi_irecv(var3dnode(:,:,i),np(i)*nvrt,MPI_REAL4,i-1,200+itotal,comm_schism,rrqst2(i),ierr)
else if(imode==2) then !elem
call mpi_irecv(var3delem(:,:,i),ne(i)*nvrt,MPI_REAL4,i-1,200+itotal,comm_schism,rrqst2(i),ierr)
else !side
call mpi_irecv(var3dside(:,:,i),ns(i)*nvrt,MPI_REAL4,i-1,200+itotal,comm_schism,rrqst2(i),ierr)
endif !imode
enddo !i
call mpi_waitall(nproc_compute,rrqst2,MPI_STATUSES_IGNORE,ierr)

!Combine in memory
do i=1,nproc_compute
if(imode==1) then !node
var3dnode_gb(1:nvrt,iplg(1:np(i),i))=var3dnode(1:nvrt,1:np(i),i)
else if(imode==2) then !elem
var3delem_gb(1:nvrt,ielg(1:ne(i),i))=var3delem(1:nvrt,1:ne(i),i)
else !side
var3dside_gb(1:nvrt,islg(1:ns(i),i))=var3dside(1:nvrt,1:ns(i),i)
endif !imode
enddo !i

varname3=out_name(icount_out_name)
if(imode==1) then !node
call nc_writeout3D(1,it,nvrt,np_global,var3dnode_gb,varname3,iout_23d(icount_out_name))
else if(imode==2) then !elem
call nc_writeout3D(2,it,nvrt,ne_global,var3delem_gb,varname3,iout_23d(icount_out_name))
else !side
call nc_writeout3D(3,it,nvrt,ns_global,var3dside_gb,varname3,iout_23d(icount_out_name))
endif !imode
endif !myrank_schism
enddo !m

end subroutine scribe_recv_write

!===============================================================================
subroutine scribe_finalize
implicit none
Expand Down
52 changes: 48 additions & 4 deletions src/Hydro/schism_init.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -499,8 +499,8 @@ subroutine schism_init(iorder,indir,iths,ntime)
read(15,nml=SCHOUT)
close(15)

!zcor should always be there
iof_hydro(25)=1
!zcor should be on usually
! iof_hydro(25)=1

!... Dump param.nml for checking
if(myrank==0) then
Expand Down Expand Up @@ -6348,8 +6348,27 @@ subroutine schism_init(iorder,indir,iths,ntime)
endif !iof_wwm
enddo !i

do i=27,31
if(iof_wwm(i)/=0) then
ncount_2dnode=ncount_2dnode+1
iout_23d(ncount_2dnode)=1
select case(i)
case(27)
out_name(ncount_2dnode)='rollerDissRate'
case(28)
out_name(ncount_2dnode)='dissRateDepBreaking'
case(29)
out_name(ncount_2dnode)='dissRateBottFriction'
case(30)
out_name(ncount_2dnode)='dissRateWhiteCapping'
case(31)
out_name(ncount_2dnode)='energyInputAtmos'
end select
endif !iof_wwm
enddo !i

!Vectors count as 2
if(iof_wwm(27)/=0) then
if(iof_wwm(32)/=0) then
ncount_2dnode=ncount_2dnode+2
iout_23d(ncount_2dnode-1:ncount_2dnode)=1
out_name(ncount_2dnode-1)='waveEnergyDirX'
Expand Down Expand Up @@ -6583,6 +6602,23 @@ subroutine schism_init(iorder,indir,iths,ntime)
out_name(counter_out_name)='horizontalVelY'
endif

#ifdef USE_WWM
do i=35,36
if(iof_wwm(i)/=0) then
ncount_3dnode=ncount_3dnode+2
counter_out_name=counter_out_name+2
iout_23d(counter_out_name-1:counter_out_name)=2
if(i==35) then
out_name(counter_out_name-1)='stokesDriftVelX'
out_name(counter_out_name)='stokesDriftVelY'
else
out_name(counter_out_name-1)='rollStokesDriftVelX'
out_name(counter_out_name)='rollStokesDriftVelY'
endif
endif !iof_wwm
enddo !i
#endif /*USE_WWM*/

#ifdef USE_GEN
do i=1,ntrs(3)
if(iof_gen(i)==1) then
Expand Down Expand Up @@ -6710,7 +6746,15 @@ subroutine schism_init(iorder,indir,iths,ntime)
enddo !i

#ifdef USE_WWM
if(iof_wwm(28)/=0) then
if(iof_wwm(33)/=0) then
ncount_3dside=ncount_3dside+1
counter_out_name=counter_out_name+1
iout_23d(counter_out_name)=8
out_name(counter_out_name)='verticalStokesVel'
endif

!Vector
if(iof_wwm(34)/=0) then
ncount_3dside=ncount_3dside+2
counter_out_name=counter_out_name+2
iout_23d(counter_out_name-1:counter_out_name)=8
Expand Down
Loading

0 comments on commit 567b612

Please sign in to comment.