Filtering of the offline ensemble members, and more ... (NOAA-EMC#539)

* filter offline ensemble

* fixed py norm

* point to public oops

* update for oops 2258, point ufo to dev

* reverted ufo branch and soca obs list

* wrong date in yaml

CMakeLists.txt

@@ -94,7 +94,7 @@ if(BUILD_GDASBUNDLE)
   endif()
   ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop )
   ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" BRANCH main-ecbuild RECURSIVE )
-  ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop )
+  ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/change_mask_value )
 
   # Build JEDI/DA or other peripherals
   ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" )

parm/soca/berror/saber_blocks.yaml

@@ -27,21 +27,22 @@ components:
 
       - linear variable change name: BkgErrSOCA
         read_from_file: 3
-        basename: ./
-        ocn_filename: 'ocn.bkgerr_stddev.incr.{{ATM_WINDOW_BEGIN}}.nc'
-        ice_filename: 'ice.bkgerr_stddev.incr.{{ATM_WINDOW_BEGIN}}.nc'
-        remap_filename: ./INPUT/MOM.res.nc
-        date: '{{ATM_WINDOW_BEGIN}}'
+        basename: ./static_ens/
+        ocn_filename: 'ocn.orig_ens_stddev.incr.{{ATM_WINDOW_BEGIN}}.nc'
+        ice_filename: 'ice.orig_ens_stddev.incr.{{ATM_WINDOW_BEGIN}}.nc'
+        #remap_filename: ./INPUT/MOM.res.nc
+        date: '{{ATM_WINDOW_MIDDLE}}'
         t_min: 0.1
-        t_max: 5.0
-        s_min: 0.0
-        s_max: 0.0
+        t_max: 10.0
+        s_min: 0.1
+        s_max: 10.0
         ssh_min: 0.0 # std ssh=0 => ssh balance applied as
         ssh_max: 0.0 #              strong constraint
         cicen_min: 0.1
         cicen_max: 0.5
         hicen_min: 0.0
         hicen_max: 0.0
+        standard deviation: true
 
       - linear variable change name: BalanceSOCA
         ksshts:
@@ -53,34 +54,29 @@ components:
     members from template:
       template:
         read_from_file: 1
-        date: '{{ATM_WINDOW_BEGIN}}'
+        date: '{{ATM_WINDOW_MIDDLE}}'
         basename: ./static_ens/
-        ocn_filename: ocn.%mem%.nc
-        ice_filename: ice.%mem%.nc
-        remap_filename: ./INPUT/MOM.res.nc
-        state variables: [tocn, socn, ssh, uocn, vocn, hocn, cicen, hicen, hsnon, mld, layer_depth]
+        ocn_filename: 'ocn.filtered.%mem%.incr.{{ATM_WINDOW_BEGIN}}.nc'
+        ice_filename: 'ice.filtered.%mem%.incr.{{ATM_WINDOW_BEGIN}}.nc'
+        state variables: [tocn, socn, ssh, uocn, vocn, cicen, hicen, hsnon]
       pattern: '%mem%'
       nmembers: ${CLIM_ENS_SIZE}
     localization:
       localization method: SABER
       saber central block:
         saber block name: BUMP_NICAS
-        active variables: [tocn, socn, ssh, uocn, vocn, cicen, hicen, hsnon]
         read:
           io:
             data directory: bump
-            #files prefix: soca_bump
           drivers:
-            multivariate strategy: crossed
+            multivariate strategy: duplicated
             read local nicas: true
-          grids:
-          - model:
-              variables: [tocn, socn, uocn, vocn]
-#            io:
-#              files prefix: bump
-          - model:
-              variables: [ssh, cicen, hicen, hsnon]
-#            io:
-#              files prefix: bump
+          model:
+            do not cross mask boundaries: false
+
   weight:
-    value: 1.0
+    read_from_file: 3
+    basename: ./
+    ocn_filename: 'ocn.ens_weights.incr.{{ATM_WINDOW_MIDDLE}}.nc'
+    ice_filename: 'ice.ens_weights.incr.{{ATM_WINDOW_MIDDLE}}.nc'
+    date: '{{ATM_WINDOW_MIDDLE}}'

parm/soca/berror/soca_apply_steric.yaml

@@ -0,0 +1,43 @@
+input geometry:
+  geom_grid_file: soca_gridspec.nc
+  mom6_input_nml: mom_input.nml
+  fields metadata: fields_metadata.yaml
+
+output geometry:
+  geom_grid_file: soca_gridspec.nc
+  mom6_input_nml: mom_input.nml
+  fields metadata: fields_metadata.yaml
+
+linear variable change:
+  input variables: [tocn, socn, ssh, uocn, vocn, hocn, cicen, hicen, hsnon]
+  output variables: [tocn, socn, ssh, uocn, vocn, hocn, cicen, hicen, hsnon]
+  do inverse: false
+  linear variable changes:
+  - linear variable change name: BkgErrFILT
+    ocean_depth_min: 500  # zero where ocean is shallower than 500m
+    rescale_bkgerr: 0.3   # rescale perturbation
+    efold_z: 1500.0       # Apply exponential decay
+  - linear variable change name: BalanceSOCA  # linear steric height from (T,S) perturbation
+
+increments:
+- date: '{{ATM_WINDOW_BEGIN}}'
+  input variables: [tocn, socn, ssh, uocn, vocn, hocn, cicen, hicen, hsnon]
+  input:
+     read_from_file: 1
+     basename: ./static_ens/
+     ocn_filename: 'ocn.bal.ens.MEMNUM.{{ATM_WINDOW_BEGIN}}.PT0S.nc'
+     ice_filename: 'ice.bal.ens.MEMNUM.{{ATM_WINDOW_BEGIN}}.PT0S.nc'
+     date: '{{ATM_WINDOW_BEGIN}}'
+     state variables: [ssh, tocn, socn, uocn, vocn, cicen, hicen, hsnon]
+  trajectory:
+    read_from_file: 1
+    basename: ./INPUT/
+    ocn_filename: MOM.res.nc
+    ice_filename: cice.res.nc
+    date: '{{ATM_WINDOW_BEGIN}}'
+    state variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh, hocn, mld, layer_depth]
+  output:
+     datadir: ./static_ens
+     exp: filtered.MEMNUM
+     type: incr
+     date: '{{ATM_WINDOW_BEGIN}}'

parm/soca/berror/soca_bump_loc.yaml

@@ -9,7 +9,7 @@ background:
   basename: ./INPUT/
   ocn_filename: MOM.res.nc
   ice_filename: cice.res.nc
-  state variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh]
+  state variables: [tocn]
 
 background error:
   covariance model: SABER
@@ -19,7 +19,7 @@ background error:
       io:
         data directory: bump
       drivers:
-        multivariate strategy: crossed
+        multivariate strategy: duplicated
         compute nicas: true
         write local nicas: true
         write nicas grids: true
@@ -28,26 +28,21 @@ background error:
       nicas:
         resolution: !ENV ${NICAS_RESOL}
         max horizontal grid size: ${NICAS_GRID_SIZE}
-      grids:
-      - model:
-          variables: [socn, tocn, uocn, vocn]
-      - model:
-          variables: [ssh, cicen, hicen, hsnon]
 
       input model files:
       - parameter: rh
         file:
           read_from_file: 1
           date: *date
           basename: ./
-          ocn_filename: ocn.cor_rh.incr.0001-01-01T00:00:00Z.nc
-          ice_filename: ice.cor_rh.incr.0001-01-01T00:00:00Z.nc
-          state variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh]
+          ocn_filename: ocn.loc_rh.incr.0001-01-01T00:00:00Z.nc
+          ice_filename: ice.loc_rh.incr.0001-01-01T00:00:00Z.nc
+          state variables: [tocn]
       - parameter: rv
         file:
           read_from_file: 1
           date: *date
           basename: ./
-          ocn_filename: ocn.cor_rv.incr.0001-01-01T00:00:00Z.nc
-          ice_filename: ice.cor_rv.incr.0001-01-01T00:00:00Z.nc
-          state variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh]
+          ocn_filename: ocn.loc_rv.incr.0001-01-01T00:00:00Z.nc
+          ice_filename: ice.loc_rv.incr.0001-01-01T00:00:00Z.nc
+          state variables: [tocn]

parm/soca/berror/soca_bump_split.yaml

@@ -9,7 +9,7 @@ background:
   basename: ./INPUT/
   ocn_filename: MOM.res.nc
   ice_filename: cice.res.nc
-  state variables: [cicen, hicen, hsnon, socn, tocn, ssh]
+  state variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh]
 
 background error:
   covariance model: SABER

parm/soca/berror/soca_clim_ens_moments.yaml

@@ -0,0 +1,29 @@
+geometry:
+  geom_grid_file: soca_gridspec.nc
+  mom6_input_nml: mom_input.nml
+  fields metadata: fields_metadata.yaml
+
+ensemble:
+  members from template:
+    template:
+      read_from_file: 1
+      date: '{{ATM_WINDOW_BEGIN}}'
+      basename: ./static_ens/
+      ocn_filename: ocn.%mem%.nc
+      ice_filename: ice.%mem%.nc
+      remap_filename: ./INPUT/MOM.res.nc
+      state variables: [tocn, socn, ssh, uocn, vocn, hocn, cicen, hicen, hsnon]
+    pattern: '%mem%'
+    nmembers: ${CLIM_ENS_SIZE}
+
+mean output:
+  datadir: ./static_ens/
+  exp: orig_ens_mean
+  type: incr
+  date: '{{ATM_WINDOW_BEGIN}}'
+
+standard deviation output:
+  datadir: ./static_ens/
+  exp: orig_ens_stddev
+  type: incr
+  date: '{{ATM_WINDOW_BEGIN}}'

parm/soca/berror/soca_clim_ens_perts.yaml

@@ -0,0 +1,35 @@
+geometry:
+  geom_grid_file: soca_gridspec.nc
+  mom6_input_nml: mom_input.nml
+  fields metadata: fields_metadata.yaml
+
+recenter variables:  [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh, hocn, mld, layer_depth]
+
+zero center: True
+
+center:
+  read_from_file: 1
+  basename: ./INPUT/
+  ocn_filename: MOM.res.nc
+  ice_filename: cice.res.nc
+  date: '{{ATM_WINDOW_BEGIN}}'
+  state variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh, hocn, mld, layer_depth]
+
+ensemble:
+  members from template:
+    template:
+      read_from_file: 1
+      date: '{{ATM_WINDOW_BEGIN}}'
+      basename: ./static_ens/
+      ocn_filename: ocn.%mem%.nc
+      ice_filename: ice.%mem%.nc
+      remap_filename: ./INPUT/MOM.res.nc
+      state variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh, hocn, mld, layer_depth]
+    pattern: '%mem%'
+    nmembers: ${CLIM_ENS_SIZE}
+
+recentered output:
+  datadir: ./static_ens
+  exp: bal
+  type: ens
+  date: '{{ATM_WINDOW_BEGIN}}'

parm/soca/berror/soca_ensweights.yaml

@@ -0,0 +1,27 @@
+geometry:
+  mom6_input_nml: mom_input.nml
+  fields metadata: ./fields_metadata.yaml
+
+date: '{{ATM_WINDOW_MIDDLE}}'
+
+variables:
+  ice: [cicen, hicen, hsnon]
+  ocean: [tocn, socn, uocn, vocn, ssh]
+
+background:
+  date: '{{ATM_WINDOW_BEGIN}}'
+  basename: ./INPUT/
+  ocn_filename: MOM.res.nc
+  ice_filename: cice.res.nc
+  read_from_file: 1
+
+weights:
+  # Need to provide weights^2 when reading from file
+  ice: 0.0025   #  5% of original variance
+  ocean: 0.01   # 10%      "       "
+
+output:
+  datadir: ./
+  date: '{{ATM_WINDOW_MIDDLE}}'
+  exp: ens_weights
+  type: incr

parm/soca/berror/soca_setlocscales.yaml

@@ -0,0 +1,26 @@
+resolution:
+  mom6_input_nml: mom_input.nml
+  fields metadata: ./fields_metadata.yaml
+
+date: 0001-01-01T00:00:00Z
+
+corr variables: [tocn, cicen]
+
+scales:
+  vert layers: 70 # in units of layer
+  tocn:
+    rossby mult: 1.00
+    min grid mult: 2.0
+  cicen:
+    rossby mult: 1.00
+    min grid mult: 2.0
+
+rh output:
+  datadir: ./
+  exp: loc_rh
+  type: incr
+
+rv output:
+  datadir: ./
+  exp: loc_rv
+  type: incr

parm/soca/obs/config/salt_profile_fnmoc.yaml

@@ -9,8 +9,13 @@ obs space:
       type: H5File
       obsfile: !ENV ${DATA}/diags/salt_profile_fnmoc.${PDY}${cyc}.nc4
   simulated variables: [salinity]
+  io pool:
+    max pool size: 1
 obs operator:
-  name: MarineVertInterp
+  name: VertInterp
   observation alias file: ./obsop_name_map.yaml
+  vertical coordinate: sea_water_depth
+  observation vertical coordinate: depth
+  interpolation method: linear
 obs error:
   covariance model: diagonal

parm/soca/obs/config/temp_profile_fnmoc.yaml

@@ -8,8 +8,10 @@ obs space:
     engine:
       type: H5File
       obsfile: !ENV ${DATA}/diags/temp_profile_fnmoc.${PDY}${cyc}.nc4
-  simulated variables: [sea_water_temperature]
+  simulated variables: [waterTemperature]
+  io pool:
+    max pool size: 1
 obs operator:
-  name: Identity
+  name: InsituTemperature
 obs error:
   covariance model: diagonal