Add BUFR2IODA python API converter to prepoceanobs task (#914)

This PR adds BUFR2IODA python API converter to prepoceanobs task and does the following: 1. Subsampled monthly bathy, tesac and trkob in situ BUFR files are tested. 2. The subsampled files have been renamed (from <obtype>.201804.dcom_subsampled) as <cycle>.t<hh>z.<obtype>.tm00.bufr_d 3. Runs for one cycle: 2018041512. closes #892 Overlapping windows application will be added in a subsequent PR --------- Co-authored-by: Guillaume Vernieres <guillaume.vernieres@noaa.gov> Co-authored-by: AndrewEichmann-NOAA <andrew.eichmann@noaa.gov> Co-authored-by: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Co-authored-by: Cory Martin <cory.r.martin@noaa.gov>
NOAA-EMC · Mar 28, 2024 · 33b4cb9 · 33b4cb9
1 parent 30d803c
commit 33b4cb9
Show file tree

Hide file tree

Showing 46 changed files with 3,967 additions and 54 deletions.
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "subpfl",
+  "subsets"          : "SUBPFL",
+  "source"           : "NCEP data tank",
+  "data_type"        : "argo",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ ARGO profiles",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "bathy",
+  "subsets"          : "BATHY",
+  "source"           : "NCEP data tank",
+  "data_type"        : "bathy",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ Bathythermal profiles",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoy.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoy.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "dbuoy",
+  "subsets"          : "DBUOY",
+  "source"           : "NCEP data tank",
+  "data_type"        : "dbuoy",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ drifting buoy (TAC:drifters, etc) profiles",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoyb.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoyb.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "dbuoyb",
+  "subsets"          : "DBUOYB",
+  "source"           : "NCEP data tank",
+  "data_type"        : "dbuoyb",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ drifting buoy (BUFR:drifters, etc) profiles",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "subpfl",
+  "subsets"          : "SUBPFL",
+  "source"           : "NCEP data tank",
+  "data_type"        : "glider",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ GLIDER profiles",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_marinemammals.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_marinemammals.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "tesac",
+  "subsets"          : "TESAC",
+  "source"           : "NCEP data tank",
+  "data_type"        : "marinemammals",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ Marine Mammals profiles",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoy.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoy.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "mbuoy",
+  "subsets"          : "MBUOY",
+  "source"           : "NCEP data tank",
+  "data_type"        : "mbuoy",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ moored buoy (TAC:Tropical Mooring) profiles",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoyb.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoyb.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "mbuoyb",
+  "subsets"          : "MBUOYB",
+  "source"           : "NCEP data tank",
+  "data_type"        : "mbuoyb",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ  moored buoy (BUFR:Tropical Mooring) profiles",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "tesac",
+  "subsets"          : "TESAC",
+  "source"           : "NCEP data tank",
+  "data_type"        : "tesac",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ TESAC profiles",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "xbtctd",
+  "subsets"          : "XBTCTD",
+  "source"           : "NCEP data tank",
+  "data_type"        : "xbtctd",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ XBT/XCTD profiles",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_altkob.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_altkob.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "altkob",
+  "subsets"          : "ALTKOB",
+  "source"           : "NCEP data tank",
+  "data_type"        : "altkob",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ ALTKOB surface",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_ships.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_ships.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "ships",
+  "subsets"          : "SHIPS",
+  "source"           : "NCEP data tank",
+  "data_type"        : "bathy",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ ship (TAC: restriced) surface",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_shipsu.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_shipsu.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "shipsu",
+  "subsets"          : "SHIPSU",
+  "source"           : "NCEP data tank",
+  "data_type"        : "shipsu",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ ship (TAC: unrestricted) surface",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.json
@@ -0,0 +1,12 @@
+{
+  "data_format"      : "trkob",
+  "subsets"          : "TRACKOB",
+  "source"           : "NCEP data tank",
+  "data_type"        : "trackob",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "data_description" : "6-hrly in situ TRACKOB surface",
+  "data_provider"    : "U.S. NOAA"
+}
diff --git a/parm/soca/obs/config/dev/insitu_profile_argo.yaml b/parm/soca/obs/config/dev/insitu_profile_argo.yaml
@@ -0,0 +1,22 @@
+obs space:
+  name: insitu_profile_argo
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_argo.${PDY}${cyc}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/diags/insitu_profile_argo.${PDY}${cyc}.nc4
+  simulated variables: [waterTemperature, salinity]
+  observed variables: [waterTemperature, salinity]
+  io pool:
+    max pool size: 1
+obs operator:
+  name: Composite
+  components:
+  - name: InsituTemperature
+    variables:
+    - name: waterTemperature
+obs error:
+  covariance model: diagonal
diff --git a/parm/soca/obs/config/dev/insitu_profile_bathy.yaml b/parm/soca/obs/config/dev/insitu_profile_bathy.yaml
@@ -0,0 +1,158 @@
+obs space:
+  name: insitu_profile_bathy
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_bathy.${PDY}${cyc}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/diags/insitu_profile_bathy.${PDY}${cyc}.nc4
+  simulated variables: [waterTemperature]
+  io pool:
+    max pool size: 1
+obs operator:
+  name: InsituTemperature
+obs error:
+  covariance model: diagonal
+
+#--------------------------------------------------------------------------------------
+#                   START OF OBS FILTERS
+# The QC filters used here are based on the document by IODE that can be found at
+# https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf
+#--------------------------------------------------------------------------------------
+obs filters:
+
+  # Land Check
+  - filter: Domain Check
+    where:
+    - variable: {name: GeoVaLs/sea_area_fraction}
+      minvalue: 0.5
+#--------------------------------------------------------------------------------------
+## Filters for T:
+#--------------------------------------------------------------------------------------
+  #------------------------------------------------------------------------------------
+  ### Global range test
+  #------------------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: -2.5
+    maxvalue: 40.0
+
+  #------------------------------------------------------------------------------------
+  ### Regional range tests
+  #------------------------------------------------------------------------------------
+
+  #### Red Sea
+  #------------------------------------------------------------------------------------
+  ####
+  #### the document linked here describes Red sea as the are between 10N, 40E;
+  #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden.
+  #### A more reasonable choice seemed to be a box that extends from 10 N to
+  #### 30 N and 30 E to 45 East .
+  #------------------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: 21.7
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 10
+      maxvalue: 30
+    - variable:
+        name: MetaData/longitude
+      minvalue: 30
+
+  #### Mediterranean Sea
+  #-----------------------------------------------------------------------------
+  ##### Area 1/3 for Mediterranean Sea
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: 10.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 30
+      maxvalue: 40
+    - variable:
+        name: MetaData/longitude
+      minvalue: -6
+      maxvalue: 40
+  ##### Area 2/3 for Mediterranean Sea
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: 10.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 40
+      maxvalue: 41.5
+    - variable:
+        name: MetaData/longitude
+      minvalue: 20
+      maxvalue: 30
+  ##### Area 3/3 for Mediterranean Sea
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: 10.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 40
+      maxvalue: 46
+    - variable:
+        name: MetaData/longitude
+      minvalue: 0
+      maxvalue: 20
+
+  #### Northwestern shelves
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: -2.0
+    maxvalue: 24.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 50
+      maxvalue: 60
+    - variable:
+        name: MetaData/longitude
+      minvalue: -20
+      maxvalue: 10
+  #### Southwestern shelves
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: -2.0
+    maxvalue: 30
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 25
+      maxvalue: 50
+    - variable:
+        name: MetaData/longitude
+      minvalue: -30
+      maxvalue: 0
+
+  #### Arctic Sea
+  #-----------------------------------------------------------------------------
+  - filter: Bounds Check
+    filter variables: [{name: waterTemperature}]
+    minvalue: -1.92
+    maxvalue: 25.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 60
+
+  - filter: Background Check
+    filter variables: [{name: waterTemperature}]
+    threshold: 5.0
+    absolute threshold: 5.0
+
diff --git a/...soca/obs/config/insitu_profile_bathy.yaml → .../obs/config/dev/insitu_profile_dbuoy.yaml b/...soca/obs/config/insitu_profile_bathy.yaml → .../obs/config/dev/insitu_profile_dbuoy.yaml
@@ -1,13 +1,13 @@
 obs space:
-  name: insitu_profile_bathy
+  name: insitu_profile_dbuoy
   obsdatain:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_bathy.${PDY}${cyc}.nc4
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_dbuoy.${PDY}${cyc}.nc4
   obsdataout:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/diags/insitu_profile_bathy.${PDY}${cyc}.nc4
+      obsfile: !ENV ${DATA}/diags/insitu_profile_dbuoy.${PDY}${cyc}.nc4
   simulated variables: [waterTemperature, salinity]
   observed variables: [waterTemperature, salinity]
   io pool: