Add in the start of supporting infrastructure for BUFR to IODA direct…

… conversion (NOAA-EMC#575)

* save templated json file using example from emily

* script to generate JSON from template

* start to refactor run script

* save before debugging

* commit before draft pr

* code norms

* Add the option to run observer without data filtering (HofX + obs error assignment only)
Point the test data set (obs/geoval) generated with GSI develop version + CRTM-2.4.0_emc

* Modify UFO from develop to feature/ufo_updates_satwind (only temporary)
The branch feature/ufo_updates_satwind is in UFO PR.
Will set UFO back to develop once the UFO PR merged into UFO develop.

* Update satewind.yaml for end-to-end processing:
(1) Point GeoVaLS/land-type_index_NPOESS to GeoVaLs/water_area_fraction and GeoVaLs/land_area_fraction
(2) Modify the use of ObsFunction/SatWindsSPDBCheck to ObsFunction/WindsSPDBCheck (drop "sat" from the function name)

* Add satwind yaml files for goes-16 and goes-17.
The data filtering for GOES AMVs is extracted from ./testing/satwind.yaml
Two data filtering steps are turned off pending further implementation and investigation:
(1) tropopause pressure check ---- compared to GSI, this check removes more data; need to check tropopause pressure implementation in fv3-jedi
(2) turn off duplicate check since it dependes on "useage parameter" from GSI as an input from obs space.

* Add a directory and bufr converter python driver for GOES satwind

* Fix coding norm errors - 1st try

* Fix coding norm errors --- 2nd try

* Fix codind norm --- 3rd try

* Fix coding norm --- 4th try

* Fix coding norm --- 4th try again

* revert back to develop for ufo

* commit with things moved and other changes before modification to py script

* use argparse; start to use logger

* address my own comments

* remove verbose call

* make the logger say the name of the script

* add atmos to expected DMPDIR path

* Add launch date to json configuration

* Add a few more entries for global attributes. Changes do not change or break results

* Cosmetic fix: change Launch to launch.

* Fix coding norm errors

---------

Co-authored-by: Emily Liu <emily.liu@noaa.gov>

parm/atm/obs/testing/satwind.yaml

@@ -619,9 +619,8 @@ obs prior filters:
     where:
     - variable: MetaData/satelliteIdentifier
       is_in: 250-299
-    - variable: GeoVaLs/land_type_index_NPOESS
-      minvalue: 1.
-      maxvalue: 1.
+    - variable: GeoVaLs/land_area_fraction
+      minvalue: 0.999
     test variables:
     - name: MetaData/pressure
     maxvalue: 85000.
@@ -811,8 +810,8 @@ obs prior filters:
     - name: windNorthward
     where:
     - variable:
-        name: GeoVaLs/land_type_index_NPOESS
-      minvalue: 1.
+        name: GeoVaLs/water_area_fraction
+      maxvalue: 0.001
     - variable:
         name: ObsType/windEastward
       is_in: 247
@@ -831,8 +830,8 @@ obs prior filters:
     - name: windNorthward
     where:
     - variable:
-        name: GeoVaLs/land_type_index_NPOESS
-      minvalue: 1.
+        name: GeoVaLs/water_area_fraction
+      maxvalue: 0.001
     - variable:
         name: ObsType/windEastward
       is_in: 244, 257-260
@@ -913,7 +912,7 @@ obs post filters:
     filter variables:
     - name: windEastward
     function absolute threshold:
-    - name: ObsFunction/SatWindsSPDBCheck
+    - name: ObsFunction/WindsSPDBCheck
       options:
         wndtype: [  240,  241,  242,  243,  244,  245,  246,  247,  248,  249, 250,  251,  252,  253,   254,  255,  256,  257,  258,  259,  260]
         cgross:  [  2.5,  2.5,  2.5,  1.5,  2.5,  1.3,  1.3,  2.5,  2.5,  2.5, 2.5,  1.3,  2.5,  1.5,   1.5,  2.5,  2.5,  2.5,  2.5,  2.5,  2.5]
@@ -927,7 +926,7 @@ obs post filters:
     filter variables:
     - name: windNorthward
     function absolute threshold:
-    - name: ObsFunction/SatWindsSPDBCheck
+    - name: ObsFunction/WindsSPDBCheck
       options:
         wndtype: [  240,  241,  242,  243,  244,  245,  246,  247,  248,  249, 250,  251,  252,  253,   254,  255,  256,  257,  258,  259,  260]
         cgross:  [  2.5,  2.5,  2.5,  1.5,  2.5,  1.3,  1.3,  2.5,  2.5,  2.5, 2.5,  1.3,  2.5,  1.5,   1.5,  2.5,  2.5,  2.5,  2.5,  2.5,  2.5]
@@ -939,7 +938,7 @@ obs post filters:
   # The last error inflation check is for duplicate observations. This one needs
   # to come last, because we don't want to inflate errors for duplication if one
   # of the duplicates should be rejected.
-  - filter: BlackList
+  - filter: RejectList
     filter variables:
     - name: windEastward
     action:
@@ -949,7 +948,7 @@ obs post filters:
         options:
           use_air_pressure: true
           variable: windEastward
-  - filter: BlackList
+  - filter: RejectList
     filter variables:
     - name: windNorthward
     action:

parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.json

@@ -0,0 +1,17 @@
+{
+  "data_format"      : "bufr_d",
+  "data_type"        : "satwnd",
+  "cycle_type"       : "{{ DUMP }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "subsets"          : [ "NC005030", "NC005031", "NC005032", "NC005034", "NC005039" ],
+  "data_description" : "NC005030 NESDIS SATWIND, GOES IR(LW);  NC005031 NESDIS SATWIND, GOES WV-IMG/DL; NC005032 NESDIS SATWIND, GOES VIS; NC005034 NESDIS SATWIND, GOES WV-IMG/CT; NC005039 NESDIS SATWIND, GOES IR(SW)", 
+  "data_provider"    : "U.S. NOAA/NESDIS",
+  "sensor_info"      : { "sensor_name": "ABI",  "sensor_full_name": "Advanced Baseline Imager",  "sensor_id": 617 },
+  "satellite_info"   : [
+                          { "satellite_name": "GOES-16", "satellite_full_name": "Geostationary Operational Satellite - 16", "satellite_id": 270, "launch time": "20161119" },
+                          { "satellite_name": "GOES-17", "satellite_full_name": "Geostationary Operational Satellite - 17", "satellite_id": 271, "launch time": "20180301" },
+                          { "satellite_name": "GOES-18", "satellite_full_name": "Geostationary Operational Satellite - 18", "satellite_id": 272, "launch time": "20220301" }
+                       ]
+}