From b4522a20bd073f21d41a36eaefb60c78e701932d Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Thu, 8 Feb 2024 14:31:50 -0600 Subject: [PATCH 01/28] Add BUFR2IODA python API converter to prepoceanobs --- .../bufr2ioda_insitu_profile_bathy.json | 12 + .../bufr2ioda_insitu_profile_tesac.json | 12 + .../bufr2ioda_insitu_surface_trkob.json | 12 + parm/soca/obsprep/obsprep_config.yaml | 12 +- scripts/exglobal_prep_ocean_obs.py | 48 ++- test/soca/gw/setup_obsprep.sh | 5 +- .../bufr2ioda_insitu_profile_bathy.py | 274 +++++++++++++++ .../bufr2ioda_insitu_profile_tesac.py | 319 ++++++++++++++++++ .../bufr2ioda_insitu_surface_trkob.py | 281 +++++++++++++++ ush/ioda/bufr2ioda/gen_bufr2ioda_json.py | 5 +- 10 files changed, 965 insertions(+), 15 deletions(-) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.json create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.py create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.py create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.json new file mode 100644 index 000000000..1acfec4ec --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.json @@ -0,0 +1,12 @@ +{ + "data_format" : "bathy", + "subsets" : "BATHY", + "source" : "NCEP data tank", + "data_type" : "bathy", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ Bathythermal profiles", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.json new file mode 100644 index 000000000..c079504cd --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.json @@ -0,0 +1,12 @@ +{ + "data_format" : "tesac", + "subsets" : "TESAC", + "source" : "NCEP data tank", + "data_type" : "tesac", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ TESAC profiles", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.json new file mode 100644 index 000000000..ccec0cd37 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.json @@ -0,0 +1,12 @@ +{ + "data_format" : "trkob", + "subsets" : "TRACKOB", + "source" : "NCEP data tank", + "data_type" : "trackob", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ TRACKOB surface", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/soca/obsprep/obsprep_config.yaml b/parm/soca/obsprep/obsprep_config.yaml index 38ca66759..f25715ac6 100644 --- a/parm/soca/obsprep/obsprep_config.yaml +++ b/parm/soca/obsprep/obsprep_config.yaml @@ -153,17 +153,17 @@ observations: name: insitu_profile_bathy provider: GTS dmpdir subdir: atmos - output file: '*.bathy_profile.ioda.nc' - dmpdir regex: 'bathy.*.dcom_subsampled' + output file: 'bathy_profile.ioda.nc' + dmpdir regex: 'gdas.*.bathy.*.bufr_d' - obs space: name: insitu_profile_tesac provider: GTS dmpdir subdir: atmos - output file: '*.tesac_profile.ioda.nc' - dmpdir regex: 'tesac.*.dcom_subsampled' + output file: 'tesac_profile.ioda.nc' + dmpdir regex: 'gdas.*.tesac.*.bufr_d' - obs space: name: insitu_surface_trkob provider: GTS dmpdir subdir: atmos - output file: '*.trkob_surface.ioda.nc' - dmpdir regex: 'trkob.*.dcom_subsampled' + output file: 'trkob_surface.ioda.nc' + dmpdir regex: 'gdas.*.trkob.*.bufr_d' diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index c07fb27c8..68307ecf5 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -4,14 +4,19 @@ from datetime import datetime, timedelta from multiprocessing import Process import os +from pathlib import Path import subprocess from soca import prep_marine_obs from wxflow import YAMLFile, save_as_yaml, FileHandler, Logger +from wxflow import add_to_datetime, to_timedelta, datetime_to_YMDH +from gen_bufr2ioda_json import gen_bufr_json logger = Logger() cyc = os.getenv('cyc') PDY = os.getenv('PDY') +RUN = os.getenv('RUN') +COMIN_OBS = os.getenv('COMIN_OBS') # Set the window times cdateDatetime = datetime.strptime(PDY + cyc, '%Y%m%d%H') @@ -29,12 +34,49 @@ OBSPREP_YAML = os.getenv('OBSPREP_YAML') +# BUFR2IODA json and python scripts +JSON_TMPL_DIR = os.getenv('JSON_TMPL_DIR') +BUFR2IODA_PY_DIR = os.getenv('BUFR2IODA_PY_DIR') + if os.path.exists(OBSPREP_YAML): obsprepConfig = YAMLFile(OBSPREP_YAML) else: logger.critical(f"OBSPREP_YAML file {OBSPREP_YAML} does not exist") raise FileNotFoundError +if not os.path.exists(COMOUT_OBS): + os.makedirs(COMOUT_OBS) + + +def bufr2ioda(obtype, PDY, cyc, RUN, COMIN_OBS, COMOUT_OBS): + logger.info(f"Process {obtype} for {RUN}.{PDY}/{cyc} from {COMIN_OBS} to {COMOUT_OBS}") + + # Load configuration + config = { + 'RUN': RUN, + 'current_cycle': cdateDatetime, + 'DMPDIR': COMIN_OBS, + 'COM_OBS': COMOUT_OBS, + } + + json_output_file = os.path.join(COMOUT_OBS, f"{obtype}_{datetime_to_YMDH(cdateDatetime)}.json") + filename = 'bufr2ioda_' + obtype + '.json' + template = os.path.join(JSON_TMPL_DIR,filename) + + # Generate cycle specific json from TEMPLATE + gen_bufr_json(config, template, json_output_file) + + bufr2iodapy = BUFR2IODA_PY_DIR + '/bufr2ioda_' + obtype + '.py' + logger.info(f"BUFR2IODA python scripts: {bufr2iodapy}") + + try: + subprocess.run(['python', bufr2iodapy, '-c', json_output_file, '-v']) + logger.info(f"BUFR2IODA python API converter on obs space {obtype} ran successfully") + except subprocess.CalledProcessError as e: + logger.info(f"BUFR2IODA python API converter failed with error {e}, \ + return code {e.returncode}") + return e.returncode + def run_netcdf_to_ioda(obsspace_to_convert): name, iodaYamlFilename = obsspace_to_convert @@ -94,9 +136,8 @@ def run_netcdf_to_ioda(obsspace_to_convert): outputFilename = f"gdas.t{cyc}z.{obs_space_name}.{PDY}{cyc}.nc4" obsprepSpace['output file'] = outputFilename - # Skip in situ IODA conversion for now if obsprepSpaceName.split('_')[0] == 'insitu': - logger.info("Skipping insitu conversion for now") + bufr2ioda(obsprepSpaceName, PDY, cyc, RUN, COMIN_OBS, COMOUT_OBS) else: iodaYamlFilename = obsprepSpaceName + '2ioda.yaml' save_as_yaml(obsprepSpace, iodaYamlFilename) @@ -122,7 +163,4 @@ def run_netcdf_to_ioda(obsspace_to_convert): for process in processes: process.join() -if not os.path.exists(COMOUT_OBS): - os.makedirs(COMOUT_OBS) - FileHandler({'copy': files_to_save}).sync() diff --git a/test/soca/gw/setup_obsprep.sh b/test/soca/gw/setup_obsprep.sh index 43f8b1877..81e419333 100755 --- a/test/soca/gw/setup_obsprep.sh +++ b/test/soca/gw/setup_obsprep.sh @@ -41,11 +41,12 @@ for PDY in "${PDYs[@]}"; do done else # Copy subsampled monthly in situ BUFR - # Example filename: tesac.201804.dcom_subsampled + # Example filename: tesac.201804.dcom_subsampled have been copied as + # gdas.t00z.tesac.tm00.bufr_d (the original filename) # TODO: SP to replace these with daily and monthly subsampled with proper dates fullsubdir="$PDYdir/$cyc/atmos" mkdir -p "$fullsubdir" - for file in "$testdatadir_bufr/*.${PDY:0:6}.dcom_subsampled"; do + for file in "$testdatadir_bufr/*.bufr_d"; do cp -p $file $fullsubdir done fi diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.py new file mode 100755 index 000000000..71cc42d92 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +# (C) Copyright 2024 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import numpy.ma as ma +import os +import argparse +import math +import calendar +import time +import copy +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger +import warnings +# suppress warnings +warnings.filterwarnings('ignore') + + +def Compute_sequenceNumber(lon): + lon_u, seqNum = np.unique(lon, return_inverse=True) + seqNum = seqNum.astype(np.int32) + logger.debug(f"Len of Sequence Number: {len(seqNum)}") + + return seqNum + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + # General Information + converter = 'BUFR to IODA Converter' + platform_description = 'Profiles from BATHYthermal: temperature' + + bufrfile = f"{cycle_datetime}-{cycle_type}.t{hh}z.{data_format}.tm00.bufr_d" + DATA_PATH = os.path.join(dump_dir, bufrfile) + logger.debug(f"{bufrfile}, {DATA_PATH}") + + # ========================================== + # Make the QuerySet for all the data we want + # ========================================== + start_time = time.time() + + logger.debug(f"Making QuerySet ...") + q = bufr.QuerySet() + + # MetaData + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('ryear', '*/RCYR') + q.add('rmonth', '*/RCMO') + q.add('rday', '*/RCDY') + q.add('rhour', '*/RCHR') + q.add('rminute', '*/RCMI') + q.add('stationID', '*/RPID') + q.add('latitude', '*/CLAT') + q.add('longitude', '*/CLON') + q.add('depth', '*/BTOCN/DBSS') + + # ObsValue + q.add('temp', '*/BTOCN/STMP') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ========================================================== + # Open the BUFR file and execute the QuerySet + # Use the ResultSet returned to get numpy arrays of the data + # ========================================================== + start_time = time.time() + + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + logger.debug(f" ... Executing QuerySet: get MetaData ...") + dateTime = r.get_datetime('year', 'month', 'day', 'hour', 'minute', group_by='depth') + dateTime = dateTime.astype(np.int64) + rcptdateTime = r.get_datetime('ryear', 'rmonth', 'rday', 'rhour', 'rminute', group_by='depth') + rcptdateTime = rcptdateTime.astype(np.int64) + stationID = r.get('stationID', group_by='depth') + lat = r.get('latitude', group_by='depth') + lon = r.get('longitude', group_by='depth') + depth = r.get('depth', group_by='depth') + + # ObsValue + logger.debug(f" ... Executing QuerySet: get ObsValue ...") + temp = r.get('temp', group_by='depth') + temp -= 273.15 + + # Add mask based on min, max values + mask = ((temp > -10.0) & (temp <= 50.0)) + lat = lat[mask] + lon = lon[mask] + depth = depth[mask] + stationID = stationID[mask] + dateTime = dateTime[mask] + rcptdateTime = rcptdateTime[mask] + temp = temp[mask] + + logger.debug(f"Get sequenceNumber based on unique longitude...") + seqNum = Compute_sequenceNumber(lon) + + # ObsError + logger.debug(f"Generating ObsError array with constant value (instrument error)...") + ObsError_temp = np.float32(np.ma.masked_array(np.full((len(temp)), 0.24))) + + # PreQC + logger.debug(f"Generating PreQC array with 0...") + PreQC = (np.ma.masked_array(np.full((len(temp)), 0))).astype(np.int32) + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + + # ================================================== + # Check values of BUFR variables, dimension and type + # ================================================== + + logger.debug(f" temp min, max, length, dtype = {temp.min()}, {temp.max()}, {len(temp)}, {temp.dtype}") + logger.debug(f" lon min, max, length, dtype = {lon.min()}, {lon.max()}, {len(lon)}, {lon.dtype}") + logger.debug(f" lat min, max, length, dtype = {lat.min()}, {lat.max()}, {len(lat)}, {lat.dtype}") + logger.debug(f" depth min, max, length, dtype = {depth.min()}, {depth.max()}, {len(depth)}, {depth.dtype}") + logger.debug(f" PreQC min, max, length, dtype = {PreQC.min()}, {PreQC.max()}, {len(PreQC)}, {PreQC.dtype}") + logger.debug(f" ObsError_temp min, max, length, dtype = {ObsError_temp.min()}, {ObsError_temp.max()}, {len(ObsError_temp)}, {ObsError_temp.dtype}") + logger.debug(f" stationID shape, dtype = {stationID.shape}, {stationID.astype(str).dtype}") + logger.debug(f" dateTime shape, dtype = {dateTime.shape}, {dateTime.dtype}") + logger.debug(f" rcptdateTime shape, dytpe = {rcptdateTime.shape}, {rcptdateTime.dtype}") + logger.debug(f" sequence Num shape, dtype = {seqNum.shape}, {seqNum.dtype}") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}_profiles.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeRange', [str(dateTime.min()), str(dateTime.max())]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, and attributes") + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # rcptDatetime + obsspace.create_var('MetaData/rcptdateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'receipt Datetime') \ + .write_data(rcptdateTime) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Station Identification + obsspace.create_var('MetaData/stationID', dtype=stationID.dtype, fillval=stationID.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stationID) + + # Depth + obsspace.create_var('MetaData/depth', dtype=depth.dtype, fillval=depth.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Water depth') \ + .write_data(depth) + + # Sequence Number + obsspace.create_var('MetaData/sequenceNumber', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'Sequence Number') \ + .write_data(seqNum) + + # PreQC + obsspace.create_var('PreQC/waterTemperature', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + # ObsValue + obsspace.create_var('ObsValue/waterTemperature', dtype=temp.dtype, fillval=temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('valid_range', np.array([-10.0, 50.0], dtype=np.float32)) \ + .write_attr('long_name', 'water Temperature') \ + .write_data(temp) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: {running_time} \ + seconds") + + logger.debug(f"All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_insitu_profile_bathy.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.py new file mode 100755 index 000000000..400aedbef --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +# (C) Copyright 2024 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import numpy.ma as ma +import os +import argparse +import math +import calendar +import time +import copy +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger +import warnings +# suppress warnings +warnings.filterwarnings('ignore') + + +def Compute_sequenceNumber(lon): + + lon_u, seqNum = np.unique(lon, return_inverse=True) + seqNum = seqNum.astype(np.int32) + logger.debug(f"Len of Sequence Number: {len(seqNum)}") + + return seqNum + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + # General Information + converter = 'BUFR to IODA Converter' + platform_description = 'Profiles from TESAC: temperature and salinity' + + bufrfile = f"{cycle_datetime}-{cycle_type}.t{hh}z.{data_format}.tm00.bufr_d" + DATA_PATH = os.path.join(dump_dir, bufrfile) + logger.debug(f"{bufrfile}, {DATA_PATH}") + + # ========================================== + # Make the QuerySet for all the data we want + # ========================================== + start_time = time.time() + + logger.debug(f"Making QuerySet ...") + q = bufr.QuerySet() + + # MetaData + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('ryear', '*/RCYR') + q.add('rmonth', '*/RCMO') + q.add('rday', '*/RCDY') + q.add('rhour', '*/RCHR') + q.add('rminute', '*/RCMI') + q.add('stationID', '*/RPID') + q.add('latitude', '*/CLAT') + q.add('longitude', '*/CLON') + q.add('depth', '*/BTOCN/DBSS') + + # ObsValue + q.add('temp', '*/BTOCN/STMP') + q.add('saln', '*/BTOCN/SALN') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ========================================================== + # Open the BUFR file and execute the QuerySet + # Use the ResultSet returned to get numpy arrays of the data + # ========================================================== + start_time = time.time() + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + logger.debug(f" ... Executing QuerySet: get MetaData ...") + dateTime = r.get_datetime('year', 'month', 'day', 'hour', 'minute', group_by='depth') + dateTime = dateTime.astype(np.int64) + rcptdateTime = r.get_datetime('ryear', 'rmonth', 'rday', 'rhour', 'rminute', group_by='depth') + rcptdateTime = rcptdateTime.astype(np.int64) + stationID = r.get('stationID', group_by='depth') + lat = r.get('latitude', group_by='depth') + lon = r.get('longitude', group_by='depth') + depth = r.get('depth', group_by='depth') + + # ObsValue + logger.debug(f" ... Executing QuerySet: get ObsValue ...") + temp = r.get('temp', group_by='depth') + temp -= 273.15 + saln = r.get('saln', group_by='depth') + + # Add mask based on min, max values + mask = ((temp > -10.0) & (temp <= 50.0)) & ((saln >= 0.0) & (saln <= 45.0)) + lat = lat[mask] + lon = lon[mask] + depth = depth[mask] + stationID = stationID[mask] + dateTime = dateTime[mask] + rcptdateTime = rcptdateTime[mask] + temp = temp[mask] + saln = saln[mask] + + logger.debug(f"Get sequenceNumber based on unique longitude...") + seqNum = Compute_sequenceNumber(lon) + + # ================================== + # Separate TESAC profiles tesac tank + # ================================== + logger.debug(f"Creating the mask for TESAC floats based on station ID ...") + + digit_mask = [item.isdigit() for item in stationID] + indices_true = [index for index, value in enumerate(digit_mask) if value] + + # Apply index + stationID = stationID[indices_true] + lat = lat[indices_true] + lon = lon[indices_true] + depth = depth[indices_true] + temp = temp[indices_true] + saln = saln[indices_true] + seqNum = seqNum[indices_true] + dateTime = dateTime[indices_true] + rcptdateTime = rcptdateTime[indices_true] + + # ObsError + logger.debug(f"Generating ObsError array with constant value (instrument error)...") + ObsError_temp = np.float32(np.ma.masked_array(np.full((len(indices_true)), 0.02))) + ObsError_saln = np.float32(np.ma.masked_array(np.full((len(indices_true)), 0.01))) + + # PreQC + logger.debug(f"Generating PreQC array with 0...") + PreQC = (np.ma.masked_array(np.full((len(indices_true)), 0))).astype(np.int32) + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + + # ================================================== + # Check values of BUFR variables, dimension and type + # ================================================== + logger.debug(f" temp min, max, length, dtype = {temp.min()}, {temp.max()}, {len(temp)}, {temp.dtype}") + logger.debug(f" saln min, max, length, dtype = {saln.min()}, {saln.max()}, {len(saln)}, {saln.dtype}") + logger.debug(f" lon min, max, length, dtype = {lon.min()}, {lon.max()}, {len(lon)}, {lon.dtype}") + logger.debug(f" lat min, max, length, dtype = {lat.min()}, {lat.max()}, {len(lat)}, {lat.dtype}") + logger.debug(f" depth min, max, length, dtype = {depth.min()}, {depth.max()}, {len(depth)}, {depth.dtype}") + logger.debug(f" PreQC min, max, length, dtype = {PreQC.min()}, {PreQC.max()}, {len(PreQC)}, {PreQC.dtype}") + logger.debug(f" ObsError_temp min, max, length, dtype = {ObsError_temp.min()}, {ObsError_temp.max()}, {len(ObsError_temp)}, {ObsError_temp.dtype}") + logger.debug(f" ObsError_saln min, max, length, dtype = {ObsError_saln.min()}, {ObsError_saln.max()}, {len(ObsError_saln)}, {ObsError_saln.dtype}") + logger.debug(f" stationID shape, dtype = {stationID.shape}, {stationID.astype(str).dtype}") + logger.debug(f" dateTime shape, dtype = {dateTime.shape}, {dateTime.dtype}") + logger.debug(f" rcptdateTime shape, dytpe = {rcptdateTime.shape}, {rcptdateTime.dtype}") + logger.debug(f" sequence Num shape, dtype = {seqNum.shape}, {seqNum.dtype}") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}_profiles.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeRange', [str(dateTime.min()), str(dateTime.max())]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, and attributes") + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # rcptDatetime + obsspace.create_var('MetaData/rcptdateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'receipt Datetime') \ + .write_data(rcptdateTime) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Station Identification + obsspace.create_var('MetaData/stationID', dtype=stationID.dtype, fillval=stationID.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stationID) + + # Depth + obsspace.create_var('MetaData/depth', dtype=depth.dtype, fillval=depth.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Water depth') \ + .write_data(depth) + + # Sequence Number + obsspace.create_var('MetaData/sequenceNumber', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'Sequence Number') \ + .write_data(seqNum) + + # PreQC + obsspace.create_var('PreQC/waterTemperature', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + obsspace.create_var('PreQC/salinity', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + # ObsError + obsspace.create_var('ObsError/waterTemperature', dtype=ObsError_temp.dtype, fillval=ObsError_temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_temp) + + obsspace.create_var('ObsError/salinity', dtype=ObsError_saln.dtype, fillval=ObsError_saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_saln) + + # ObsValue + obsspace.create_var('ObsValue/waterTemperature', dtype=temp.dtype, fillval=temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('valid_range', np.array([-10.0, 50.0], dtype=np.float32)) \ + .write_attr('long_name', 'water Temperature') \ + .write_data(temp) + + obsspace.create_var('ObsValue/salinity', dtype=saln.dtype, fillval=saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('valid_range', np.array([0.0, 45.0], dtype=np.float32)) \ + .write_attr('long_name', 'salinity') \ + .write_data(saln) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: {running_time} \ + seconds") + + logger.debug(f"All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_insitu_profile_tesac.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py new file mode 100755 index 000000000..e0d9e0c10 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +# (C) Copyright 2024 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import numpy.ma as ma +import os +import argparse +import math +import calendar +import time +import copy +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger +import warnings +# suppress warnings +warnings.filterwarnings('ignore') + + +def Compute_sequenceNumber(lon): + + lon_u, seqNum = np.unique(lon, return_inverse=True) + seqNum = seqNum.astype(np.int32) + logger.debug(f"Len of Sequence Number: {len(seqNum)}") + + return seqNum + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + # General Information + converter = 'BUFR to IODA Converter' + platform_description = 'Surface obs from TRACKOB: temperature and salinity' + + bufrfile = f"{cycle_datetime}-{cycle_type}.t{hh}z.{data_format}.tm00.bufr_d" + DATA_PATH = os.path.join(dump_dir, bufrfile) + logger.debug(f"{bufrfile}, {DATA_PATH}") + + # ========================================== + # Make the QuerySet for all the data we want + # ========================================== + start_time = time.time() + + logger.debug(f"Making QuerySet ...") + q = bufr.QuerySet() + + # MetaData + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('ryear', '*/RCYR') + q.add('rmonth', '*/RCMO') + q.add('rday', '*/RCDY') + q.add('rhour', '*/RCHR') + q.add('rminute', '*/RCMI') + q.add('stationID', '*/RPID') + q.add('latitude', '*/CLAT') + q.add('longitude', '*/CLON') + + # ObsValue + q.add('temp', '*/BTOCN/STMP') + q.add('saln', '*/BTOCN/SALN') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ========================================================== + # Open the BUFR file and execute the QuerySet + # Use the ResultSet returned to get numpy arrays of the data + # ========================================================== + start_time = time.time() + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + logger.debug(f" ... Executing QuerySet: get MetaData ...") + dateTime = r.get_datetime('year', 'month', 'day', 'hour', 'minute', group_by='depth') + dateTime = dateTime.astype(np.int64) + rcptdateTime = r.get_datetime('ryear', 'rmonth', 'rday', 'rhour', 'rminute', group_by='depth') + rcptdateTime = rcptdateTime.astype(np.int64) + stationID = r.get('stationID', group_by='depth') + lat = r.get('latitude', group_by='depth') + lon = r.get('longitude', group_by='depth') + + # ObsValue + logger.debug(f" ... Executing QuerySet: get ObsValue ...") + temp = r.get('temp', group_by='depth') + temp -= 273.15 + saln = r.get('saln', group_by='depth') + + # Add mask based on min, max values + mask = ((temp > -10.0) & (temp <= 50.0)) & ((saln >= 0.0) & (saln <= 45.0)) + lat = lat[mask] + lon = lon[mask] + stationID = stationID[mask] + dateTime = dateTime[mask] + rcptdateTime = rcptdateTime[mask] + temp = temp[mask] + saln = saln[mask] + + # ObsError + logger.debug(f"Generating ObsError array with constant value (instrument error)...") + ObsError_temp = np.float32(np.ma.masked_array(np.full((len(mask)), 0.30))) + ObsError_saln = np.float32(np.ma.masked_array(np.full((len(mask)), 1.00))) + + # PreQC + logger.debug(f"Generating PreQC array with 0...") + PreQC = (np.ma.masked_array(np.full((len(mask)), 0))).astype(np.int32) + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + + # ================================================== + # Check values of BUFR variables, dimension and type + # ================================================== + logger.debug(f" temp min, max, length, dtype = {temp.min()}, {temp.max()}, {len(temp)}, {temp.dtype}") + logger.debug(f" saln min, max, length, dtype = {saln.min()}, {saln.max()}, {len(saln)}, {saln.dtype}") + logger.debug(f" lon min, max, length, dtype = {lon.min()}, {lon.max()}, {len(lon)}, {lon.dtype}") + logger.debug(f" lat min, max, length, dtype = {lat.min()}, {lat.max()}, {len(lat)}, {lat.dtype}") + logger.debug(f" PreQC min, max, length, dtype = {PreQC.min()}, {PreQC.max()}, {len(PreQC)}, {PreQC.dtype}") + logger.debug(f" ObsError_temp min, max, length, dtype = {ObsError_temp.min()}, {ObsError_temp.max()}, {len(ObsError_temp)}, {ObsError_temp.dtype}") + logger.debug(f" ObsError_saln min, max, length, dtype = {ObsError_saln.min()}, {ObsError_saln.max()}, {len(ObsError_saln)}, {ObsError_saln.dtype}") + logger.debug(f" stationID shape, dtype = {stationID.shape}, {stationID.astype(str).dtype}") + logger.debug(f" dateTime shape, dtype = {dateTime.shape}, {dateTime.dtype}") + logger.debug(f" rcptdateTime shape, dytpe = {rcptdateTime.shape}, {rcptdateTime.dtype}") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}_profiles.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeRange', [str(dateTime.min()), str(dateTime.max())]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, and attributes") + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # rcptDatetime + obsspace.create_var('MetaData/rcptdateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'receipt Datetime') \ + .write_data(rcptdateTime) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Station Identification + obsspace.create_var('MetaData/stationID', dtype=stationID.dtype, fillval=stationID.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stationID) + + # PreQC + obsspace.create_var('PreQC/seaSurfaceTemperature', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + obsspace.create_var('PreQC/seaSurfaceSalinity', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + # ObsError + obsspace.create_var('ObsError/seaSurfaceTemperature', dtype=ObsError_temp.dtype, fillval=ObsError_temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_temp) + + obsspace.create_var('ObsError/seaSurfaceSalinity', dtype=ObsError_saln.dtype, fillval=ObsError_saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_saln) + + # ObsValue + obsspace.create_var('ObsValue/seaSurfaceTemperature', dtype=temp.dtype, fillval=temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('valid_range', np.array([-10.0, 50.0], dtype=np.float32)) \ + .write_attr('long_name', 'Sea Surface Temperature') \ + .write_data(temp) + + obsspace.create_var('ObsValue/seaSurfaceSalinity', dtype=saln.dtype, fillval=saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('valid_range', np.array([0.0, 45.0], dtype=np.float32)) \ + .write_attr('long_name', 'Sea Surface Salinity') \ + .write_data(saln) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: {running_time} \ + seconds") + + logger.debug(f"All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_insitu_surface_trkob.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py b/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py index 0748e20f6..783b58197 100755 --- a/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py +++ b/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py @@ -15,7 +15,7 @@ def gen_bufr_json(config, template, output): # read in templated JSON and do substitution - logger.info(f"Using {template} as input") + logger.info(f"Using {template} as input {config}") bufr_config = parse_j2yaml(template, config) # write out JSON json_object = json.dumps(bufr_config, indent=4) @@ -29,8 +29,9 @@ def gen_bufr_json(config, template, output): parser.add_argument('-t', '--template', type=str, help='Input JSON template', required=True) parser.add_argument('-o', '--output', type=str, help='Output JSON file', required=True) args = parser.parse_args() - # get the config from your environment + # get the config from your environmentgg config = cast_strdict_as_dtypedict(os.environ) + logger.info(f"Config: {config}") # we need to add in current cycle from PDYcyc config['current_cycle'] = add_to_datetime(config['PDY'], to_timedelta(f"{config['cyc']}H")) # call the parsing function From 8419802bfb6fe1518f8c0d4e227fd877bf1c1329 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Thu, 8 Feb 2024 14:42:03 -0600 Subject: [PATCH 02/28] fix coding norm --- scripts/exglobal_prep_ocean_obs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 68307ecf5..976ce181c 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -61,7 +61,7 @@ def bufr2ioda(obtype, PDY, cyc, RUN, COMIN_OBS, COMOUT_OBS): json_output_file = os.path.join(COMOUT_OBS, f"{obtype}_{datetime_to_YMDH(cdateDatetime)}.json") filename = 'bufr2ioda_' + obtype + '.json' - template = os.path.join(JSON_TMPL_DIR,filename) + template = os.path.join(JSON_TMPL_DIR, filename) # Generate cycle specific json from TEMPLATE gen_bufr_json(config, template, json_output_file) From 7ef2abf9ba2d8da1a2347006f7d4ad1f724f953a Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Thu, 8 Feb 2024 14:44:47 -0600 Subject: [PATCH 03/28] fix coding norm --- ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py index e0d9e0c10..f6410e654 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py @@ -274,7 +274,7 @@ def bufr_to_ioda(config, logger): with open(args.config, "r") as json_file: config = json.load(json_file) - bufr_to_ioda(config, logger) + bufr_to_ioda(config, logger) end_time = time.time() running_time = end_time - start_time From 536d9c2eab73c6da370d19be38239626eaec13b1 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Thu, 8 Feb 2024 14:46:15 -0600 Subject: [PATCH 04/28] fix coding norm --- ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py index f6410e654..6ca99d313 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py @@ -274,7 +274,7 @@ def bufr_to_ioda(config, logger): with open(args.config, "r") as json_file: config = json.load(json_file) - bufr_to_ioda(config, logger) + bufr_to_ioda(config, logger) end_time = time.time() running_time = end_time - start_time From 02fe4e6ed4f010d91a41f2e08b96349687118136 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Tue, 13 Feb 2024 14:18:28 -0600 Subject: [PATCH 05/28] cleaned up obsprep_config.yaml: added type, removed output_file --- parm/soca/obsprep/obsprep_config.yaml | 61 +++++++++++++++------------ scripts/exglobal_prep_ocean_obs.py | 2 +- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/parm/soca/obsprep/obsprep_config.yaml b/parm/soca/obsprep/obsprep_config.yaml index f25715ac6..c98632ada 100644 --- a/parm/soca/obsprep/obsprep_config.yaml +++ b/parm/soca/obsprep/obsprep_config.yaml @@ -3,22 +3,24 @@ observations: - obs space: name: sss_smap dmpdir subdir: ocean/sss - dmpdir regex: SMAP_L2B_SSS_NRT_*.h5 provider: SMAP + type: nc + dmpdir regex: 'SMAP_L2B_SSS_NRT_*.h5' + - obs space: name: sss_smos provider: SMOS dmpdir subdir: ocean/sss - output file: sss_smos.ioda.nc - dmpdir regex: SM_OPER_MIR_OSUDP2_*.nc + type: nc + dmpdir regex: 'SM_OPER_MIR_OSUDP2_*.nc' # ADT - obs space: name: adt_rads_all dmpdir subdir: ocean/adt - dmpdir regex: rads_adt_*.nc provider: RADS - output file: adt_rads_all.nc4 + type: nc + dmpdir regex: 'rads_adt_*.nc' window: back: 8 # look back 8 six-hourly obs dumps forward: 1 # look forward 1 six-hourly bin @@ -28,21 +30,22 @@ observations: name: icec_amsr2_north provider: AMSR2 dmpdir subdir: ocean/icec - output file: icec_amsr2_north.ioda.nc - dmpdir regex: AMSR2-SEAICE-NH_v2r2_GW1_s*.nc + type: nc + dmpdir regex: 'AMSR2-SEAICE-NH_v2r2_GW1_s*.nc' + - obs space: name: icec_amsr2_south provider: AMSR2 dmpdir subdir: ocean/icec - output file: icec_amsr2_south.ioda.nc - dmpdir regex: AMSR2-SEAICE-SH_v2r2_GW1_s*.nc + type: nc + dmpdir regex: 'AMSR2-SEAICE-SH_v2r2_GW1_s*.nc' # SST - obs space: name: sst_avhrr_ma_l3u provider: GHRSST - dmpdir subdir: 'ocean/sst' - output file: sst_avhrr_ma_l3u.ioda.nc + dmpdir subdir: ocean/sst + type: nc dmpdir regex: '*-L3U_GHRSST-SSTsubskin-AVHRRF_MA-ACSPO_V*.nc' bounds: units: C @@ -55,8 +58,8 @@ observations: - obs space: name: sst_avhrr_mb_l3u provider: GHRSST - dmpdir subdir: 'ocean/sst' - output file: sst_avhrr_mb_l3u.ioda.nc + dmpdir subdir: ocean/sst + type: nc dmpdir regex: '*-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V*.nc' bounds: units: C @@ -69,8 +72,8 @@ observations: - obs space: name: sst_avhrr_mc_l3u provider: GHRSST - dmpdir subdir: 'ocean/sst' - output file: sst_avhrr_mc_l3u.ioda.nc + dmpdir subdir: ocean/sst + type: nc dmpdir regex: '*-L3U_GHRSST-SSTsubskin-AVHRRF_MC-ACSPO_V*.nc' bounds: units: C @@ -84,7 +87,7 @@ observations: name: sst_viirs_npp_l3u provider: GHRSST dmpdir subdir: 'ocean/sst' - output file: sst_viirs_npp_l3u.ioda.nc + type: nc dmpdir regex: '*-L3U_GHRSST-SSTsubskin-VIIRS_NPP-ACSPO_V*.nc' bounds: units: C @@ -97,8 +100,8 @@ observations: - obs space: name: sst_viirs_n20_l3u provider: GHRSST - dmpdir subdir: 'ocean/sst' - output file: sst_viirs_n20_l3u.ioda.nc + dmpdir subdir: ocean/sst + type: nc dmpdir regex: '*-L3U_GHRSST-SSTsubskin-VIIRS_N20-ACSPO_V*.nc' bounds: units: C @@ -111,8 +114,8 @@ observations: - obs space: name: sst_abi_g16_l3c provider: GHRSST - dmpdir subdir: 'ocean/sst' - output file: sst_abi_g16_l3c.ioda.nc + dmpdir subdir: ocean/sst + type: nc dmpdir regex: '*-L3C_GHRSST-SSTsubskin-ABI_G16-ACSPO_V*.nc' bounds: units: C @@ -125,8 +128,8 @@ observations: - obs space: name: sst_abi_g17_l3c provider: GHRSST - dmpdir subdir: 'ocean/sst' - output file: sst_abi_g17_l3c.ioda.nc + dmpdir subdir: ocean/sst + type: nc dmpdir regex: '*-L3C_GHRSST-SSTsubskin-ABI_G17-ACSPO_V*.nc' bounds: units: C @@ -139,8 +142,8 @@ observations: - obs space: name: sst_ahi_h08_l3c provider: GHRSST - dmpdir subdir: 'ocean/sst' - output file: sst_ahi_h08_l3c.ioda.nc + dmpdir subdir: ocean/sst + type: nc dmpdir regex: '*-L3C_GHRSST-SSTsubskin-AHI_H08-ACSPO_V*.nc' bounds: units: C @@ -149,21 +152,25 @@ observations: binning: stride: 15 min number of obs: 10 + +# in situ - obs space: name: insitu_profile_bathy provider: GTS dmpdir subdir: atmos - output file: 'bathy_profile.ioda.nc' + type: bufr dmpdir regex: 'gdas.*.bathy.*.bufr_d' + - obs space: name: insitu_profile_tesac provider: GTS dmpdir subdir: atmos - output file: 'tesac_profile.ioda.nc' + type: bufr dmpdir regex: 'gdas.*.tesac.*.bufr_d' + - obs space: name: insitu_surface_trkob provider: GTS dmpdir subdir: atmos - output file: 'trkob_surface.ioda.nc' + type: bufr dmpdir regex: 'gdas.*.trkob.*.bufr_d' diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 976ce181c..0d8d332ad 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -136,7 +136,7 @@ def run_netcdf_to_ioda(obsspace_to_convert): outputFilename = f"gdas.t{cyc}z.{obs_space_name}.{PDY}{cyc}.nc4" obsprepSpace['output file'] = outputFilename - if obsprepSpaceName.split('_')[0] == 'insitu': + if obsprepSpace['type'] == 'bufr': bufr2ioda(obsprepSpaceName, PDY, cyc, RUN, COMIN_OBS, COMOUT_OBS) else: iodaYamlFilename = obsprepSpaceName + '2ioda.yaml' From 30eeae7297908abaa0564f5a0b789c59f37b5fa7 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Wed, 14 Feb 2024 11:54:54 -0600 Subject: [PATCH 06/28] copy BUFR IODA output to ROTDIR --- scripts/exglobal_prep_ocean_obs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 0d8d332ad..b6546c94e 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -49,17 +49,17 @@ def bufr2ioda(obtype, PDY, cyc, RUN, COMIN_OBS, COMOUT_OBS): - logger.info(f"Process {obtype} for {RUN}.{PDY}/{cyc} from {COMIN_OBS} to {COMOUT_OBS}") + logger.info(f"Process {obtype} for {RUN}.{PDY}/{cyc} from {COMIN_OBS} to {COMIN_OBS}") # Load configuration config = { 'RUN': RUN, 'current_cycle': cdateDatetime, 'DMPDIR': COMIN_OBS, - 'COM_OBS': COMOUT_OBS, + 'COM_OBS': COMIN_OBS, } - json_output_file = os.path.join(COMOUT_OBS, f"{obtype}_{datetime_to_YMDH(cdateDatetime)}.json") + json_output_file = os.path.join(COMIN_OBS, f"{obtype}_{datetime_to_YMDH(cdateDatetime)}.json") filename = 'bufr2ioda_' + obtype + '.json' template = os.path.join(JSON_TMPL_DIR, filename) @@ -137,7 +137,9 @@ def run_netcdf_to_ioda(obsspace_to_convert): obsprepSpace['output file'] = outputFilename if obsprepSpace['type'] == 'bufr': - bufr2ioda(obsprepSpaceName, PDY, cyc, RUN, COMIN_OBS, COMOUT_OBS) + bufr2ioda(obsprepSpaceName, PDY, cyc, RUN, COMIN_OBS, COMIN_OBS) + files_to_save.append([obsprepSpace['output file'], + os.path.join(COMOUT_OBS, obsprepSpace['output file'])]) else: iodaYamlFilename = obsprepSpaceName + '2ioda.yaml' save_as_yaml(obsprepSpace, iodaYamlFilename) From 75ffc6f748c6d22f6ad377505de2dd8a5c119f0c Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Wed, 14 Feb 2024 11:56:44 -0600 Subject: [PATCH 07/28] updated BUFR IODA output filename --- ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.py | 2 +- ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.py | 2 +- ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.py index 71cc42d92..f67087e01 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_bathy.py @@ -166,7 +166,7 @@ def bufr_to_ioda(config, logger): # Create the dimensions dims = {'Location': np.arange(0, lat.shape[0])} - iodafile = f"{cycle_type}.t{hh}z.{data_type}_profiles.{data_format}.nc" + iodafile = f"{cycle_type}.t{hh}z.insitu_profile_{data_format}.{cycle_datetime}.nc4" OUTPUT_PATH = os.path.join(ioda_dir, iodafile) logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.py index 400aedbef..26e27a96a 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profile_tesac.py @@ -190,7 +190,7 @@ def bufr_to_ioda(config, logger): # Create the dimensions dims = {'Location': np.arange(0, lat.shape[0])} - iodafile = f"{cycle_type}.t{hh}z.{data_type}_profiles.{data_format}.nc" + iodafile = f"{cycle_type}.t{hh}z.insitu_profile_{data_format}.{cycle_datetime}.nc4" OUTPUT_PATH = os.path.join(ioda_dir, iodafile) logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py index 6ca99d313..3b9a0a3c1 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_trkob.py @@ -83,6 +83,7 @@ def bufr_to_ioda(config, logger): q.add('stationID', '*/RPID') q.add('latitude', '*/CLAT') q.add('longitude', '*/CLON') + q.add('depth', '*/BTOCN/DBSS') # ObsValue q.add('temp', '*/BTOCN/STMP') @@ -163,7 +164,7 @@ def bufr_to_ioda(config, logger): # Create the dimensions dims = {'Location': np.arange(0, lat.shape[0])} - iodafile = f"{cycle_type}.t{hh}z.{data_type}_profiles.{data_format}.nc" + iodafile = f"{cycle_type}.t{hh}z.insitu_surface_{data_format}.{cycle_datetime}.nc4" OUTPUT_PATH = os.path.join(ioda_dir, iodafile) logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") From 7623bd9a236298df1a37c29a919e7bd49e32899c Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Fri, 16 Feb 2024 15:13:59 -0600 Subject: [PATCH 08/28] all the monthly and fixed and drifting buoys added --- parm/soca/obs/obs_list.yaml | 13 +++++ parm/soca/obsprep/obsprep_config.yaml | 68 ++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index 8ed8309aa..2c4f488df 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -15,6 +15,19 @@ observers: # Ice concentration - !INC ${OBS_YAML_DIR}/icec_amsr2_north.yaml - !INC ${OBS_YAML_DIR}/icec_amsr2_south.yaml + +# in situ: monthly - !INC ${OBS_YAML_DIR}/insitu_profile_bathy.yaml +- !INC ${OBS_YAML_DIR}/insitu_profile_argo.yaml +- !INC ${OBS_YAML_DIR}/insitu_profile_glider.yaml - !INC ${OBS_YAML_DIR}/insitu_profile_tesac.yaml +- !INC ${OBS_YAML_DIR}/insitu_profile_marinemammal.yaml +- !INC ${OBS_YAML_DIR}/insitu_profile_xbtctd.yaml +- !INC ${OBS_YAML_DIR}/insitu_surface_altkob.yaml - !INC ${OBS_YAML_DIR}/insitu_surface_trkob.yaml + +# in situ: daily +- !INC ${OBS_YAML_DIR}/insitu_profile_dbuoy.yaml +- !INC ${OBS_YAML_DIR}/insitu_profile_dbuoyb.yaml +- !INC ${OBS_YAML_DIR}/insitu_profile_mbuoy.yaml +- !INC ${OBS_YAML_DIR}/insitu_profile_mbuoyb.yaml diff --git a/parm/soca/obsprep/obsprep_config.yaml b/parm/soca/obsprep/obsprep_config.yaml index c98632ada..bb6062633 100644 --- a/parm/soca/obsprep/obsprep_config.yaml +++ b/parm/soca/obsprep/obsprep_config.yaml @@ -153,7 +153,7 @@ observations: stride: 15 min number of obs: 10 -# in situ +# in situ: monthly - obs space: name: insitu_profile_bathy provider: GTS @@ -161,6 +161,20 @@ observations: type: bufr dmpdir regex: 'gdas.*.bathy.*.bufr_d' +- obs space: + name: insitu_profile_argo + provider: GTS + dmpdir subdir: atmos + type: bufr + dmpdir regex: 'gdas.*.argo.*.bufr_d' + +- obs space: + name: insitu_profile_glider + provider: GTS + dmpdir subdir: atmos + type: bufr + dmpdir regex: 'gdas.*.glider.*.bufr_d' + - obs space: name: insitu_profile_tesac provider: GTS @@ -168,9 +182,61 @@ observations: type: bufr dmpdir regex: 'gdas.*.tesac.*.bufr_d' +- obs space: + name: insitu_profile_marinemammal + provider: GTS + dmpdir subdir: atmos + type: bufr + dmpdir regex: 'gdas.*.marinemammal.*.bufr_d' + +- obs space: + name: insitu_profile_xbtctd + provider: GTS + dmpdir subdir: atmos + type: bufr + dmpdir regex: 'gdas.*.xbtctd.*.bufr_d' + +- obs space: + name: insitu_surface_altkob + provider: GTS + dmpdir subdir: atmos + type: bufr + dmpdir regex: 'gdas.*.altkob.*.bufr_d' + - obs space: name: insitu_surface_trkob provider: GTS dmpdir subdir: atmos type: bufr dmpdir regex: 'gdas.*.trkob.*.bufr_d' + +# in situ: daily +- obs space: + name: insitu_profile_dbuoy + provider: GTS + dmpdir subdir: atmos + type: bufr + dmpdir regex: 'gdas.*.dbuoy.*.bufr_d' + +- obs space: + name: insitu_profile_dbuoyb + provider: GTS + dmpdir subdir: atmos + type: bufr + dmpdir regex: 'gdas.*.dbuoyb.*.bufr_d' + +- obs space: + name: insitu_profile_mbuoy + provider: GTS + dmpdir subdir: atmos + type: bufr + dmpdir regex: 'gdas.*.mbuoy.*.bufr_d' + +- obs space: + name: insitu_profile_mbuoyb + provider: GTS + dmpdir subdir: atmos + type: bufr + dmpdir regex: 'gdas.*.mbuoyb.*.bufr_d' + + From f6b93da0c7604e6178872d9c68f828e8d33f34ce Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Fri, 16 Feb 2024 15:15:03 -0600 Subject: [PATCH 09/28] ObsSpace yamls: all the monthly and fixed and drifting buoys added --- parm/soca/obs/config/insitu_profile_argo.yaml | 22 +++++++++++++++++++ .../soca/obs/config/insitu_profile_dbuoy.yaml | 22 +++++++++++++++++++ .../obs/config/insitu_profile_dbuoyb.yaml | 22 +++++++++++++++++++ .../obs/config/insitu_profile_glider.yaml | 22 +++++++++++++++++++ .../config/insitu_profile_marinemammal.yaml | 22 +++++++++++++++++++ .../soca/obs/config/insitu_profile_mbuoy.yaml | 22 +++++++++++++++++++ .../obs/config/insitu_profile_mbuoyb.yaml | 22 +++++++++++++++++++ .../obs/config/insitu_profile_xbtctd.yaml | 22 +++++++++++++++++++ .../obs/config/insitu_surface_altkob.yaml | 18 +++++++++++++++ 9 files changed, 194 insertions(+) create mode 100644 parm/soca/obs/config/insitu_profile_argo.yaml create mode 100644 parm/soca/obs/config/insitu_profile_dbuoy.yaml create mode 100644 parm/soca/obs/config/insitu_profile_dbuoyb.yaml create mode 100644 parm/soca/obs/config/insitu_profile_glider.yaml create mode 100644 parm/soca/obs/config/insitu_profile_marinemammal.yaml create mode 100644 parm/soca/obs/config/insitu_profile_mbuoy.yaml create mode 100644 parm/soca/obs/config/insitu_profile_mbuoyb.yaml create mode 100644 parm/soca/obs/config/insitu_profile_xbtctd.yaml create mode 100644 parm/soca/obs/config/insitu_surface_altkob.yaml diff --git a/parm/soca/obs/config/insitu_profile_argo.yaml b/parm/soca/obs/config/insitu_profile_argo.yaml new file mode 100644 index 000000000..cda8dc6a6 --- /dev/null +++ b/parm/soca/obs/config/insitu_profile_argo.yaml @@ -0,0 +1,22 @@ +obs space: + name: insitu_profile_argo + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_argo.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/insitu_profile_argo.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature, salinity] + observed variables: [waterTemperature, salinity] + io pool: + max pool size: 1 +obs operator: + name: Composite + components: + - name: InsituTemperature + variables: + - name: waterTemperature +obs error: + covariance model: diagonal diff --git a/parm/soca/obs/config/insitu_profile_dbuoy.yaml b/parm/soca/obs/config/insitu_profile_dbuoy.yaml new file mode 100644 index 000000000..3ab948b9d --- /dev/null +++ b/parm/soca/obs/config/insitu_profile_dbuoy.yaml @@ -0,0 +1,22 @@ +obs space: + name: insitu_profile_dbuoy + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_dbuoy.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/insitu_profile_dbuoy.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature, salinity] + observed variables: [waterTemperature, salinity] + io pool: + max pool size: 1 +obs operator: + name: Composite + components: + - name: InsituTemperature + variables: + - name: waterTemperature +obs error: + covariance model: diagonal diff --git a/parm/soca/obs/config/insitu_profile_dbuoyb.yaml b/parm/soca/obs/config/insitu_profile_dbuoyb.yaml new file mode 100644 index 000000000..cc11a33df --- /dev/null +++ b/parm/soca/obs/config/insitu_profile_dbuoyb.yaml @@ -0,0 +1,22 @@ +obs space: + name: insitu_profile_dbuoyb + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_dbuoyb.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/insitu_profile_dbuoyb.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature, salinity] + observed variables: [waterTemperature, salinity] + io pool: + max pool size: 1 +obs operator: + name: Composite + components: + - name: InsituTemperature + variables: + - name: waterTemperature +obs error: + covariance model: diagonal diff --git a/parm/soca/obs/config/insitu_profile_glider.yaml b/parm/soca/obs/config/insitu_profile_glider.yaml new file mode 100644 index 000000000..8d92d3630 --- /dev/null +++ b/parm/soca/obs/config/insitu_profile_glider.yaml @@ -0,0 +1,22 @@ +obs space: + name: insitu_profile_glider + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_glider.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/insitu_profile_glider.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature, salinity] + observed variables: [waterTemperature, salinity] + io pool: + max pool size: 1 +obs operator: + name: Composite + components: + - name: InsituTemperature + variables: + - name: waterTemperature +obs error: + covariance model: diagonal diff --git a/parm/soca/obs/config/insitu_profile_marinemammal.yaml b/parm/soca/obs/config/insitu_profile_marinemammal.yaml new file mode 100644 index 000000000..70a3aefe2 --- /dev/null +++ b/parm/soca/obs/config/insitu_profile_marinemammal.yaml @@ -0,0 +1,22 @@ +obs space: + name: insitu_profile_marinemammal + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_marinemammal.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/insitu_profile_marinemammal.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature, salinity] + observed variables: [waterTemperature, salinity] + io pool: + max pool size: 1 +obs operator: + name: Composite + components: + - name: InsituTemperature + variables: + - name: waterTemperature +obs error: + covariance model: diagonal diff --git a/parm/soca/obs/config/insitu_profile_mbuoy.yaml b/parm/soca/obs/config/insitu_profile_mbuoy.yaml new file mode 100644 index 000000000..e5e831084 --- /dev/null +++ b/parm/soca/obs/config/insitu_profile_mbuoy.yaml @@ -0,0 +1,22 @@ +obs space: + name: insitu_profile_mbuoy + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_mbuoy.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/insitu_profile_mbuoy.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature, salinity] + observed variables: [waterTemperature, salinity] + io pool: + max pool size: 1 +obs operator: + name: Composite + components: + - name: InsituTemperature + variables: + - name: waterTemperature +obs error: + covariance model: diagonal diff --git a/parm/soca/obs/config/insitu_profile_mbuoyb.yaml b/parm/soca/obs/config/insitu_profile_mbuoyb.yaml new file mode 100644 index 000000000..03f730aee --- /dev/null +++ b/parm/soca/obs/config/insitu_profile_mbuoyb.yaml @@ -0,0 +1,22 @@ +obs space: + name: insitu_profile_mbuoyb + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_mbuoyb.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/insitu_profile_mbuoyb.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature, salinity] + observed variables: [waterTemperature, salinity] + io pool: + max pool size: 1 +obs operator: + name: Composite + components: + - name: InsituTemperature + variables: + - name: waterTemperature +obs error: + covariance model: diagonal diff --git a/parm/soca/obs/config/insitu_profile_xbtctd.yaml b/parm/soca/obs/config/insitu_profile_xbtctd.yaml new file mode 100644 index 000000000..148f9b3d7 --- /dev/null +++ b/parm/soca/obs/config/insitu_profile_xbtctd.yaml @@ -0,0 +1,22 @@ +obs space: + name: insitu_profile_xbtctd + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_xbtctd.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/insitu_profile_xbtctd.${PDY}${cyc}.nc4 + simulated variables: [waterTemperature, salinity] + observed variables: [waterTemperature, salinity] + io pool: + max pool size: 1 +obs operator: + name: Composite + components: + - name: InsituTemperature + variables: + - name: waterTemperature +obs error: + covariance model: diagonal diff --git a/parm/soca/obs/config/insitu_surface_altkob.yaml b/parm/soca/obs/config/insitu_surface_altkob.yaml new file mode 100644 index 000000000..d39bf59b1 --- /dev/null +++ b/parm/soca/obs/config/insitu_surface_altkob.yaml @@ -0,0 +1,18 @@ +obs space: + name: insitu_surface_altkob + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_surface_altkob.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/insitu_surface_altkob.${PDY}${cyc}.nc4 + simulated variables: [seaSurfaceTemperature] + io pool: + max pool size: 1 +obs operator: + name: Identity + observation alias file: obsop_name_map.yaml +obs error: + covariance model: diagonal From d7d657f1873ea759bc8a004b5bd997b8318931c3 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Tue, 20 Feb 2024 12:03:24 -0600 Subject: [PATCH 10/28] arranged modules alphabetically --- scripts/exglobal_prep_ocean_obs.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index cb0d76e07..634c50cfe 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -7,8 +7,13 @@ from pathlib import Path import subprocess from soca import prep_marine_obs -from wxflow import YAMLFile, save_as_yaml, FileHandler, Logger -from wxflow import add_to_datetime, to_timedelta, datetime_to_YMDH +from wxflow import add_to_datetime, \ + datetime_to_YMDH, \ + FileHandler, \ + Logger, \ + save_as_yaml, \ + to_timedelta, \ + YAMLFile from gen_bufr2ioda_json import gen_bufr_json logger = Logger() @@ -22,7 +27,7 @@ cdateDatetime = datetime.strptime(PDY + cyc, '%Y%m%d%H') windowBeginDatetime = cdateDatetime - timedelta(hours=3) windowEndDatetime = cdateDatetime + timedelta(hours=3) -windowBegin = windowBeginDatetime.strftime('%Y-%m-%dT%H:%M:%SZ') + windowEnd = windowEndDatetime.strftime('%Y-%m-%dT%H:%M:%SZ') OCNOBS2IODAEXEC = os.getenv('OCNOBS2IODAEXEC') From e522587b512758565ec2e19da611f4b982008cf3 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Tue, 20 Feb 2024 12:09:07 -0600 Subject: [PATCH 11/28] fix coding norm --- scripts/exglobal_prep_ocean_obs.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 634c50cfe..dbb176954 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -7,13 +7,7 @@ from pathlib import Path import subprocess from soca import prep_marine_obs -from wxflow import add_to_datetime, \ - datetime_to_YMDH, \ - FileHandler, \ - Logger, \ - save_as_yaml, \ - to_timedelta, \ - YAMLFile +from wxflow import add_to_datetime, datetime_to_YMDH, FileHandler, Logger, save_as_yaml, to_timedelta, YAMLFile from gen_bufr2ioda_json import gen_bufr_json logger = Logger() From b644cc1dee8d939f828a178c0a036e4ba1584c63 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Wed, 21 Feb 2024 11:53:09 -0600 Subject: [PATCH 12/28] module import update --- scripts/exglobal_prep_ocean_obs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index dbb176954..cc41a7073 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -7,7 +7,13 @@ from pathlib import Path import subprocess from soca import prep_marine_obs -from wxflow import add_to_datetime, datetime_to_YMDH, FileHandler, Logger, save_as_yaml, to_timedelta, YAMLFile +from wxflow import (add_to_datetime, +datetime_to_YMDH, +FileHandler, +Logger, +save_as_yaml, +to_timedelta, +YAMLFile) from gen_bufr2ioda_json import gen_bufr_json logger = Logger() From 1dba0750564a44974544e964b9e3df950d761e4e Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Wed, 21 Feb 2024 11:55:56 -0600 Subject: [PATCH 13/28] module import update+1 --- scripts/exglobal_prep_ocean_obs.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index cc41a7073..6564e0342 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -7,12 +7,12 @@ from pathlib import Path import subprocess from soca import prep_marine_obs -from wxflow import (add_to_datetime, -datetime_to_YMDH, -FileHandler, -Logger, -save_as_yaml, -to_timedelta, +from wxflow import (add_to_datetime, +datetime_to_YMDH, +FileHandler, +Logger, +save_as_yaml, +to_timedelta, YAMLFile) from gen_bufr2ioda_json import gen_bufr_json From 60ecb97176ed9550d1263b29c3076ff3a40d2695 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Wed, 21 Feb 2024 11:58:04 -0600 Subject: [PATCH 14/28] module import update+2 --- scripts/exglobal_prep_ocean_obs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 6564e0342..3c2f16ac5 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -7,13 +7,15 @@ from pathlib import Path import subprocess from soca import prep_marine_obs -from wxflow import (add_to_datetime, +from wxflow import ( +add_to_datetime, datetime_to_YMDH, FileHandler, Logger, save_as_yaml, to_timedelta, -YAMLFile) +YAMLFile +) from gen_bufr2ioda_json import gen_bufr_json logger = Logger() From 2da25c83ea175f48bf8d39706ff176a6e75f8d90 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Wed, 21 Feb 2024 12:00:18 -0600 Subject: [PATCH 15/28] module import update+3 --- scripts/exglobal_prep_ocean_obs.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 3c2f16ac5..99e3ca9d7 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -8,13 +8,13 @@ import subprocess from soca import prep_marine_obs from wxflow import ( -add_to_datetime, -datetime_to_YMDH, -FileHandler, -Logger, -save_as_yaml, -to_timedelta, -YAMLFile + add_to_datetime, + datetime_to_YMDH, + FileHandler, + Logger, + save_as_yaml, + to_timedelta, + YAMLFile ) from gen_bufr2ioda_json import gen_bufr_json From b5ab1e8a9b50169d1d844480c57d2557718ade3a Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Wed, 21 Feb 2024 12:02:38 -0600 Subject: [PATCH 16/28] module import update+4 --- scripts/exglobal_prep_ocean_obs.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 99e3ca9d7..2bb1a59eb 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -8,13 +8,13 @@ import subprocess from soca import prep_marine_obs from wxflow import ( - add_to_datetime, - datetime_to_YMDH, - FileHandler, - Logger, - save_as_yaml, - to_timedelta, - YAMLFile + add_to_datetime, + datetime_to_YMDH, + FileHandler, + Logger, + save_as_yaml, + to_timedelta, + YAMLFile ) from gen_bufr2ioda_json import gen_bufr_json From 8dfc8695d997c124d6079d3cfb421a75bbfb7752 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Wed, 21 Feb 2024 12:05:31 -0600 Subject: [PATCH 17/28] module import update+5 --- scripts/exglobal_prep_ocean_obs.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 2bb1a59eb..9269e7812 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -8,13 +8,13 @@ import subprocess from soca import prep_marine_obs from wxflow import ( - add_to_datetime, - datetime_to_YMDH, - FileHandler, - Logger, - save_as_yaml, - to_timedelta, - YAMLFile + add_to_datetime, + datetime_to_YMDH, + FileHandler, + Logger, + save_as_yaml, + to_timedelta, + YAMLFile ) from gen_bufr2ioda_json import gen_bufr_json From 1a24a91f9c62413630b509b10788055f9b346d57 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Mon, 26 Feb 2024 10:43:01 -0600 Subject: [PATCH 18/28] fix prep obs failure --- parm/soca/obsprep/obsprep_config.yaml | 2 +- scripts/exglobal_prep_ocean_obs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/soca/obsprep/obsprep_config.yaml b/parm/soca/obsprep/obsprep_config.yaml index bb6062633..f47be1fa3 100644 --- a/parm/soca/obsprep/obsprep_config.yaml +++ b/parm/soca/obsprep/obsprep_config.yaml @@ -86,7 +86,7 @@ observations: - obs space: name: sst_viirs_npp_l3u provider: GHRSST - dmpdir subdir: 'ocean/sst' + dmpdir subdir: ocean/sst type: nc dmpdir regex: '*-L3U_GHRSST-SSTsubskin-VIIRS_NPP-ACSPO_V*.nc' bounds: diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 9269e7812..59f4137f4 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -29,7 +29,7 @@ cdateDatetime = datetime.strptime(PDY + cyc, '%Y%m%d%H') windowBeginDatetime = cdateDatetime - timedelta(hours=3) windowEndDatetime = cdateDatetime + timedelta(hours=3) - +windowBegin = windowBeginDatetime.strftime('%Y-%m-%dT%H:%M:%SZ') windowEnd = windowEndDatetime.strftime('%Y-%m-%dT%H:%M:%SZ') OCNOBS2IODAEXEC = os.getenv('OCNOBS2IODAEXEC') From 5a8c588916a73ff9c37afb028e6e754f47c48ed9 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Wed, 28 Feb 2024 08:12:02 -0600 Subject: [PATCH 19/28] update insitu yamls --- .../soca/obs/config/insitu_profile_bathy.yaml | 150 ++++++++++- .../soca/obs/config/insitu_profile_tesac.yaml | 242 ++++++++++++++++- .../config/insitu_profile_tesac_salinity.yaml | 253 ++++++++++++++++++ .../soca/obs/config/insitu_surface_trkob.yaml | 29 +- .../config/insitu_surface_trkob_salinity.yaml | 18 ++ parm/soca/obs/obs_list.yaml | 2 + 6 files changed, 682 insertions(+), 12 deletions(-) create mode 100644 parm/soca/obs/config/insitu_profile_tesac_salinity.yaml create mode 100644 parm/soca/obs/config/insitu_surface_trkob_salinity.yaml diff --git a/parm/soca/obs/config/insitu_profile_bathy.yaml b/parm/soca/obs/config/insitu_profile_bathy.yaml index ab08c20bc..5d8c45f19 100644 --- a/parm/soca/obs/config/insitu_profile_bathy.yaml +++ b/parm/soca/obs/config/insitu_profile_bathy.yaml @@ -8,15 +8,151 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/insitu_profile_bathy.${PDY}${cyc}.nc4 - simulated variables: [waterTemperature, salinity] - observed variables: [waterTemperature, salinity] + simulated variables: [waterTemperature] io pool: max pool size: 1 obs operator: - name: Composite - components: - - name: InsituTemperature - variables: - - name: waterTemperature + name: InsituTemperature obs error: covariance model: diagonal + +#-------------------------------------------------------------------------------------- +# START OF OBS FILTERS +# The QC filters used here are based on the document by IODE that can be found at +# https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf +#-------------------------------------------------------------------------------------- +obs filters: + + # Land Check + - filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_area_fraction} + minvalue: 0.5 +#-------------------------------------------------------------------------------------- +## Filters for T: +#-------------------------------------------------------------------------------------- + #------------------------------------------------------------------------------------ + ### Global range test + #------------------------------------------------------------------------------------ + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: -2.5 + maxvalue: 40.0 + + #------------------------------------------------------------------------------------ + ### Regional range tests + #------------------------------------------------------------------------------------ + + #### Red Sea + #------------------------------------------------------------------------------------ + #### + #### the document linked here describes Red sea as the are between 10N, 40E; + #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden. + #### A more reasonable choice seemed to be a box that extends from 10 N to + #### 30 N and 30 E to 45 East . + #------------------------------------------------------------------------------------ + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: 21.7 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 10 + maxvalue: 30 + - variable: + name: MetaData/longitude + minvalue: 30 + + #### Mediterranean Sea + #----------------------------------------------------------------------------- + ##### Area 1/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: 10.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 30 + maxvalue: 40 + - variable: + name: MetaData/longitude + minvalue: -6 + maxvalue: 40 + ##### Area 2/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: 10.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 40 + maxvalue: 41.5 + - variable: + name: MetaData/longitude + minvalue: 20 + maxvalue: 30 + ##### Area 3/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: 10.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 40 + maxvalue: 46 + - variable: + name: MetaData/longitude + minvalue: 0 + maxvalue: 20 + + #### Northwestern shelves + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: -2.0 + maxvalue: 24.0 + where: + - variable: + name: MetaData/latitude + minvalue: 50 + maxvalue: 60 + - variable: + name: MetaData/longitude + minvalue: -20 + maxvalue: 10 + #### Southwestern shelves + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: -2.0 + maxvalue: 30 + where: + - variable: + name: MetaData/latitude + minvalue: 25 + maxvalue: 50 + - variable: + name: MetaData/longitude + minvalue: -30 + maxvalue: 0 + + #### Arctic Sea + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: -1.92 + maxvalue: 25.0 + where: + - variable: + name: MetaData/latitude + minvalue: 60 + + - filter: Background Check + filter variables: [{name: waterTemperature}] + threshold: 5.0 + absolute threshold: 5.0 + diff --git a/parm/soca/obs/config/insitu_profile_tesac.yaml b/parm/soca/obs/config/insitu_profile_tesac.yaml index 7f40527b4..8012d1b7e 100644 --- a/parm/soca/obs/config/insitu_profile_tesac.yaml +++ b/parm/soca/obs/config/insitu_profile_tesac.yaml @@ -8,8 +8,8 @@ obs space: engine: type: H5File obsfile: !ENV ${DATA}/diags/insitu_profile_tesac.${PDY}${cyc}.nc4 - simulated variables: [waterTemperature, salinity] - observed variables: [waterTemperature, salinity] + simulated variables: [waterTemperature] #, salinity] + observed variables: [waterTemperature] #, salinity] io pool: max pool size: 1 obs operator: @@ -18,5 +18,239 @@ obs operator: - name: InsituTemperature variables: - name: waterTemperature -obs error: - covariance model: diagonal +#-------------------------------------------------------------------------------------- +# The lines below are commented out due to the composite obs operator failing looking +# for MetaData: air_pressure. +# A separate yaml file: insitu_profile_salinity.yaml is used for sailinity profiles +# +# - name: VertInterp +# observation alias file: ./obsop_name_map.yaml +# variables: +# - name: salinity +# vertical coornidate: sea_water_depth +# observation vertical coordinate: depth +# interpolation method: linear +#-------------------------------------------------------------------------------------- +#-------------------------------------------------------------------------------------- +# START OF OBS FILTERS +# The QC filters used here are based on the document by IODE that can be found at +# https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf +#-------------------------------------------------------------------------------------- +obs filters: + + # Land Check + - filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_area_fraction} + minvalue: 0.5 +#-------------------------------------------------------------------------------------- +## Filters for T: +#-------------------------------------------------------------------------------------- + #------------------------------------------------------------------------------------ + ### Global range test + #------------------------------------------------------------------------------------ + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: -2.5 + maxvalue: 40.0 + + #------------------------------------------------------------------------------------ + ### Regional range tests + #------------------------------------------------------------------------------------ + + #### Red Sea + #------------------------------------------------------------------------------------ + #### + #### the document linked here describes Red sea as the are between 10N, 40E; + #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden. + #### A more reasonable choice seemed to be a box that extends from 10 N to + #### 30 N and 30 E to 45 East . + #------------------------------------------------------------------------------------ + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: 21.7 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 10 + maxvalue: 30 + - variable: + name: MetaData/longitude + minvalue: 30 + maxvalue: 45 + + #### Mediterranean Sea + #----------------------------------------------------------------------------- + ##### Area 1/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: 10.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 30 + maxvalue: 40 + - variable: + name: MetaData/longitude + minvalue: -6 + maxvalue: 40 + ##### Area 2/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: 10.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 40 + maxvalue: 41.5 + - variable: + name: MetaData/longitude + minvalue: 20 + maxvalue: 30 + ##### Area 3/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: 10.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 40 + maxvalue: 46 + - variable: + name: MetaData/longitude + minvalue: 0 + maxvalue: 20 + + #### Northwestern shelves + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: -2.0 + maxvalue: 24.0 + where: + - variable: + name: MetaData/latitude + minvalue: 50 + maxvalue: 60 + - variable: + name: MetaData/longitude + minvalue: -20 + maxvalue: 10 + #### Southwestern shelves + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: -2.0 + maxvalue: 30 + where: + - variable: + name: MetaData/latitude + minvalue: 25 + maxvalue: 50 + - variable: + name: MetaData/longitude + minvalue: -30 + maxvalue: 0 + + #### Arctic Sea + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: waterTemperature}] + minvalue: -1.92 + maxvalue: 25.0 + where: + - variable: + name: MetaData/latitude + minvalue: 60 + + - filter: Background Check + filter variables: [{name: waterTemperature}] + threshold: 5.0 + absolute threshold: 5.0 + +# TODO: uncomment when there is data +# #------------------------------------------------------------------------------- +# ### Spike test +# #----------------------------------------------------------------------------- +# - filter: Create Diagnostic Flags +# filter variables: +# - name: waterTemperature +# # - name: salinity +# flags: +# - name: spike +# initial value: false +# - name: step +# initial value: false +# +# - filter: Spike and Step Check +# filter variables: +# - name: ObsValue/waterTemperature +# dependent: ObsValue/waterTemperature # dy/ +# independent: MetaData/depth # dx +# count spikes: true +# count steps: true +# tolerance: +# nominal value: 10 # K nominal, in the case of temperature (not really) +# gradient: 0.1 # K/m - if dy/dx greater, could be a spike +# gradient x resolution: 10 # m - can't know dx to better precision +# factors: [1,1,0.5] # multiply tolerance, for ranges bounded by... +# x boundaries: [0,500,500] # ...these values of x (depth in m) +# boundary layer: +# x range: [0.0,300.0] # when bounded by these x values (depth in m)... +# step tolerance range: [0.0,-2.0] # ...relax tolerance for steps in boundary layer... +# maximum x interval: [50.0,100.0] # ...and ignore level if dx greater than this +# action: +# name: reject +# +# #### Count spikes +# #----------------------------------------------------------------------------- +# - filter: Variable Assignment # create derived obs value containing only T spikes +# assignments: +# - name: DerivedMetaData/waterTemperature_spikes +# type: int +# function: +# name: IntObsFunction/ProfileLevelCount +# options: +# where: +# - variable: +# name: DiagnosticFlags/spike/waterTemperature +# value: is_true +# +# #### Count steps +# #----------------------------------------------------------------------------- +# - filter: Variable Assignment # create derived obs value containing only T steps +# assignments: +# - name: DerivedMetaData/waterTemperature_steps +# type: int +# function: +# name: IntObsFunction/ProfileLevelCount +# options: +# where: +# - variable: +# name: DiagnosticFlags/step/waterTemperature +# value: is_true +# #### Count total rejections +# #----------------------------------------------------------------------------- +# - filter: Variable Assignment # compute sum 2*spikes+steps +# assignments: +# - name: DerivedMetaData/waterTemperature_rejections +# type: int +# function: +# name: IntObsFunction/LinearCombination +# options: +# variables: [DerivedMetaData/waterTemperature_spikes, DerivedMetaData/waterTemperature_steps] +# coefs: [2,1] +# #### Reject entire profile if total rejctions > threshold +# #----------------------------------------------------------------------------- +# - filter: Perform Action # reject whole profile if 2*spikes+steps>=rejection threshold +# where: +# - variable: +# name: DerivedMetaData/waterTemperature_rejections +# minvalue: 3 +# action: +# name: reject +# diff --git a/parm/soca/obs/config/insitu_profile_tesac_salinity.yaml b/parm/soca/obs/config/insitu_profile_tesac_salinity.yaml new file mode 100644 index 000000000..30938849b --- /dev/null +++ b/parm/soca/obs/config/insitu_profile_tesac_salinity.yaml @@ -0,0 +1,253 @@ +obs space: + name: insitu_profile_tesac_salinity + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_profile_tesac.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/insitu_profile_tesac_salinity.${PDY}${cyc}.nc4 + simulated variables: [salinity] + #observed variables: [waterTemperature, salinity] + io pool: + max pool size: 1 +obs operator: +#---------------------------------------------------------------------------------------- +# composite obs operator is throwing an error looking for MetaData: air_pressure +# Hence the lines below and above are commented out +# name: Composite +# components: +# - name: InsituTemperature +# variables: +# - name: waterTemperature +# - name: VertInterp +# observation alias file: ./obsop_name_map.yaml +# variables: +# - name: salinity +# vertical coornidate: sea_water_depth +# observation vertical coordinate: depth +# interpolation method: linear +#--------------------------------------------------------------------------------------- + name: VertInterp + observation alias file: ./obsop_name_map.yaml + vertical coordinate: sea_water_depth + observation vertical coordinate: depth + interpolation method: linear +obs error: + covariance model: diagonal + +#-------------------------------------------------------------------------------------- +# START OF OBS FILTERS +# The QC filters used here are based on the document by IODE that can be found at +# https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf +#-------------------------------------------------------------------------------------- +obs filters: + + # Land Check + - filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_area_fraction} + minvalue: 0.5 + +#------------------------------------------------------------------------------- +## Filters for S: +#------------------------------------------------------------------------------- + #----------------------------------------------------------------------------- + ### Global range test + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 2.0 + maxvalue: 41.0 + + #----------------------------------------------------------------------------- + ### Regional range test + #----------------------------------------------------------------------------- + #### Red Sea + #----------------------------------------------------------------------------- + #### + #### the document linked here describes Red sea as the are between 10N, 40E; + #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden. + #### A more reasonable choice seemed to be a box that extends from 10 N to + #### 30 N and 30 E to 45 East . + + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 2.0 + maxvalue: 41.0 + where: + - variable: + name: MetaData/latitude + minvalue: 10 + maxvalue: 30 + - variable: + name: MetaData/longitude + minvalue: 30 + maxvalue: 45 + + #### Mediterranean Sea + #----------------------------------------------------------------------------- + ##### Area 1/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 2.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 30 + maxvalue: 40 + - variable: + name: MetaData/longitude + minvalue: -6 + maxvalue: 40 + ##### Area 2/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 2.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 40 + maxvalue: 41.5 + - variable: + name: MetaData/longitude + minvalue: 20 + maxvalue: 30 + ##### Area 3/3 for Mediterranean Sea + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 2.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 40 + maxvalue: 46 + - variable: + name: MetaData/longitude + minvalue: 0 + maxvalue: 20 + + + #### Northwestern shelves + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 0.0 + maxvalue: 37.0 + where: + - variable: + name: MetaData/latitude + minvalue: 50 + maxvalue: 60 + - variable: + name: MetaData/longitude + minvalue: -20 + maxvalue: 10 + + #### Southwestern shelves + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 0.0 + maxvalue: 38 + where: + - variable: + name: MetaData/latitude + minvalue: 25 + maxvalue: 50 + - variable: + name: MetaData/longitude + minvalue: -30 + maxvalue: 0 + + #### Arctic Sea + #----------------------------------------------------------------------------- + - filter: Bounds Check + filter variables: [{name: salinity}] + minvalue: 2.0 + maxvalue: 40.0 + where: + - variable: + name: MetaData/latitude + minvalue: 60 + + - filter: Background Check + filter variables: [{name: salinity}] + threshold: 5.0 + absolute threshold: 5.0 + +# TODO: uncomment when there is data +# #------------------------------------------------------------------------------- +# ### Spike test +# #----------------------------------------------------------------------------- +# - filter: Spike and Step Check +# filter variables: +# - name: ObsValue/salinity +# dependent: ObsValue/salinity # dy/ +# independent: MetaData/depth # dx +# count spikes: true +# count steps: true +# tolerance: +# nominal value: 1.0 # PSU nominal, in the case of salinity (not really) +# threshold: 0.6 # weird salinity thing +# factors: [1,1,0.2] # multiply tolerance, for ranges bounded by... +# x boundaries: [0,200,600] # ...these values of x (depth in m) +# boundary layer: +# x range: [0.0,300.0] # when bounded by these x values (depth in m)... +# maximum x interval: [50.0,100.0] # ...and ignore level if dx greater than this +# action: +# name: reject +# +# #### Count spikes +# #----------------------------------------------------------------------------- +# - filter: Variable Assignment # create derived obs value containing only S spikes +# assignments: +# - name: DerivedMetaData/salinity_spikes +# type: int +# function: +# name: IntObsFunction/ProfileLevelCount +# options: +# where: +# - variable: +# name: DiagnosticFlags/spike/salinity +# value: is_true +# #### Count steps +# #----------------------------------------------------------------------------- +# - filter: Variable Assignment # create derived obs value containing only S steps +# assignments: +# - name: DerivedMetaData/salinity_steps +# type: int +# function: +# name: IntObsFunction/ProfileLevelCount +# options: +# where: +# - variable: +# name: DiagnosticFlags/step/salinity +# value: is_true +# #### Count total rejections +# #----------------------------------------------------------------------------- +# - filter: Variable Assignment # compute sum 2*spikes+steps +# assignments: +# - name: DerivedMetaData/salinity_rejections +# type: int +# function: +# name: IntObsFunction/LinearCombination +# options: +# variables: [DerivedMetaData/salinity_spikes, DerivedMetaData/salinity_steps] +# coefs: [2,1] +# #### Reject entire profile if total rejctions > threshold +# #----------------------------------------------------------------------------- +# - filter: Perform Action # reject whole profile if 2*spikes+steps>=rejection threshold +# where: +# - variable: +# name: DerivedMetaData/salinity_rejections +# minvalue: 3 +# action: +# name: reject +# #------------------------------------------------------------------------------- +# ### End of Spike test +# #----------------------------------------------------------------------------- diff --git a/parm/soca/obs/config/insitu_surface_trkob.yaml b/parm/soca/obs/config/insitu_surface_trkob.yaml index f62e83988..9e101e5c0 100644 --- a/parm/soca/obs/config/insitu_surface_trkob.yaml +++ b/parm/soca/obs/config/insitu_surface_trkob.yaml @@ -13,6 +13,33 @@ obs space: max pool size: 1 obs operator: name: Identity - observation alias file: obsop_name_map.yaml + observation alias file: ./obsop_name_map.yaml obs error: covariance model: diagonal + +obs filters: +- filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_area_fraction} + minvalue: 0.9 +- filter: Bounds Check + minvalue: 1.0 + maxvalue: 41.0 +- filter: Background Check + threshold: 5.0 +- filter: Domain Check + where: + - variable: {name: ObsError/seaSurfaceTemperature} + minvalue: 1.0e-8 +- filter: Domain Check + where: + - variable: { name: GeoVaLs/sea_ice_area_fraction} + maxvalue: 1.0e-5 +- filter: Domain Check + where: + - variable: {name: GeoVaLs/sea_surface_temperature} + minvalue: -1.0 +- filter: Domain Check + where: + - variable: {name: GeoVaLs/distance_from_coast} + minvalue: 100e3 diff --git a/parm/soca/obs/config/insitu_surface_trkob_salinity.yaml b/parm/soca/obs/config/insitu_surface_trkob_salinity.yaml new file mode 100644 index 000000000..f32e66588 --- /dev/null +++ b/parm/soca/obs/config/insitu_surface_trkob_salinity.yaml @@ -0,0 +1,18 @@ +obs space: + name: insitu_surface_trkob_salinity + obsdatain: + engine: + type: H5File + obsfile: !ENV ${DATA}/obs/${OPREFIX}insitu_surface_trkob.${PDY}${cyc}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ${DATA}/diags/insitu_surface_trkob_salinity.${PDY}${cyc}.nc4 + simulated variables: [seaSurfaceSalinity] +get values: + time interpolation: linear +obs operator: + name: Identity + observation alias file: ./obsop_name_map.yaml +obs error: + covariance model: diagonal diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index 2c4f488df..5e595d4a8 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -21,10 +21,12 @@ observers: - !INC ${OBS_YAML_DIR}/insitu_profile_argo.yaml - !INC ${OBS_YAML_DIR}/insitu_profile_glider.yaml - !INC ${OBS_YAML_DIR}/insitu_profile_tesac.yaml +- !INC ${OBS_YAML_DIR}/insitu_profile_tesac_salinity.yaml - !INC ${OBS_YAML_DIR}/insitu_profile_marinemammal.yaml - !INC ${OBS_YAML_DIR}/insitu_profile_xbtctd.yaml - !INC ${OBS_YAML_DIR}/insitu_surface_altkob.yaml - !INC ${OBS_YAML_DIR}/insitu_surface_trkob.yaml +- !INC ${OBS_YAML_DIR}/insitu_surface_trkob_salinity.yaml # in situ: daily - !INC ${OBS_YAML_DIR}/insitu_profile_dbuoy.yaml From a41818030c7826fa395ca430f8b25151386921dd Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Thu, 29 Feb 2024 22:08:11 +0000 Subject: [PATCH 20/28] Fixed typo --- ush/ioda/bufr2ioda/gen_bufr2ioda_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py b/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py index 783b58197..2ce0e9903 100755 --- a/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py +++ b/ush/ioda/bufr2ioda/gen_bufr2ioda_json.py @@ -29,7 +29,7 @@ def gen_bufr_json(config, template, output): parser.add_argument('-t', '--template', type=str, help='Input JSON template', required=True) parser.add_argument('-o', '--output', type=str, help='Output JSON file', required=True) args = parser.parse_args() - # get the config from your environmentgg + # get the config from your environment config = cast_strdict_as_dtypedict(os.environ) logger.info(f"Config: {config}") # we need to add in current cycle from PDYcyc From bf077ab551ae9fffea0b5a38509592dd4ee7411a Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Mon, 4 Mar 2024 15:42:17 -0600 Subject: [PATCH 21/28] copy daily insitu subsampled --- test/soca/gw/setup_obsprep.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/soca/gw/setup_obsprep.sh b/test/soca/gw/setup_obsprep.sh index 81e419333..5759cf829 100755 --- a/test/soca/gw/setup_obsprep.sh +++ b/test/soca/gw/setup_obsprep.sh @@ -41,8 +41,15 @@ for PDY in "${PDYs[@]}"; do done else # Copy subsampled monthly in situ BUFR - # Example filename: tesac.201804.dcom_subsampled have been copied as + # Example filename: tesac.201804.dcom_subsampled are copied as # gdas.t00z.tesac.tm00.bufr_d (the original filename) + for typ in bathy dbuoy dbuoyb mbuoy mbuoyb ships shipsu tesac trkob; do + if [ $(ls $testdatadir_bufr/${typ}.*subsampled 2>/dev/null | wc -l) -gt 0 ]; then + cp -p $testdatadir_bufr/${typ}.*subsampled \ + $testdatadir_bufr/gdas.t${cyc}z.${typ}.tm00.bufr_d + fi + done + # TODO: SP to replace these with daily and monthly subsampled with proper dates fullsubdir="$PDYdir/$cyc/atmos" mkdir -p "$fullsubdir" From 466d433083ba0182421b48cb852bf3ec27c2837b Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Mon, 4 Mar 2024 15:43:41 -0600 Subject: [PATCH 22/28] Add json files for situ surface --- .../bufr2ioda/bufr2ioda_insitu_profile_dbuoy.json | 12 ++++++++++++ .../bufr2ioda/bufr2ioda_insitu_profile_dbuoyb.json | 12 ++++++++++++ .../bufr2ioda/bufr2ioda_insitu_profile_mbuoy.json | 12 ++++++++++++ .../bufr2ioda/bufr2ioda_insitu_profile_mbuoyb.json | 12 ++++++++++++ .../bufr2ioda/bufr2ioda_insitu_surface_ships.json | 12 ++++++++++++ .../bufr2ioda/bufr2ioda_insitu_surface_shipsu.json | 12 ++++++++++++ 6 files changed, 72 insertions(+) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoy.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoyb.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoy.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoyb.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_ships.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_shipsu.json diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoy.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoy.json new file mode 100644 index 000000000..8bb4ebd47 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoy.json @@ -0,0 +1,12 @@ +{ + "data_format" : "dbuoy", + "subsets" : "DBUOY", + "source" : "NCEP data tank", + "data_type" : "dbuoy", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ drifting buoy (TAC:drifters, etc) profiles", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoyb.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoyb.json new file mode 100644 index 000000000..d93d9b4dd --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_dbuoyb.json @@ -0,0 +1,12 @@ +{ + "data_format" : "dbuoyb", + "subsets" : "DBUOYB", + "source" : "NCEP data tank", + "data_type" : "dbuoyb", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ drifting buoy (BUFR:drifters, etc) profiles", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoy.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoy.json new file mode 100644 index 000000000..9858445e2 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoy.json @@ -0,0 +1,12 @@ +{ + "data_format" : "mbuoy", + "subsets" : "MBUOY", + "source" : "NCEP data tank", + "data_type" : "mbuoy", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ moored buoy (TAC:Tropical Mooring) profiles", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoyb.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoyb.json new file mode 100644 index 000000000..bef4db29e --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_mbuoyb.json @@ -0,0 +1,12 @@ +{ + "data_format" : "mbuoyb", + "subsets" : "MBUOYB", + "source" : "NCEP data tank", + "data_type" : "mbuoyb", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ moored buoy (BUFR:Tropical Mooring) profiles", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_ships.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_ships.json new file mode 100644 index 000000000..53bfc5bbd --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_ships.json @@ -0,0 +1,12 @@ +{ + "data_format" : "ships", + "subsets" : "SHIPS", + "source" : "NCEP data tank", + "data_type" : "bathy", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ ship (TAC: restriced) surface", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_shipsu.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_shipsu.json new file mode 100644 index 000000000..48b05fe6e --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_shipsu.json @@ -0,0 +1,12 @@ +{ + "data_format" : "shipsu", + "subsets" : "SHIPSU", + "source" : "NCEP data tank", + "data_type" : "shipsu", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ ship (TAC: unrestricted) surface", + "data_provider" : "U.S. NOAA" +} From e6b3e38833e7ee7b49e2a211246a1fc5e0b51676 Mon Sep 17 00:00:00 2001 From: Shastri Paturi Date: Mon, 25 Mar 2024 14:27:40 -0400 Subject: [PATCH 23/28] Update obsprep_config.yaml --- parm/soca/obsprep/obsprep_config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/soca/obsprep/obsprep_config.yaml b/parm/soca/obsprep/obsprep_config.yaml index ab99c57f9..3e4d55c6a 100644 --- a/parm/soca/obsprep/obsprep_config.yaml +++ b/parm/soca/obsprep/obsprep_config.yaml @@ -167,14 +167,14 @@ observations: provider: GTS dmpdir subdir: atmos type: bufr - dmpdir regex: 'gdas.*.argo.*.bufr_d' + dmpdir regex: 'gdas.*.subpfl.*.bufr_d' - obs space: name: insitu_profile_glider provider: GTS dmpdir subdir: atmos type: bufr - dmpdir regex: 'gdas.*.glider.*.bufr_d' + dmpdir regex: 'gdas.*.subpfl.*.bufr_d' - obs space: name: insitu_profile_tesac @@ -188,7 +188,7 @@ observations: provider: GTS dmpdir subdir: atmos type: bufr - dmpdir regex: 'gdas.*.marinemammal.*.bufr_d' + dmpdir regex: 'gdas.*.tesac.*.bufr_d' - obs space: name: insitu_profile_xbtctd From c92feef285f14b30971040b1b9c088e71736be49 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Mon, 25 Mar 2024 14:48:34 -0500 Subject: [PATCH 24/28] monthly insitu --- parm/soca/obs/obs_list.yaml | 8 +- ...bufr2ioda_insitu_marinemammals_profiles.py | 318 +++++++++++++++++ .../bufr2ioda_insitu_profiles_argo.py | 323 +++++++++++++++++ .../bufr2ioda_insitu_profiles_glider.py | 324 ++++++++++++++++++ .../bufr2ioda_insitu_profiles_tesac.py | 319 +++++++++++++++++ .../bufr2ioda_insitu_profiles_xbtctd.py | 299 ++++++++++++++++ .../bufr2ioda_insitu_surface_altkob.py | 278 +++++++++++++++ 7 files changed, 1865 insertions(+), 4 deletions(-) create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_insitu_marinemammals_profiles.py create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_argo.py create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_glider.py create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_tesac.py create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_xbtctd.py create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_altkob.py diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index 5e595d4a8..f7be24da5 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -29,7 +29,7 @@ observers: - !INC ${OBS_YAML_DIR}/insitu_surface_trkob_salinity.yaml # in situ: daily -- !INC ${OBS_YAML_DIR}/insitu_profile_dbuoy.yaml -- !INC ${OBS_YAML_DIR}/insitu_profile_dbuoyb.yaml -- !INC ${OBS_YAML_DIR}/insitu_profile_mbuoy.yaml -- !INC ${OBS_YAML_DIR}/insitu_profile_mbuoyb.yaml +#- !INC ${OBS_YAML_DIR}/insitu_profile_dbuoy.yaml +#- !INC ${OBS_YAML_DIR}/insitu_profile_dbuoyb.yaml +#- !INC ${OBS_YAML_DIR}/insitu_profile_mbuoy.yaml +#- !INC ${OBS_YAML_DIR}/insitu_profile_mbuoyb.yaml diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_marinemammals_profiles.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_marinemammals_profiles.py new file mode 100755 index 000000000..e7341b76e --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_marinemammals_profiles.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +# (C) Copyright 2024 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import numpy.ma as ma +import os +import argparse +import math +import calendar +import time +import copy +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger +import warnings +# suppress warnings +warnings.filterwarnings('ignore') + + +def Compute_sequenceNumber(lon): + lon_u, seqNum = np.unique(lon, return_inverse=True) + seqNum = seqNum.astype(np.int32) + logger.debug(f"Len of Sequence Number: {len(seqNum)}") + + return seqNum + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + # General Information + converter = 'BUFR to IODA Converter' + platform_description = 'Profiles from Marine Mammals: temperature and salinity' + + bufrfile = f"{cycle_datetime}-{cycle_type}.t{hh}z.{data_format}.tm00.bufr_d" + DATA_PATH = os.path.join(dump_dir, bufrfile) + logger.debug(f"{bufrfile}, {DATA_PATH}") + + # ========================================== + # Make the QuerySet for all the data we want + # ========================================== + start_time = time.time() + + logger.debug(f"Making QuerySet ...") + q = bufr.QuerySet() + + # MetaData + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('ryear', '*/RCYR') + q.add('rmonth', '*/RCMO') + q.add('rday', '*/RCDY') + q.add('rhour', '*/RCHR') + q.add('rminute', '*/RCMI') + q.add('stationID', '*/RPID') + q.add('latitude', '*/CLAT') + q.add('longitude', '*/CLON') + q.add('depth', '*/BTOCN/DBSS') + + # ObsValue + q.add('temp', '*/BTOCN/STMP') + q.add('saln', '*/BTOCN/SALN') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ========================================================== + # Open the BUFR file and execute the QuerySet + # Use the ResultSet returned to get numpy arrays of the data + # ========================================================== + start_time = time.time() + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + logger.debug(f" ... Executing QuerySet: get MetaData ...") + dateTime = r.get_datetime('year', 'month', 'day', 'hour', 'minute', group_by='depth') + dateTime = dateTime.astype(np.int64) + rcptdateTime = r.get_datetime('ryear', 'rmonth', 'rday', 'rhour', 'rminute', group_by='depth') + rcptdateTime = rcptdateTime.astype(np.int64) + stationID = r.get('stationID', group_by='depth') + lat = r.get('latitude', group_by='depth') + lon = r.get('longitude', group_by='depth') + depth = r.get('depth', group_by='depth') + + # ObsValue + logger.debug(f" ... Executing QuerySet: get ObsValue ...") + temp = r.get('temp', group_by='depth') + temp -= 273.15 + saln = r.get('saln', group_by='depth') + + # Add mask based on min, max values + mask = ((temp > -10.0) & (temp <= 50.0)) & ((saln >= 0.0) & (saln <= 45.0)) + lat = lat[mask] + lon = lon[mask] + depth = depth[mask] + stationID = stationID[mask] + dateTime = dateTime[mask] + rcptdateTime = rcptdateTime[mask] + temp = temp[mask] + saln = saln[mask] + + logger.debug(f"Get sequenceNumber based on unique longitude...") + seqNum = Compute_sequenceNumber(lon) + + # ======================================= + # Separate marine mammals from TESAC tank + # ======================================= + logger.debug(f"Creating the mask for marine mammals from TESAC floats based on station ID ...") + + alpha_mask = [item.isalpha() for item in stationID] + indices_true = [index for index, value in enumerate(alpha_mask) if value] + + # Apply index + stationID = stationID[indices_true] + lat = lat[indices_true] + lon = lon[indices_true] + depth = depth[indices_true] + temp = temp[indices_true] + saln = saln[indices_true] + seqNum = seqNum[indices_true] + dateTime = dateTime[indices_true] + rcptdateTime = rcptdateTime[indices_true] + + # ObsError + logger.debug(f"Generating ObsError array with constant value (instrument error)...") + ObsError_temp = np.float32(np.ma.masked_array(np.full((len(indices_true)), 0.02))) + ObsError_saln = np.float32(np.ma.masked_array(np.full((len(indices_true)), 0.01))) + + # PreQC + logger.debug(f"Generating PreQC array with 0...") + PreQC = (np.ma.masked_array(np.full((len(indices_true)), 0))).astype(np.int32) + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + + # ================================================== + # Check values of BUFR variables, dimension and type + # ================================================== + logger.debug(f" temp min, max, length, dtype = {temp.min()}, {temp.max()}, {len(temp)}, {temp.dtype}") + logger.debug(f" saln min, max, length, dtype = {saln.min()}, {saln.max()}, {len(saln)}, {saln.dtype}") + logger.debug(f" lon min, max, length, dtype = {lon.min()}, {lon.max()}, {len(lon)}, {lon.dtype}") + logger.debug(f" lat min, max, length, dtype = {lat.min()}, {lat.max()}, {len(lat)}, {lat.dtype}") + logger.debug(f" depth min, max, length, dtype = {depth.min()}, {depth.max()}, {len(depth)}, {depth.dtype}") + logger.debug(f" PreQC min, max, length, dtype = {PreQC.min()}, {PreQC.max()}, {len(PreQC)}, {PreQC.dtype}") + logger.debug(f" ObsError_temp min, max, length, dtype = {ObsError_temp.min()}, {ObsError_temp.max()}, {len(ObsError_temp)}, {ObsError_temp.dtype}") + logger.debug(f" ObsError_saln min, max, length, dtype = {ObsError_saln.min()}, {ObsError_saln.max()}, {len(ObsError_saln)}, {ObsError_saln.dtype}") + logger.debug(f" stationID shape, dtype = {stationID.shape}, {stationID.astype(str).dtype}") + logger.debug(f" dateTime shape, dtype = {dateTime.shape}, {dateTime.dtype}") + logger.debug(f" rcptdateTime shape, dytpe = {rcptdateTime.shape}, {rcptdateTime.dtype}") + logger.debug(f" sequence Num shape, dtype = {seqNum.shape}, {seqNum.dtype}") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}_profiles.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeRange', [str(dateTime.min()), str(dateTime.max())]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, and attributes") + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # rcptDatetime + obsspace.create_var('MetaData/rcptdateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'receipt Datetime') \ + .write_data(rcptdateTime) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Station Identification + obsspace.create_var('MetaData/stationID', dtype=stationID.dtype, fillval=stationID.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stationID) + + # Depth + obsspace.create_var('MetaData/depth', dtype=depth.dtype, fillval=depth.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Water depth') \ + .write_data(depth) + + # Sequence Number + obsspace.create_var('MetaData/sequenceNumber', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'Sequence Number') \ + .write_data(seqNum) + + # PreQC + obsspace.create_var('PreQC/waterTemperature', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + obsspace.create_var('PreQC/salinity', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + # ObsError + obsspace.create_var('ObsError/waterTemperature', dtype=ObsError_temp.dtype, fillval=ObsError_temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_temp) + + obsspace.create_var('ObsError/salinity', dtype=ObsError_saln.dtype, fillval=ObsError_saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_saln) + + # ObsValue + obsspace.create_var('ObsValue/waterTemperature', dtype=temp.dtype, fillval=temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('valid_range', np.array([-10.0, 50.0], dtype=np.float32)) \ + .write_attr('long_name', 'water Temperature') \ + .write_data(temp) + + obsspace.create_var('ObsValue/salinity', dtype=saln.dtype, fillval=saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('valid_range', np.array([0.0, 45.0], dtype=np.float32)) \ + .write_attr('long_name', 'salinity') \ + .write_data(saln) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: {running_time} \ + seconds") + + logger.debug(f"All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_insitu_marinemammals_profiles.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_argo.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_argo.py new file mode 100755 index 000000000..d734a260f --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_argo.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +# (C) Copyright 2024 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import numpy.ma as ma +import os +import argparse +import math +import calendar +import time +import copy +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger +import warnings +# suppress warnings +warnings.filterwarnings('ignore') + + +def Compute_sequenceNumber(lon): + lon_u, seqNum = np.unique(lon, return_inverse=True) + seqNum = seqNum.astype(np.int32) + logger.debug(f"Len of Sequence Number: {len(seqNum)}") + + return seqNum + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + # General Information + converter = 'BUFR to IODA Converter' + platform_description = 'ARGO profiles from subpfl: temperature and salinity' + + bufrfile = f"{cycle_datetime}-{cycle_type}.t{hh}z.{data_format}.tm00.bufr_d" + DATA_PATH = os.path.join(dump_dir, bufrfile) + logger.debug(f"{bufrfile}, {DATA_PATH}") + + # ========================================== + # Make the QuerySet for all the data we want + # ========================================== + start_time = time.time() + + logger.debug(f"Making QuerySet ...") + q = bufr.QuerySet() + + # MetaData + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('ryear', '*/RCYR') + q.add('rmonth', '*/RCMO') + q.add('rday', '*/RCDY') + q.add('rhour', '*/RCHR') + q.add('rminute', '*/RCMI') + q.add('stationID', '*/WMOP') + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + q.add('depth', '*/GLPFDATA/WPRES') + + # ObsValue + q.add('temp', '*/GLPFDATA/SSTH') + q.add('saln', '*/GLPFDATA/SALNH') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ========================================================== + # Open the BUFR file and execute the QuerySet + # Use the ResultSet returned to get numpy arrays of the data + # ========================================================== + start_time = time.time() + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + logger.debug(f" ... Executing QuerySet: get MetaData ...") + dateTime = r.get_datetime('year', 'month', 'day', 'hour', 'minute', group_by='depth') + dateTime = dateTime.astype(np.int64) + rcptdateTime = r.get_datetime('ryear', 'rmonth', 'rday', 'rhour', 'rminute', group_by='depth') + rcptdateTime = rcptdateTime.astype(np.int64) + stationID = r.get('stationID', group_by='depth') + lat = r.get('latitude', group_by='depth') + lon = r.get('longitude', group_by='depth') + depth = r.get('depth', group_by='depth') + # convert depth in pressure units to meters (rho * g * h) + depth = np.float32(depth.astype(float) * 0.0001) + + # ObsValue + logger.debug(f" ... Executing QuerySet: get ObsValue ...") + temp = r.get('temp', group_by='depth') + temp -= 273.15 + saln = r.get('saln', group_by='depth') + + # Add mask based on min, max values + mask = ((temp > -10.0) & (temp <= 50.0)) & ((saln >= 0.0) & (saln <= 45.0)) + temp = temp[mask] + lat = lat[mask] + lon = lon[mask] + depth = depth[mask] + stationID = stationID[mask] + + logger.debug(f"Get sequenceNumber based on unique longitude...") + seqNum = Compute_sequenceNumber(lon) + + # ======================================= + # Separate ARGO profiles from subpfl tank + # ======================================= + logger.debug(f"Finding index for ARGO floats where the second number of the stationID=9...") + index_list = [] + for index, number in enumerate(stationID): + # Convert the number to a string + number_str = str(number) + + # Check if the second character is equal to '9' + if number_str[1] == '9': + index_list.append(index) + logger.debug(f"Indexing Done...") + + # Apply index + stationID = stationID[index_list] + lat = lat[index_list] + lon = lon[index_list] + depth = depth[index_list] + temp = temp[index_list] + saln = saln[index_list] + seqNum = seqNum[index_list] + dateTime = dateTime[index_list] + rcptdateTime = rcptdateTime[index_list] + + # ObsError + logger.debug(f"Generating ObsError array with constant value (instrument error)...") + ObsError_temp = np.float32(np.ma.masked_array(np.full((len(index_list)), 0.02))) + ObsError_saln = np.float32(np.ma.masked_array(np.full((len(index_list)), 0.01))) + + # PreQC + logger.debug(f"Generating PreQC array with 0...") + PreQC = (np.ma.masked_array(np.full((len(index_list)), 0))).astype(np.int32) + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + + # ================================================== + # Check values of BUFR variables, dimension and type + # ================================================== + logger.debug(f" temp min, max, length, dtype = {temp.min()}, {temp.max()}, {len(temp)}, {temp.dtype}") + logger.debug(f" saln min, max, length, dtype = {saln.min()}, {saln.max()}, {len(saln)}, {saln.dtype}") + logger.debug(f" lon min, max, length, dtype = {lon.min()}, {lon.max()}, {len(lon)}, {lon.dtype}") + logger.debug(f" lat min, max, length, dtype = {lat.min()}, {lat.max()}, {len(lat)}, {lat.dtype}") + logger.debug(f" depth min, max, length, dtype = {depth.min()}, {depth.max()}, {len(depth)}, {depth.dtype}") + logger.debug(f" PreQC min, max, length, dtype = {PreQC.min()}, {PreQC.max()}, {len(PreQC)}, {PreQC.dtype}") + logger.debug(f" ObsError_temp min, max, length, dtype = {ObsError_temp.min()}, {ObsError_temp.max()}, {len(ObsError_temp)}, {ObsError_temp.dtype}") + logger.debug(f" ObsError_saln min, max, length, dtype = {ObsError_saln.min()}, {ObsError_saln.max()}, {len(ObsError_saln)}, {ObsError_saln.dtype}") + logger.debug(f" stationID shape, dtype = {stationID.shape}, {stationID.astype(str).dtype}") + logger.debug(f" dateTime shape, dtype = {dateTime.shape}, {dateTime.dtype}") + logger.debug(f" rcptdateTime shape, dytpe = {rcptdateTime.shape}, {rcptdateTime.dtype}") + logger.debug(f" sequence Num shape, dtype = {seqNum.shape}, {seqNum.dtype}") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}_profiles.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeRange', [str(dateTime.min()), str(dateTime.max())]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, and attributes") + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # rcptDatetime + obsspace.create_var('MetaData/rcptdateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'receipt Datetime') \ + .write_data(rcptdateTime) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Station Identification + obsspace.create_var('MetaData/stationID', dtype=stationID.dtype, fillval=stationID.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stationID) + + # Depth + obsspace.create_var('MetaData/depth', dtype=depth.dtype, fillval=depth.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Water depth') \ + .write_data(depth) + + # Sequence Number + obsspace.create_var('MetaData/sequenceNumber', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'Sequence Number') \ + .write_data(seqNum) + + # PreQC + obsspace.create_var('PreQC/waterTemperature', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + obsspace.create_var('PreQC/salinity', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + # ObsError + obsspace.create_var('ObsError/waterTemperature', dtype=ObsError_temp.dtype, fillval=ObsError_temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_temp) + + obsspace.create_var('ObsError/salinity', dtype=ObsError_saln.dtype, fillval=ObsError_saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_saln) + + # ObsValue + obsspace.create_var('ObsValue/waterTemperature', dtype=temp.dtype, fillval=temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('valid_range', np.array([-10.0, 50.0], dtype=np.float32)) \ + .write_attr('long_name', 'water Temperature') \ + .write_data(temp) + + obsspace.create_var('ObsValue/salinity', dtype=saln.dtype, fillval=saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('valid_range', np.array([0.0, 45.0], dtype=np.float32)) \ + .write_attr('long_name', 'salinity') \ + .write_data(saln) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: {running_time} \ + seconds") + + logger.debug(f"All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_insitu_profiles_argo.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_glider.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_glider.py new file mode 100755 index 000000000..59dbb266d --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_glider.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +# (C) Copyright 2024 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import numpy.ma as ma +import os +import argparse +import math +import calendar +import time +import copy +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger +import warnings +# suppress warnings +warnings.filterwarnings('ignore') + + +def Compute_sequenceNumber(lon): + lon_u, seqNum = np.unique(lon, return_inverse=True) + seqNum = seqNum.astype(np.int32) + logger.debug(f"Len of Sequence Number: {len(seqNum)}") + + return seqNum + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + # General Information + converter = 'BUFR to IODA Converter' + platform_description = 'GLIDER profiles from subpfl: temperature and salinity' + + bufrfile = f"{cycle_datetime}-{cycle_type}.t{hh}z.{data_format}.tm00.bufr_d" + DATA_PATH = os.path.join(dump_dir, bufrfile) + logger.debug(f"{bufrfile}, {DATA_PATH}") + + # ========================================== + # Make the QuerySet for all the data we want + # ========================================== + start_time = time.time() + + logger.debug(f"Making QuerySet ...") + q = bufr.QuerySet() + + # MetaData + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('ryear', '*/RCYR') + q.add('rmonth', '*/RCMO') + q.add('rday', '*/RCDY') + q.add('rhour', '*/RCHR') + q.add('rminute', '*/RCMI') + q.add('stationID', '*/WMOP') + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + q.add('depth', '*/GLPFDATA/WPRES') + + # ObsValue + q.add('temp', '*/GLPFDATA/SSTH') + q.add('saln', '*/GLPFDATA/SALNH') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ========================================================== + # Open the BUFR file and execute the QuerySet + # Use the ResultSet returned to get numpy arrays of the data + # ========================================================== + start_time = time.time() + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + logger.debug(f" ... Executing QuerySet: get MetaData ...") + dateTime = r.get_datetime('year', 'month', 'day', 'hour', 'minute', group_by='depth') + dateTime = dateTime.astype(np.int64) + rcptdateTime = r.get_datetime('ryear', 'rmonth', 'rday', 'rhour', 'rminute', group_by='depth') + rcptdateTime = rcptdateTime.astype(np.int64) + stationID = r.get('stationID', group_by='depth') + lat = r.get('latitude', group_by='depth') + lon = r.get('longitude', group_by='depth') + depth = r.get('depth', group_by='depth') + # convert depth in pressure units to meters (rho * g * h) + depth = np.float32(depth.astype(float) * 0.0001) + + # ObsValue + logger.debug(f" ... Executing QuerySet: get ObsValue ...") + temp = r.get('temp', group_by='depth') + temp -= 273.15 + saln = r.get('saln', group_by='depth') + + # Add mask based on min, max values + mask = ((temp > -10.0) & (temp <= 50.0)) & ((saln >= 0.0) & (saln <= 45.0)) + temp = temp[mask] + saln = saln[mask] + lat = lat[mask] + lon = lon[mask] + depth = depth[mask] + stationID = stationID[mask] + dateTime = dateTime[mask] + rcptdateTime = rcptdateTime[mask] + + logger.debug(f"Get sequenceNumber based on unique longitude...") + seqNum = Compute_sequenceNumber(lon) + + # ========================================= + # Separate GLIDER profiles from subpfl tank + # ========================================= + logger.debug(f"Creating the mask for GLIDER floats based on station ID numbers ...") + + mask_gldr = (stationID >= 68900) & (stationID <= 68999) | (stationID >= 1800000) & (stationID <= 1809999) | \ + (stationID >= 2800000) & (stationID <= 2809999) | (stationID >= 3800000) & (stationID <= 3809999) | \ + (stationID >= 4800000) & (stationID <= 4809999) | (stationID >= 5800000) & (stationID <= 5809999) | \ + (stationID >= 6800000) & (stationID <= 6809999) | (stationID >= 7800000) & (stationID <= 7809999) + + # Apply mask + stationID = stationID[mask_gldr] + lat = lat[mask_gldr] + lon = lon[mask_gldr] + depth = depth[mask_gldr] + temp = temp[mask_gldr] + saln = saln[mask_gldr] + seqNum = seqNum[mask_gldr] + dateTime = dateTime[mask_gldr] + rcptdateTime = rcptdateTime[mask_gldr] + + logger.debug(f"masking Done...") + + # ObsError + logger.debug(f"Generating ObsError array with constant value (instrument error)...") + ObsError_temp = np.float32(np.ma.masked_array(np.full((len(mask_gldr)), 0.02))) + ObsError_saln = np.float32(np.ma.masked_array(np.full((len(mask_gldr)), 0.01))) + + # PreQC + logger.debug(f"Generating PreQC array with 0...") + PreQC = (np.ma.masked_array(np.full((len(mask_gldr)), 0))).astype(np.int32) + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + + # ================================================== + # Check values of BUFR variables, dimension and type + # ================================================== + logger.debug(f" temp min, max, length, dtype = {temp.min()}, {temp.max()}, {len(temp)}, {temp.dtype}") + logger.debug(f" saln min, max, length, dtype = {saln.min()}, {saln.max()}, {len(saln)}, {saln.dtype}") + logger.debug(f" lon min, max, length, dtype = {lon.min()}, {lon.max()}, {len(lon)}, {lon.dtype}") + logger.debug(f" lat min, max, length, dtype = {lat.min()}, {lat.max()}, {len(lat)}, {lat.dtype}") + logger.debug(f" depth min, max, length, dtype = {depth.min()}, {depth.max()}, {len(depth)}, {depth.dtype}") + logger.debug(f" PreQC min, max, length, dtype = {PreQC.min()}, {PreQC.max()}, {len(PreQC)}, {PreQC.dtype}") + logger.debug(f" ObsError_temp min, max, length, dtype = {ObsError_temp.min()}, {ObsError_temp.max()}, {len(ObsError_temp)}, {ObsError_temp.dtype}") + logger.debug(f" ObsError_saln min, max, length, dtype = {ObsError_saln.min()}, {ObsError_saln.max()}, {len(ObsError_saln)}, {ObsError_saln.dtype}") + logger.debug(f" stationID shape, dtype = {stationID.shape}, {stationID.astype(str).dtype}") + logger.debug(f" dateTime shape, dtype = {dateTime.shape}, {dateTime.dtype}") + logger.debug(f" rcptdateTime shape, dytpe = {rcptdateTime.shape}, {rcptdateTime.dtype}") + logger.debug(f" sequence Num shape, dtype = {seqNum.shape}, {seqNum.dtype}") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}_profiles.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeRange', [str(dateTime.min()), str(dateTime.max())]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, and attributes") + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # rcptDatetime + obsspace.create_var('MetaData/rcptdateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'receipt Datetime') \ + .write_data(rcptdateTime) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Station Identification + obsspace.create_var('MetaData/stationID', dtype=stationID.dtype, fillval=stationID.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stationID) + + # Depth + obsspace.create_var('MetaData/depth', dtype=depth.dtype, fillval=depth.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Water depth') \ + .write_data(depth) + + # Sequence Number + obsspace.create_var('MetaData/sequenceNumber', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'Sequence Number') \ + .write_data(seqNum) + + # PreQC + obsspace.create_var('PreQC/waterTemperature', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + obsspace.create_var('PreQC/salinity', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + # ObsError + obsspace.create_var('ObsError/waterTemperature', dtype=ObsError_temp.dtype, fillval=ObsError_temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_temp) + + obsspace.create_var('ObsError/salinity', dtype=ObsError_saln.dtype, fillval=ObsError_saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_saln) + + # ObsValue + obsspace.create_var('ObsValue/waterTemperature', dtype=temp.dtype, fillval=temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('valid_range', np.array([-10.0, 50.0], dtype=np.float32)) \ + .write_attr('long_name', 'water Temperature') \ + .write_data(temp) + + obsspace.create_var('ObsValue/salinity', dtype=saln.dtype, fillval=saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('valid_range', np.array([0.0, 45.0], dtype=np.float32)) \ + .write_attr('long_name', 'salinity') \ + .write_data(saln) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: {running_time} \ + seconds") + + logger.debug(f"All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_insitu_profiles_glider.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_tesac.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_tesac.py new file mode 100755 index 000000000..23812a515 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_tesac.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +# (C) Copyright 2024 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import numpy.ma as ma +import os +import argparse +import math +import calendar +import time +import copy +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger +import warnings +# suppress warnings +warnings.filterwarnings('ignore') + + +def Compute_sequenceNumber(lon): + + lon_u, seqNum = np.unique(lon, return_inverse=True) + seqNum = seqNum.astype(np.int32) + logger.debug(f"Len of Sequence Number: {len(seqNum)}") + + return seqNum + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + # General Information + converter = 'BUFR to IODA Converter' + platform_description = 'Profiles from TESAC: temperature and salinity' + + bufrfile = f"{cycle_datetime}-{cycle_type}.t{hh}z.{data_format}.tm00.bufr_d" + DATA_PATH = os.path.join(dump_dir, bufrfile) + logger.debug(f"{bufrfile}, {DATA_PATH}") + + # ========================================== + # Make the QuerySet for all the data we want + # ========================================== + start_time = time.time() + + logger.debug(f"Making QuerySet ...") + q = bufr.QuerySet() + + # MetaData + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('ryear', '*/RCYR') + q.add('rmonth', '*/RCMO') + q.add('rday', '*/RCDY') + q.add('rhour', '*/RCHR') + q.add('rminute', '*/RCMI') + q.add('stationID', '*/RPID') + q.add('latitude', '*/CLAT') + q.add('longitude', '*/CLON') + q.add('depth', '*/BTOCN/DBSS') + + # ObsValue + q.add('temp', '*/BTOCN/STMP') + q.add('saln', '*/BTOCN/SALN') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ========================================================== + # Open the BUFR file and execute the QuerySet + # Use the ResultSet returned to get numpy arrays of the data + # ========================================================== + start_time = time.time() + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + logger.debug(f" ... Executing QuerySet: get MetaData ...") + dateTime = r.get_datetime('year', 'month', 'day', 'hour', 'minute', group_by='depth') + dateTime = dateTime.astype(np.int64) + rcptdateTime = r.get_datetime('ryear', 'rmonth', 'rday', 'rhour', 'rminute', group_by='depth') + rcptdateTime = rcptdateTime.astype(np.int64) + stationID = r.get('stationID', group_by='depth') + lat = r.get('latitude', group_by='depth') + lon = r.get('longitude', group_by='depth') + depth = r.get('depth', group_by='depth') + + # ObsValue + logger.debug(f" ... Executing QuerySet: get ObsValue ...") + temp = r.get('temp', group_by='depth') + temp -= 273.15 + saln = r.get('saln', group_by='depth') + + # Add mask based on min, max values + mask = ((temp > -10.0) & (temp <= 50.0)) & ((saln >= 0.0) & (saln <= 45.0)) + lat = lat[mask] + lon = lon[mask] + depth = depth[mask] + stationID = stationID[mask] + dateTime = dateTime[mask] + rcptdateTime = rcptdateTime[mask] + temp = temp[mask] + saln = saln[mask] + + logger.debug(f"Get sequenceNumber based on unique longitude...") + seqNum = Compute_sequenceNumber(lon) + + # ================================== + # Separate TESAC profiles tesac tank + # ================================== + logger.debug(f"Creating the mask for TESAC floats based on station ID ...") + + digit_mask = [item.isdigit() for item in stationID] + indices_true = [index for index, value in enumerate(digit_mask) if value] + + # Apply index + stationID = stationID[indices_true] + lat = lat[indices_true] + lon = lon[indices_true] + depth = depth[indices_true] + temp = temp[indices_true] + saln = saln[indices_true] + seqNum = seqNum[indices_true] + dateTime = dateTime[indices_true] + rcptdateTime = rcptdateTime[indices_true] + + # ObsError + logger.debug(f"Generating ObsError array with constant value (instrument error)...") + ObsError_temp = np.float32(np.ma.masked_array(np.full((len(indices_true)), 0.02))) + ObsError_saln = np.float32(np.ma.masked_array(np.full((len(indices_true)), 0.01))) + + # PreQC + logger.debug(f"Generating PreQC array with 0...") + PreQC = (np.ma.masked_array(np.full((len(indices_true)), 0))).astype(np.int32) + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + + # ================================================== + # Check values of BUFR variables, dimension and type + # ================================================== + logger.debug(f" temp min, max, length, dtype = {temp.min()}, {temp.max()}, {len(temp)}, {temp.dtype}") + logger.debug(f" saln min, max, length, dtype = {saln.min()}, {saln.max()}, {len(saln)}, {saln.dtype}") + logger.debug(f" lon min, max, length, dtype = {lon.min()}, {lon.max()}, {len(lon)}, {lon.dtype}") + logger.debug(f" lat min, max, length, dtype = {lat.min()}, {lat.max()}, {len(lat)}, {lat.dtype}") + logger.debug(f" depth min, max, length, dtype = {depth.min()}, {depth.max()}, {len(depth)}, {depth.dtype}") + logger.debug(f" PreQC min, max, length, dtype = {PreQC.min()}, {PreQC.max()}, {len(PreQC)}, {PreQC.dtype}") + logger.debug(f" ObsError_temp min, max, length, dtype = {ObsError_temp.min()}, {ObsError_temp.max()}, {len(ObsError_temp)}, {ObsError_temp.dtype}") + logger.debug(f" ObsError_saln min, max, length, dtype = {ObsError_saln.min()}, {ObsError_saln.max()}, {len(ObsError_saln)}, {ObsError_saln.dtype}") + logger.debug(f" stationID shape, dtype = {stationID.shape}, {stationID.astype(str).dtype}") + logger.debug(f" dateTime shape, dtype = {dateTime.shape}, {dateTime.dtype}") + logger.debug(f" rcptdateTime shape, dytpe = {rcptdateTime.shape}, {rcptdateTime.dtype}") + logger.debug(f" sequence Num shape, dtype = {seqNum.shape}, {seqNum.dtype}") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.insitu_profile_{data_format}.{cycle_datetime}.nc4" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeRange', [str(dateTime.min()), str(dateTime.max())]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, and attributes") + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # rcptDatetime + obsspace.create_var('MetaData/rcptdateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'receipt Datetime') \ + .write_data(rcptdateTime) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Station Identification + obsspace.create_var('MetaData/stationID', dtype=stationID.dtype, fillval=stationID.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stationID) + + # Depth + obsspace.create_var('MetaData/depth', dtype=depth.dtype, fillval=depth.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Water depth') \ + .write_data(depth) + + # Sequence Number + obsspace.create_var('MetaData/sequenceNumber', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'Sequence Number') \ + .write_data(seqNum) + + # PreQC + obsspace.create_var('PreQC/waterTemperature', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + obsspace.create_var('PreQC/salinity', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + # ObsError + obsspace.create_var('ObsError/waterTemperature', dtype=ObsError_temp.dtype, fillval=ObsError_temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_temp) + + obsspace.create_var('ObsError/salinity', dtype=ObsError_saln.dtype, fillval=ObsError_saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_saln) + + # ObsValue + obsspace.create_var('ObsValue/waterTemperature', dtype=temp.dtype, fillval=temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('valid_range', np.array([-10.0, 50.0], dtype=np.float32)) \ + .write_attr('long_name', 'water Temperature') \ + .write_data(temp) + + obsspace.create_var('ObsValue/salinity', dtype=saln.dtype, fillval=saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('valid_range', np.array([0.0, 45.0], dtype=np.float32)) \ + .write_attr('long_name', 'salinity') \ + .write_data(saln) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: {running_time} \ + seconds") + + logger.debug(f"All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_insitu_profiles_tesac.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_xbtctd.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_xbtctd.py new file mode 100755 index 000000000..8923a5aad --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_profiles_xbtctd.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +# (C) Copyright 2024 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import numpy.ma as ma +import os +import argparse +import math +import calendar +import time +import copy +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger +import warnings +# suppress warnings +warnings.filterwarnings('ignore') + + +def Compute_sequenceNumber(lon): + lon_u, seqNum = np.unique(lon, return_inverse=True) + seqNum = seqNum.astype(np.int32) + logger.debug(f"Len of Sequence Number: {len(seqNum)}") + + return seqNum + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + # General Information + converter = 'BUFR to IODA Converter' + platform_description = 'Profiles from XBT/CTD: temperature and salinity' + + bufrfile = f"{cycle_datetime}-{cycle_type}.t{hh}z.{data_format}.tm00.bufr_d" + DATA_PATH = os.path.join(dump_dir, bufrfile) + logger.debug(f"{bufrfile}, {DATA_PATH}") + + # ========================================== + # Make the QuerySet for all the data we want + # ========================================== + start_time = time.time() + + logger.debug(f"Making QuerySet ...") + q = bufr.QuerySet() + + # MetaData + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('ryear', '*/RCYR') + q.add('rmonth', '*/RCMO') + q.add('rday', '*/RCDY') + q.add('rhour', '*/RCHR') + q.add('rminute', '*/RCMI') + q.add('stationID', '*/WMOP') + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + q.add('depth', '*/TMSLPFSQ/DBSS') + + # ObsValue + q.add('temp', '*/TMSLPFSQ/SST1') + q.add('saln', '*/TMSLPFSQ/SALNH') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ========================================================== + # Open the BUFR file and execute the QuerySet + # Use the ResultSet returned to get numpy arrays of the data + # ========================================================== + start_time = time.time() + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + logger.debug(f" ... Executing QuerySet: get MetaData ...") + dateTime = r.get_datetime('year', 'month', 'day', 'hour', 'minute', group_by='depth') + dateTime = dateTime.astype(np.int64) + rcptdateTime = r.get_datetime('ryear', 'rmonth', 'rday', 'rhour', 'rminute', group_by='depth') + rcptdateTime = rcptdateTime.astype(np.int64) + stationID = r.get('stationID', group_by='depth') + lat = r.get('latitude', group_by='depth') + lon = r.get('longitude', group_by='depth') + depth = r.get('depth', group_by='depth') + + # ObsValue + logger.debug(f" ... Executing QuerySet: get ObsValue ...") + temp = r.get('temp', group_by='depth') + temp -= 273.15 + saln = r.get('saln', group_by='depth') + + # Add mask based on min, max values + mask = ((temp > -10.0) & (temp <= 50.0)) & ((saln >= 0.0) & (saln <= 45.0)) + lat = lat[mask] + lon = lon[mask] + depth = depth[mask] + stationID = stationID[mask] + dateTime = dateTime[mask] + rcptdateTime = rcptdateTime[mask] + temp = temp[mask] + saln = saln[mask] + + logger.debug(f"Get sequenceNumber based on unique longitude...") + seqNum = Compute_sequenceNumber(lon) + + # ObsError + logger.debug(f"Generating ObsError array with constant value (instrument error)...") + ObsError_temp = np.float32(np.ma.masked_array(np.full((len(temp)), 0.12))) + ObsError_saln = np.float32(np.ma.masked_array(np.full((len(temp)), 1.00))) + + # PreQC + logger.debug(f"Generating PreQC array with 0...") + PreQC = (np.ma.masked_array(np.full((len(temp)), 0))).astype(np.int32) + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + + # ================================================== + # Check values of BUFR variables, dimension and type + # ================================================== + logger.debug(f" temp min, max, length, dtype = {temp.min()}, {temp.max()}, {len(temp)}, {temp.dtype}") + logger.debug(f" saln min, max, length, dtype = {saln.min()}, {saln.max()}, {len(saln)}, {saln.dtype}") + logger.debug(f" lon min, max, length, dtype = {lon.min()}, {lon.max()}, {len(lon)}, {lon.dtype}") + logger.debug(f" lat min, max, length, dtype = {lat.min()}, {lat.max()}, {len(lat)}, {lat.dtype}") + logger.debug(f" depth min, max, length, dtype = {depth.min()}, {depth.max()}, {len(depth)}, {depth.dtype}") + logger.debug(f" PreQC min, max, length, dtype = {PreQC.min()}, {PreQC.max()}, {len(PreQC)}, {PreQC.dtype}") + logger.debug(f" ObsError_temp min, max, length, dtype = {ObsError_temp.min()}, {ObsError_temp.max()}, {len(ObsError_temp)}, {ObsError_temp.dtype}") + logger.debug(f" ObsError_saln min, max, length, dtype = {ObsError_saln.min()}, {ObsError_saln.max()}, {len(ObsError_saln)}, {ObsError_saln.dtype}") + logger.debug(f" stationID shape, dtype = {stationID.shape}, {stationID.astype(str).dtype}") + logger.debug(f" dateTime shape, dtype = {dateTime.shape}, {dateTime.dtype}") + logger.debug(f" rcptdateTime shape, dytpe = {rcptdateTime.shape}, {rcptdateTime.dtype}") + logger.debug(f" sequence Num shape, dtype = {seqNum.shape}, {seqNum.dtype}") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}_profiles.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeRange', [str(dateTime.min()), str(dateTime.max())]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, and attributes") + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # rcptDatetime + obsspace.create_var('MetaData/rcptdateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'receipt Datetime') \ + .write_data(rcptdateTime) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Station Identification + obsspace.create_var('MetaData/stationID', dtype=stationID.dtype, fillval=stationID.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stationID) + + # Depth + obsspace.create_var('MetaData/depth', dtype=depth.dtype, fillval=depth.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Water depth') \ + .write_data(depth) + + # Sequence Number + obsspace.create_var('MetaData/sequenceNumber', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'Sequence Number') \ + .write_data(seqNum) + + # PreQC + obsspace.create_var('PreQC/waterTemperature', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + obsspace.create_var('PreQC/salinity', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + # ObsError + obsspace.create_var('ObsError/waterTemperature', dtype=ObsError_temp.dtype, fillval=ObsError_temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_temp) + + obsspace.create_var('ObsError/salinity', dtype=ObsError_saln.dtype, fillval=ObsError_saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_saln) + + # ObsValue + obsspace.create_var('ObsValue/waterTemperature', dtype=temp.dtype, fillval=temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('valid_range', np.array([-10.0, 50.0], dtype=np.float32)) \ + .write_attr('long_name', 'water Temperature') \ + .write_data(temp) + + obsspace.create_var('ObsValue/salinity', dtype=saln.dtype, fillval=saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('valid_range', np.array([0.0, 45.0], dtype=np.float32)) \ + .write_attr('long_name', 'salinity') \ + .write_data(saln) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: {running_time} \ + seconds") + + logger.debug(f"All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_xbtctd_profiles.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_altkob.py b/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_altkob.py new file mode 100755 index 000000000..0df4c88a0 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_insitu_surface_altkob.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +# (C) Copyright 2024 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import numpy.ma as ma +import os +import argparse +import math +import calendar +import time +import copy +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger +import warnings +# suppress warnings +warnings.filterwarnings('ignore') + + +def Compute_sequenceNumber(lon): + lon_u, seqNum = np.unique(lon, return_inverse=True) + seqNum = seqNum.astype(np.int32) + logger.debug(f"Len of Sequence Number: {len(seqNum)}") + + return seqNum + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + # General Information + converter = 'BUFR to IODA Converter' + platform_description = 'Surface obs from ALTKOB: temperature and salinity' + + bufrfile = f"{cycle_datetime}-{cycle_type}.t{hh}z.{data_format}.tm00.bufr_d" + DATA_PATH = os.path.join(dump_dir, bufrfile) + logger.debug(f"{bufrfile}, {DATA_PATH}") + + # =========================================== + # Make the QuerySet for all the data we want + # =========================================== + start_time = time.time() + + logger.debug(f"Making QuerySet ...") + q = bufr.QuerySet() + + # MetaData + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('ryear', '*/RCYR') + q.add('rmonth', '*/RCMO') + q.add('rday', '*/RCDY') + q.add('rhour', '*/RCHR') + q.add('rminute', '*/RCMI') + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + + # ObsValue + q.add('temp', '*/SST0') + q.add('saln', '*/SSS0') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # =============================================================== + # Open the BUFR file and execute the QuerySet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + logger.debug(f" ... Executing QuerySet: get MetaData ...") + dateTime = r.get_datetime('year', 'month', 'day', 'hour', 'minute', group_by='depth') + dateTime = dateTime.astype(np.int64) + rcptdateTime = r.get_datetime('ryear', 'rmonth', 'rday', 'rhour', 'rminute', group_by='depth') + rcptdateTime = rcptdateTime.astype(np.int64) + lat = r.get('latitude', group_by='depth') + lon = r.get('longitude', group_by='depth') + + # ObsValue + logger.debug(f" ... Executing QuerySet: get ObsValue ...") + temp = r.get('temp', group_by='depth') + temp -= 273.15 + saln = r.get('saln', group_by='depth') + + # Add mask based on min, max values + mask = ((temp > -10.0) & (temp <= 50.0)) & ((saln >= 0.0) & (saln <= 45.0)) + lat = lat[mask] + lon = lon[mask] + dateTime = dateTime[mask] + rcptdateTime = rcptdateTime[mask] + temp = temp[mask] + saln = saln[mask] + + # ObsError + logger.debug(f"Generating ObsError array with constant value (instrument error)...") + ObsError_temp = np.float32(np.ma.masked_array(np.full((len(mask)), 0.30))) + ObsError_saln = np.float32(np.ma.masked_array(np.full((len(mask)), 1.00))) + + # PreQC + logger.debug(f"Generating PreQC array with 0...") + PreQC = (np.ma.masked_array(np.full((len(mask)), 0))).astype(np.int32) + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + # ================================================== + # Check values of BUFR variables, dimension and type + # ================================================== + logger.debug(f" temp min, max, length, dtype = {temp.min()}, {temp.max()}, {len(temp)}, {temp.dtype}") + logger.debug(f" saln min, max, length, dtype = {saln.min()}, {saln.max()}, {len(saln)}, {saln.dtype}") + logger.debug(f" lon min, max, length, dtype = {lon.min()}, {lon.max()}, {len(lon)}, {lon.dtype}") + logger.debug(f" lat min, max, length, dtype = {lat.min()}, {lat.max()}, {len(lat)}, {lat.dtype}") + logger.debug(f" depth min, max, length, dtype = {depth.min()}, {depth.max()}, {len(depth)}, {depth.dtype}") + logger.debug(f" PreQC min, max, length, dtype = {PreQC.min()}, {PreQC.max()}, {len(PreQC)}, {PreQC.dtype}") + logger.debug(f" ObsError_temp min, max, length, dtype = {ObsError_temp.min()}, {ObsError_temp.max()}, {len(ObsError_temp)}, {ObsError_temp.dtype}") + logger.debug(f" ObsError_saln min, max, length, dtype = {ObsError_saln.min()}, {ObsError_saln.max()}, {len(ObsError_saln)}, {ObsError_saln.dtype}") + logger.debug(f" stationID shape, dtype = {stationID.shape}, {stationID.astype(str).dtype}") + logger.debug(f" dateTime shape, dtype = {dateTime.shape}, {dateTime.dtype}") + logger.debug(f" rcptdateTime shape, dytpe = {rcptdateTime.shape}, {rcptdateTime.dtype}") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}_profiles.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeRange', [str(dateTime.min()), str(dateTime.max())]) + obsspace.write_attr('platformLongDescription', platform_description) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, and attributes") + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # rcptDatetime + obsspace.create_var('MetaData/rcptdateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'receipt Datetime') \ + .write_data(rcptdateTime) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Station Identification + obsspace.create_var('MetaData/stationID', dtype=stationID.dtype, fillval=stationID.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stationID) + + # PreQC + obsspace.create_var('PreQC/seaSurfaceTemperature', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + obsspace.create_var('PreQC/seaSurfaceSalinity', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + # ObsError + obsspace.create_var('ObsError/seaSurfaceTemperature', dtype=ObsError_temp.dtype, fillval=ObsError_temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_temp) + + obsspace.create_var('ObsError/seaSurfaceSalinity', dtype=ObsError_saln.dtype, fillval=ObsError_saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_saln) + + # ObsValue + obsspace.create_var('ObsValue/seaSurfaceTemperature', dtype=temp.dtype, fillval=temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('valid_range', np.array([-10.0, 50.0], dtype=np.float32)) \ + .write_attr('long_name', 'Sea Surface Temperature') \ + .write_data(temp) + + obsspace.create_var('ObsValue/seaSurfaceSalinity', dtype=saln.dtype, fillval=saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('valid_range', np.array([0.0, 45.0], dtype=np.float32)) \ + .write_attr('long_name', 'Sea Surface Salinity') \ + .write_data(saln) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: {running_time} \ + seconds") + + logger.debug(f"All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_insitu_surface_altkob.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") From 12a09035b724dfe8289a1a1aa336ec0e310da966 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Mon, 25 Mar 2024 15:14:14 -0500 Subject: [PATCH 25/28] monthly insitu json --- .../bufr2ioda/bufr2ioda_insitu_profile_argo.json | 12 ++++++++++++ .../bufr2ioda/bufr2ioda_insitu_profile_glider.json | 12 ++++++++++++ .../bufr2ioda_insitu_profile_marinemammals.json | 12 ++++++++++++ .../bufr2ioda/bufr2ioda_insitu_profile_xbtctd.json | 12 ++++++++++++ 4 files changed, 48 insertions(+) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_marinemammals.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.json diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.json new file mode 100644 index 000000000..d05cdc7f0 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_argo.json @@ -0,0 +1,12 @@ +{ + "data_format" : "subpfl", + "subsets" : "SUBPFL", + "source" : "NCEP data tank", + "data_type" : "argo", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ ARGO profiles", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.json new file mode 100644 index 000000000..73b0b1902 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_glider.json @@ -0,0 +1,12 @@ +{ + "data_format" : "subpfl", + "subsets" : "SUBPFL", + "source" : "NCEP data tank", + "data_type" : "glider", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ GLIDER profiles", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_marinemammals.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_marinemammals.json new file mode 100644 index 000000000..8688ae9d7 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_marinemammals.json @@ -0,0 +1,12 @@ +{ + "data_format" : "tesac", + "subsets" : "TESAC", + "source" : "NCEP data tank", + "data_type" : "marinemammals", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ Marine Mammals profiles", + "data_provider" : "U.S. NOAA" +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.json new file mode 100644 index 000000000..3a686a162 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_profile_xbtctd.json @@ -0,0 +1,12 @@ +{ + "data_format" : "xbtctd", + "subsets" : "XBTCTD", + "source" : "NCEP data tank", + "data_type" : "xbtctd", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ XBT/XCTD profiles", + "data_provider" : "U.S. NOAA" +} From 8d946ae5afdc43f5d29def837755f64faa233a89 Mon Sep 17 00:00:00 2001 From: ShastriPaturi Date: Mon, 25 Mar 2024 15:17:14 -0500 Subject: [PATCH 26/28] monthly insitu json: altkob --- .../bufr2ioda/bufr2ioda_insitu_surface_altkob.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_altkob.json diff --git a/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_altkob.json b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_altkob.json new file mode 100644 index 000000000..ccbfe0a43 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_insitu_surface_altkob.json @@ -0,0 +1,12 @@ +{ + "data_format" : "altkob", + "subsets" : "ALTKOB", + "source" : "NCEP data tank", + "data_type" : "altkob", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hrly in situ ALTKOB surface", + "data_provider" : "U.S. NOAA" +} From 98682c2d29c7a6cca4c1babb3ff724b97cecc63d Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 26 Mar 2024 21:31:57 +0000 Subject: [PATCH 27/28] make obs prep ctest work without new jjob --- parm/soca/obs/obs_list.yaml | 2 +- test/soca/gw/CMakeLists.txt | 2 +- test/soca/gw/run_jjobs.yaml.test | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index f7be24da5..9a28ff56e 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -22,7 +22,7 @@ observers: - !INC ${OBS_YAML_DIR}/insitu_profile_glider.yaml - !INC ${OBS_YAML_DIR}/insitu_profile_tesac.yaml - !INC ${OBS_YAML_DIR}/insitu_profile_tesac_salinity.yaml -- !INC ${OBS_YAML_DIR}/insitu_profile_marinemammal.yaml + #- !INC ${OBS_YAML_DIR}/insitu_profile_marinemammal.yaml - !INC ${OBS_YAML_DIR}/insitu_profile_xbtctd.yaml - !INC ${OBS_YAML_DIR}/insitu_surface_altkob.yaml - !INC ${OBS_YAML_DIR}/insitu_surface_trkob.yaml diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt index 5220c7e40..150047142 100644 --- a/test/soca/gw/CMakeLists.txt +++ b/test/soca/gw/CMakeLists.txt @@ -75,7 +75,7 @@ foreach(jjob ${jjob_list}) WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun) set_tests_properties(${test_name} PROPERTIES - ENVIRONMENT "PYTHONPATH=${PROJECT_SOURCE_DIR}/ush:${PROJECT_BINARY_DIR}/../lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}") + ENVIRONMENT "PYTHONPATH=${PROJECT_SOURCE_DIR}/ush/ioda/bufr2ioda::${PROJECT_SOURCE_DIR}/ush:${PROJECT_BINARY_DIR}/../lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}") set(setup "--skip") # Only run the setup of the first test, if not, it will hang diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index 356439204..b18b53489 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -37,6 +37,10 @@ gw environement: GDASPREPOCNOBSPY: @HOMEgfs@/sorc/gdas.cd/scripts/exglobal_prep_ocean_obs.py GDASOCNCENPY: @HOMEgfs@/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_ecen.py + source directories: + JSON_TMPL_DIR: @HOMEgfs@/sorc/gdas.cd/parm/ioda/bufr2ioda + BUFR2IODA_PY_DIR: @HOMEgfs@/sorc/gdas.cd/ush/ioda/bufr2ioda + setup_expt config: base: DO_JEDIATMVAR: "NO" From 302c3289dd9468ed97280ce2b068c13b9661fb4d Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 28 Mar 2024 09:53:14 -0400 Subject: [PATCH 28/28] yaml tidy --- .../config/{ => dev}/insitu_profile_argo.yaml | 0 .../{ => dev}/insitu_profile_bathy.yaml | 0 .../{ => dev}/insitu_profile_dbuoy.yaml | 0 .../{ => dev}/insitu_profile_dbuoyb.yaml | 0 .../{ => dev}/insitu_profile_glider.yaml | 0 .../insitu_profile_marinemammal.yaml | 0 .../{ => dev}/insitu_profile_mbuoy.yaml | 0 .../{ => dev}/insitu_profile_mbuoyb.yaml | 0 .../{ => dev}/insitu_profile_tesac.yaml | 0 .../insitu_profile_tesac_salinity.yaml | 0 .../{ => dev}/insitu_profile_xbtctd.yaml | 0 .../{ => dev}/insitu_surface_altkob.yaml | 0 .../{ => dev}/insitu_surface_trkob.yaml | 0 .../insitu_surface_trkob_salinity.yaml | 0 .../config/{ => old}/bufr/bufr_sfcships.yaml | 0 .../config/{ => old}/bufr/bufr_sfcshipsu.yaml | 0 parm/soca/obs/obs_list.yaml | 20 +++++++++---------- 17 files changed, 10 insertions(+), 10 deletions(-) rename parm/soca/obs/config/{ => dev}/insitu_profile_argo.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_profile_bathy.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_profile_dbuoy.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_profile_dbuoyb.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_profile_glider.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_profile_marinemammal.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_profile_mbuoy.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_profile_mbuoyb.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_profile_tesac.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_profile_tesac_salinity.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_profile_xbtctd.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_surface_altkob.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_surface_trkob.yaml (100%) rename parm/soca/obs/config/{ => dev}/insitu_surface_trkob_salinity.yaml (100%) rename parm/soca/obs/config/{ => old}/bufr/bufr_sfcships.yaml (100%) rename parm/soca/obs/config/{ => old}/bufr/bufr_sfcshipsu.yaml (100%) diff --git a/parm/soca/obs/config/insitu_profile_argo.yaml b/parm/soca/obs/config/dev/insitu_profile_argo.yaml similarity index 100% rename from parm/soca/obs/config/insitu_profile_argo.yaml rename to parm/soca/obs/config/dev/insitu_profile_argo.yaml diff --git a/parm/soca/obs/config/insitu_profile_bathy.yaml b/parm/soca/obs/config/dev/insitu_profile_bathy.yaml similarity index 100% rename from parm/soca/obs/config/insitu_profile_bathy.yaml rename to parm/soca/obs/config/dev/insitu_profile_bathy.yaml diff --git a/parm/soca/obs/config/insitu_profile_dbuoy.yaml b/parm/soca/obs/config/dev/insitu_profile_dbuoy.yaml similarity index 100% rename from parm/soca/obs/config/insitu_profile_dbuoy.yaml rename to parm/soca/obs/config/dev/insitu_profile_dbuoy.yaml diff --git a/parm/soca/obs/config/insitu_profile_dbuoyb.yaml b/parm/soca/obs/config/dev/insitu_profile_dbuoyb.yaml similarity index 100% rename from parm/soca/obs/config/insitu_profile_dbuoyb.yaml rename to parm/soca/obs/config/dev/insitu_profile_dbuoyb.yaml diff --git a/parm/soca/obs/config/insitu_profile_glider.yaml b/parm/soca/obs/config/dev/insitu_profile_glider.yaml similarity index 100% rename from parm/soca/obs/config/insitu_profile_glider.yaml rename to parm/soca/obs/config/dev/insitu_profile_glider.yaml diff --git a/parm/soca/obs/config/insitu_profile_marinemammal.yaml b/parm/soca/obs/config/dev/insitu_profile_marinemammal.yaml similarity index 100% rename from parm/soca/obs/config/insitu_profile_marinemammal.yaml rename to parm/soca/obs/config/dev/insitu_profile_marinemammal.yaml diff --git a/parm/soca/obs/config/insitu_profile_mbuoy.yaml b/parm/soca/obs/config/dev/insitu_profile_mbuoy.yaml similarity index 100% rename from parm/soca/obs/config/insitu_profile_mbuoy.yaml rename to parm/soca/obs/config/dev/insitu_profile_mbuoy.yaml diff --git a/parm/soca/obs/config/insitu_profile_mbuoyb.yaml b/parm/soca/obs/config/dev/insitu_profile_mbuoyb.yaml similarity index 100% rename from parm/soca/obs/config/insitu_profile_mbuoyb.yaml rename to parm/soca/obs/config/dev/insitu_profile_mbuoyb.yaml diff --git a/parm/soca/obs/config/insitu_profile_tesac.yaml b/parm/soca/obs/config/dev/insitu_profile_tesac.yaml similarity index 100% rename from parm/soca/obs/config/insitu_profile_tesac.yaml rename to parm/soca/obs/config/dev/insitu_profile_tesac.yaml diff --git a/parm/soca/obs/config/insitu_profile_tesac_salinity.yaml b/parm/soca/obs/config/dev/insitu_profile_tesac_salinity.yaml similarity index 100% rename from parm/soca/obs/config/insitu_profile_tesac_salinity.yaml rename to parm/soca/obs/config/dev/insitu_profile_tesac_salinity.yaml diff --git a/parm/soca/obs/config/insitu_profile_xbtctd.yaml b/parm/soca/obs/config/dev/insitu_profile_xbtctd.yaml similarity index 100% rename from parm/soca/obs/config/insitu_profile_xbtctd.yaml rename to parm/soca/obs/config/dev/insitu_profile_xbtctd.yaml diff --git a/parm/soca/obs/config/insitu_surface_altkob.yaml b/parm/soca/obs/config/dev/insitu_surface_altkob.yaml similarity index 100% rename from parm/soca/obs/config/insitu_surface_altkob.yaml rename to parm/soca/obs/config/dev/insitu_surface_altkob.yaml diff --git a/parm/soca/obs/config/insitu_surface_trkob.yaml b/parm/soca/obs/config/dev/insitu_surface_trkob.yaml similarity index 100% rename from parm/soca/obs/config/insitu_surface_trkob.yaml rename to parm/soca/obs/config/dev/insitu_surface_trkob.yaml diff --git a/parm/soca/obs/config/insitu_surface_trkob_salinity.yaml b/parm/soca/obs/config/dev/insitu_surface_trkob_salinity.yaml similarity index 100% rename from parm/soca/obs/config/insitu_surface_trkob_salinity.yaml rename to parm/soca/obs/config/dev/insitu_surface_trkob_salinity.yaml diff --git a/parm/soca/obs/config/bufr/bufr_sfcships.yaml b/parm/soca/obs/config/old/bufr/bufr_sfcships.yaml similarity index 100% rename from parm/soca/obs/config/bufr/bufr_sfcships.yaml rename to parm/soca/obs/config/old/bufr/bufr_sfcships.yaml diff --git a/parm/soca/obs/config/bufr/bufr_sfcshipsu.yaml b/parm/soca/obs/config/old/bufr/bufr_sfcshipsu.yaml similarity index 100% rename from parm/soca/obs/config/bufr/bufr_sfcshipsu.yaml rename to parm/soca/obs/config/old/bufr/bufr_sfcshipsu.yaml diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index 9a28ff56e..e1d6f2931 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -17,16 +17,16 @@ observers: - !INC ${OBS_YAML_DIR}/icec_amsr2_south.yaml # in situ: monthly -- !INC ${OBS_YAML_DIR}/insitu_profile_bathy.yaml -- !INC ${OBS_YAML_DIR}/insitu_profile_argo.yaml -- !INC ${OBS_YAML_DIR}/insitu_profile_glider.yaml -- !INC ${OBS_YAML_DIR}/insitu_profile_tesac.yaml -- !INC ${OBS_YAML_DIR}/insitu_profile_tesac_salinity.yaml - #- !INC ${OBS_YAML_DIR}/insitu_profile_marinemammal.yaml -- !INC ${OBS_YAML_DIR}/insitu_profile_xbtctd.yaml -- !INC ${OBS_YAML_DIR}/insitu_surface_altkob.yaml -- !INC ${OBS_YAML_DIR}/insitu_surface_trkob.yaml -- !INC ${OBS_YAML_DIR}/insitu_surface_trkob_salinity.yaml +#- !INC ${OBS_YAML_DIR}/insitu_profile_bathy.yaml +#- !INC ${OBS_YAML_DIR}/insitu_profile_argo.yaml +#- !INC ${OBS_YAML_DIR}/insitu_profile_glider.yaml +#- !INC ${OBS_YAML_DIR}/insitu_profile_tesac.yaml +#- !INC ${OBS_YAML_DIR}/insitu_profile_tesac_salinity.yaml +#- !INC ${OBS_YAML_DIR}/insitu_profile_marinemammal.yaml +#- !INC ${OBS_YAML_DIR}/insitu_profile_xbtctd.yaml +#- !INC ${OBS_YAML_DIR}/insitu_surface_altkob.yaml +#- !INC ${OBS_YAML_DIR}/insitu_surface_trkob.yaml +#- !INC ${OBS_YAML_DIR}/insitu_surface_trkob_salinity.yaml # in situ: daily #- !INC ${OBS_YAML_DIR}/insitu_profile_dbuoy.yaml