From c9e85e57a81dfd69e47f00c3a1152ff561a19835 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Mon, 18 Jul 2022 10:11:45 -0500 Subject: [PATCH 01/13] transfer of files from GSI fork to GSI-utils fork --- src/EFSOI_Utilities/scripts/README_fv3gfs | 3 + .../scripts/consolidate_osense.py | 121 +++++++++ .../scripts/convdata_codes.csv | 230 +++++++++++------- src/EFSOI_Utilities/scripts/copyefsoifiles.sh | 125 ++++++++++ src/EFSOI_Utilities/scripts/geoplot.py | 95 ++++++++ src/EFSOI_Utilities/scripts/getefsiofiles.sh | 77 ++++++ src/EFSOI_Utilities/scripts/osense.py | 196 +++++++-------- src/EFSOI_Utilities/scripts/osense2nc.py | 88 +++++++ src/EFSOI_Utilities/scripts/osense_old.py | 136 +++++++++++ .../scripts/readcycsatconsolidate.py | 134 ++++++++++ 10 files changed, 1016 insertions(+), 189 deletions(-) create mode 100755 src/EFSOI_Utilities/scripts/consolidate_osense.py create mode 100755 src/EFSOI_Utilities/scripts/copyefsoifiles.sh create mode 100644 src/EFSOI_Utilities/scripts/geoplot.py create mode 100755 src/EFSOI_Utilities/scripts/getefsiofiles.sh create mode 100644 src/EFSOI_Utilities/scripts/osense2nc.py create mode 100644 src/EFSOI_Utilities/scripts/osense_old.py create mode 100755 src/EFSOI_Utilities/scripts/readcycsatconsolidate.py diff --git a/src/EFSOI_Utilities/scripts/README_fv3gfs b/src/EFSOI_Utilities/scripts/README_fv3gfs index b7643540..a7e83112 100644 --- a/src/EFSOI_Utilities/scripts/README_fv3gfs +++ b/src/EFSOI_Utilities/scripts/README_fv3gfs @@ -15,6 +15,9 @@ from osense file generated by EFSOI executable osense.py: contains routine to read osense file +osense_old.py: contains routine to read older osense file without assimilated flag +-- try this if osense.py throws exception on reading + convdata_codes.csv: codes needed to organize convetional data sources, derived from webpage (with the help of tools in pandas): https://www.emc.ncep.noaa.gov/mmb/data_processing/prepbufr.doc/table_2.htm diff --git a/src/EFSOI_Utilities/scripts/consolidate_osense.py b/src/EFSOI_Utilities/scripts/consolidate_osense.py new file mode 100755 index 00000000..0e8e7b08 --- /dev/null +++ b/src/EFSOI_Utilities/scripts/consolidate_osense.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import osense +import pandas as pd +import pickle +import datetime +import os.path + +exp='rodeo' +#exp='baily' + +#datadir='/work2/noaa/stmp/aeichman/comrot/' + exp + '/osense/' +#datadir='/work/noaa/stmp/aeichman/comrot/' + exp + '/osense/' +#rootdir = '/Users/mossland/Desktop/work/EFSOI/' + exp +rootdir = '/work/noaa/da/aeichman/gsidev/EFSOI-dev-post/GSI/util/EFSOI_Utilities/scripts/' + exp +indir = '/work2/noaa/stmp/aeichman/comrot/rodeo/osense/' +outdir = rootdir + '/consolidated/' + +# baily +cycleinit = datetime.datetime( 2021, 12,10, 0 ) +cycleend = datetime.datetime( 2021, 12, 14, 18 ) + +# rodeo +cycleinit = datetime.datetime( 2020, 12,14, 12 ) +cycleinit = datetime.datetime( 2020, 12,14, 18 ) +#cycleend = datetime.datetime( 2021, 1, 13, 18 ) + + +# The following determines whether the conventional data points are aggregated +# by the column in convdata_codes.cvs "message02" (DETAILED = False ) or "message" +# (DETAILED = True). Generally message offers finer distinctions between the +# sources, though these are somewhat arbitrary. See convdata_codes.csv +DETAILED = True + +# these are satellite sensors to consolidate across plaforms +sensors = ['airs', + 'amsr', + 'amsua', + 'atms', + 'avhrr', + 'cris', + 'iasi', + 'mhs', + 'saphir', + 'seviri', + 'ssmis'] + +# these are fields in the osense files and can be expanded or reduced +osensefields = ['source', + 'detailed_source', + 'indxsat', + 'osense_kin', + 'osense_dry', + 'osense_moist', + 'assimilated', + 'lat', + 'lon', + 'pres'] + +# time between cycles +cycles = pd.date_range( cycleinit, cycleend, freq='6H' ) + + +for thiscycle in cycles: + + CDATE = thiscycle.strftime("%Y%m%d%H") + PDY = thiscycle.strftime("%Y%m%d") + cyc = thiscycle.strftime("%H") + + filename = indir + 'osense_' + CDATE + '_final.dat' + + if not os.path.isfile(filename): + print('skipping ' + filename) + continue + + ( convdata, satdata, idate )= osense.read_osense( filename ) + + satdata['detailed_source'] = satdata['obtype'] + satdata['source'] = satdata['obtype'] + + for sensor in sensors: + mask = satdata.source.str.contains(sensor) + satdata.loc[mask,'source'] = sensor.upper() + + + # now consolidate conventional data + + convcodes = pd.read_csv('convdata_codes.csv') + + # associate each data point with its source, by code + # this also effectively adds the columns 'source' and 'detailed_source' + convbycodes=pd.merge(convdata,convcodes,how='left', + left_on='stattype', + right_on='code') + + # drop message='NA'/message02='Empty' + # data not dumped/not used + indices=convbycodes[convbycodes['source']=='Empty'].index + convbycodes.drop(indices,inplace = True) + + # some stattypes, namely 700, have no corresponding code, and so have + # nans in the code, source, and detailed_source fields. This replaces those + # fields with the stattype + nanmask = convbycodes.code.isna() + nanstattypes = convbycodes.loc[nanmask,['stattype']] + for nanstattype in nanstattypes.stattype.unique(): + convbycodes.loc[convbycodes['stattype'] == nanstattype, \ + ['source','detailed_source']] = nanstattype + + + osensedata = pd.concat([ satdata[osensefields], convbycodes[osensefields] ] ) + + + outfilename = outdir + 'osense_' + CDATE + '.pkl' + outfile = open( outfilename, 'wb' ) + pickle.dump([ idate, osensedata ], outfile ) + outfile.close() + + + diff --git a/src/EFSOI_Utilities/scripts/convdata_codes.csv b/src/EFSOI_Utilities/scripts/convdata_codes.csv index 05c0c7be..15fcb10e 100644 --- a/src/EFSOI_Utilities/scripts/convdata_codes.csv +++ b/src/EFSOI_Utilities/scripts/convdata_codes.csv @@ -1,92 +1,138 @@ -,type,message,description,note,code -0,111,SYNDAT,"SYNTHETIC (BOGUS) TROPICAL CYCLONE STORM CENTER (generated in SYNDAT_SYNDATA) - q, Pstn",Not created (switch not set in SYNDAT_SYNDATA parm cards),111 -1,112,,"PSEUDO MEAN SEA-LEVEL PRESSURE AT TROPICAL CYCLONE STORM CENTER (generated in GSI, does not appear in pre-analysis PREPBUFR files) - Pstn",Pstn used by assimilation When implemented. may appear in post-analysis PREPBUFR file,112 -2,120,ADPUPA,"RAWINSONDE - Tv, q, Pstn, sst","Tv, q, Pstn used by assimilation sst monitored by assimilation by switch in convinfo text file read by GBL-GSI",120 -3,122,ADPUPA,"CLASS SOUNDING - Tv, q, Pstn",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Entire report neither monitored nor assimilated - not in convinfo text file read by GBL-GSI,122 -4,126,RASSDA,RASS [FROM NOAA PROFILER NETWORK (NPN) OR MULTI-AGENCY PROFILER (MAP) NETWORK] - Tv,Tv flagged for non-use by assimilation due to missing obs error Tv monitored by assimilation by switch in convinfo text file read by GBL-GSI NPN data are no longer produced after 13 September 2017. Multiu-Agency data are still available.,126 -5,130,AIRCFT,AIREP AND PIREP AIRCRAFT - Ts,Ts used by assimilation,130 -6,131 R,AIRCFT,"AMDAR AIRCRAFT - Ts, q (E-AMDAR only)",q (E-AMDAR only) not considered by assimilation Ts used by assimilation,131 -7,132,ADPUPA,"FLIGHT-LEVEL RECONNAISSANCE AND PROFILE DROPSONDE - Tv, q, Pstn","q (non-U.S. drops, NASA global Hawk drops, all levels), Pstn (all drops) [and surface level Tv (all drops), surface level q (all drops)] flagged for non-use by assimilation by switch in PREPOBS_PREPDATA parm cards Tv (all drops, above surface; reccos), q [U.S. (NOAA Gulf Stream and P-3, and USAF) drops, above surface; reccos] used by assimilation q (non-U.S. drops, NASA global Hawk drops, all levels) and Tv (all drops, at surface) monitored by assimilation due to their being flagged by switch in PREPOBS_PREPDATA parm cards",132 -8,133 R,AIRCAR,"MDCRS ACARS AIRCRAFT - Ts, q","Ts, q used by assimilation",133 -9,134 R,AIRCFT,"TAMDAR AIRCRAFT - Ts, q","Ts, q monitored by assimilation by switch in convinfo text file read by GBL-GSI",134 -10,135 R,AIRCFT,CANADIAN AMDAR AIRCRAFT - Ts,Ts monitored by assimilation by switch in convinfo text file read by GBL-GSI,135 -11,150,SPSSMI,SSM/I SUPEROBED (1 DEGREE LAT/LON) FNMOC (OPERATIONAL) RAIN RATE (DMSP) - rr,rr monitored by assimilation by switch in pcpinfo text file read by GBL-GSI Currently the assimilation obtains this directly from the spssmi dump file rather than from PREPBUFR file. The SSM/I F-13 satellite went bad in November 2009 resulting in no data being processed upstream.  The empty dumps were turned off and the PREPOBS_PREPDATA parm cards were set to no longer process these data in October 2010.,150 -12,151,GOESND,"NESDIS 1x1 F-O-V CLOUD TOP PRESSURE, TEMPERATURE; CLOUD AMOUNT (GOES)",Not dumped,151 -13,152,SPSSMI,SSM/I SUPEROBED (1 DEGREE LAT/LON) NEURAL NET-3 PRECIPITABLE WATER OVER OCEAN (DMSP) - PWt,PWt flagged for non-use by assimilation due to missing obs error PWt monitored by assimilation by switch in convinfo text file read by GBL-GSI The SSM/I F-13 satellite went bad in November 2009 resulting in no data being processed upstream.  The empty dumps were turned off and the PREPOBS_PREPDATA parm cards were set to no longer process these data in October 2010.,152 -14,153,GPSIPW,GPS-INTEGRATED PRECIPITABLE WATER (GPS-IPW) - PWt,Currently only reports from ENI (mainly over U.S.) are encoded into PREPBUFR file. European GNSS reports are skipped. PWt flagged for non-use by assimilation due to missing obs error PWt monitored by assimilation by switch in convinfo text file read by GBL-GSI,153 -15,154,,reserved for GOES IMAGER SKY-COVER DATA used only in RTMA/URMA,,154 -16,156,GOESND,NESDIS 1x1 F-O-V 4-LAYER PRECIPITABLE WATER OVER LAND - CLEAR (GOES) - PWl,Not dumped,156 -17,157,GOESND,NESDIS 1x1 F-O-V 4-LAYER PRECIPITABLE WATER OVER LAND - CLOUDY (GOES) - PWl,Not dumped,157 -18,158,GOESND,NESDIS 1x1 F-O-V 4-LAYER PRECIPITABLE WATER OVER OCEAN - CLEAR (GOES) - PWl,Not dumped,158 -19,159,GOESND,NESDIS 1x1 F-O-V 4-LAYER PRECIPITABLE WATER OVER OCEAN - CLOUDY (GOES) - PWl,Not dumped,159 -20,164,GOESND,NESDIS 1x1 F-O-V RADIANCES OVER LAND - CLEAR (GOES) - Tb,Not dumped,164 -21,165,GOESND,NESDIS 1x1 F-O-V RADIANCES OVER LAND - CLOUDY (GOES) - Tb,Not dumped,165 -22,170,,"NACELLE - Tb, q",Not dumped,170 -23,171,,"TALL TOWER - Tb, q",Not dumped,171 -24,174,GOESND,NESDIS 1x1 F-O-V RADIANCES OVER OCEAN - CLEAR (GOES) - Tb,Not dumped,174 -25,175,GOESND,NESDIS 1x1 F-O-V RADIANCES OVER OCEAN - CLOUDY (GOES) - Tb,Not dumped,175 -26,180 (R - U.S. & JMA SHIPS),SFCSHP,"SURFACE MARINE WITH REPORTED STATION PRESSURE (SHIP, BUOY, C-MAN, TIDE GAUGE) - Tv, q, Pstn, sst","Tv, q, Pstn used by assimilation sst monitored by assimilation by switch in convinfo text file read by GBL-GSI",180 -27,181 (R - WMO Res 40 SYNOPS),ADPSFC,"SURFACE LAND [SYNOPTIC (fixed and mobile), METAR] WITH REPORTED STATION PRESSURE - Tv, q, Pstn, sst","Pstn used by assimilation Tv, q flagged for non-use by assimilation due to missing obs error Tv, q, sst monitored by assimilation by switch in convinfo text file read by GBL-GSI",181 -28,182,SFCSHP,"SPLASH-LEVEL DROPSONDE OVER OCEAN - Tv, q, Pstn","Tv, q, Pstn used by assimilation",182 -29,"183 (R - WMO Res 40 SYNOPS, U.S. & JMA SHIPS)","ADPSFC, SFCSHP","SURFACE MARINE (SHIP, BUOY, C-MAN, TIDE GAUGE) OR LAND [SYNOPTIC (fixed and mobile), METAR] WITH MISSING STATION PRESSURE - Tv, q, Pstn, sst","Tv, q, Pstn (entire report) flagged for non-use by assimilation due to missing obs error Tv, q, Pstn, sst monitored by assimilation by switch in convinfo text file read by GBL-GSI Altimeter setting is also missing (always the case for synoptic). Station pressure calculated from reported mean sea-level pressure and elevation via U.S. Standard Atmosphere approximation. Elevation is greater than 7.5 meters (if less than 7.5 meters, station pressure set equal to sea-level pressure and report type set to 181).",183 -30,187,ADPSFC,"SURFACE LAND (METAR) WITH MISSING STATION PRESSURE - Tv, q, Pstn, sst","Pstn used by assimilation Tv, q flagged for non-use by assimilation due to missing obs error Tv, q, sst monitored by assimilation by switch in convinfo text file read by GBL-GSI Altimeter setting is reported (never the case for synoptic). Station pressure calculated from reported altimeter setting and elevation.",187 -31,188 R,MSONET,"SURFACE MESONET - Tv, q, Pstn",Not dumped,188 -32,191,SFCBOG,AUSTRALIAN PAOB MEAN SEA-LEVEL PRESSURE BOGUS OVER OCEAN - Pstn,sfcbog dump file not read by PREPOBS_PREPDATA This data no longer produced after 17 August 2010,191 -33,192,ADPSFC,"SURFACE LAND SYNOPIC (fixed and mobile) WITH MISSING STATION PRESSURE AND MISSING SEA-LEVEL PRESSURE - Tv, q, Pstn",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,192 -34,193,ADPSFC,"SURFACE LAND METAR WITH MISSING STATION PRESSURE, MISSING SEA-LEVEL PRESSURE AND MISSING ALTIMETER SETTING - Tv, q, Pstn",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,193 -35,194,SFCSHP,"SURFACE MARINE (SHIP, BUOY, C-MAN, TIDE GAUGE) OR LAND (SYNOPTIC, METAR) WITH MISSING STATION PRESSURE AND MISSING SEA-LEVEL PRESSURE - Tv, q, Pstn",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,194 -36,195,MSONET,"SURFACE MESONET WITH MISSING STATION PRESSURE AND MISSING ALTIMETER SETTING (SEA-LEVEL PRESSURE IS ALWAYS MISSING) - Tv, q, Pstn",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,195 -37,210,SYNDAT,"SYNTHETIC (BOGUS) TROPICAL CYCLONE - u, v","u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI Added to PREPBUFR file by later program SYNDAT_SYNDATA.",210 -38,220,ADPUPA,"RAWINSONDE - u, v (all levels), z (winds-by-height levels)","u, v used by assimilation p (vertical coordinate) calculated from z on winds-by-height levels.",220 -39,221,ADPUPA,"PIBAL - u,v,z","u, v used by assimilation p (vertical coordinate) calculated from z.",221 -40,222,ADPUPA,"CLASS SOUNDING - u, v",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Entire report neither monitored nor assimilated - not in convinfo text file read by GBL-GSI,222 -41,223,PROFLR,"NOAA PROFILER NETWORK (NPN) WIND PROFILER - u, v, z","u, v used by assimilation p (vertical coordinate) calculated from z. These data are no longer produced after 13 September 2017.",223 -42,224,VADWND,"NEXRAD VERTICAL AZIMUTH DISPLAY (VAD) from Radar Coded Message (subtype 1) - u, v, z","u, v used by assimilation p (vertical coordinate) calculated from z.",224 -43,227,PROFLR,"MULTI-AGENCY PROFILER (MAP) AND ACOUSTIC SOUNDER (SODAR) - u, v, z",Entire report tossed by switch in PREPOBS_PREPDATA parm cards,227 -44,228,PROFLR,"JAPANESE METEOROLOGICAL AGENCY (JMA) WIND PROFILER - u, v, z","u, v (entire report) flagged for non-use by assimilation due to missing obs error u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI",228 -45,229,PROFLR,"WIND PROFILER DECODED FROM PILOT (PIBAL) BULLETINS - u, v, z","u, v used by assimilation p (vertical coordinate) calculated from z.",229 -46,230,AIRCFT,"AIREP AND PIREP AIRCRAFT - u, v","u, v used by assimilation",230 -47,231 R,AIRCFT,"AMDAR AIRCRAFT - u, v","u, v used by assimilation",231 -48,232,ADPUPA,"FLIGHT-LEVEL RECONNAISSANCE AND PROFILE DROPSONDE - u, v","u, v used by assimilation",232 -49,233 R,AIRCAR,"MDCRS ACARS AIRCRAFT - u, v","u, v used by assimilation",233 -50,234 R,AIRCFT,"TAMDAR AIRCRAFT - u, v","u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI",234 -51,235 R,AIRCFT,"CANADIAN AMDAR AIRCRAFT - u, v","u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI",235 -52,240,SATWND,"NESDIS IR (SHORT-WAVE) CLOUD DRIFT (ALL LEVELS) (GOES) - u, v",The assimilation will obtain this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This will not be written into the PREPBUFR file.,240 -53,241,SATWND,"INDIA IR (LONG-WAVE) AND VISIBLE CLOUD DRIFT (ALL LEVELS) (INSAT, KALPANA) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",241 -54,242,SATWND,"JMA IR (LONG-WAVE) AND VISIBLE CLOUD DRIFT AT LEVELS BELOW 850 MB (GMS, MTSAT, HIMAWARI) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI. The GSI redefines report type 242 as JMA visible cloud drift at all levels.",242 -55,243,SATWND,"EUMETSAT IR (LONG-WAVE) AND VISIBLE CLOUD DRIFT AT LEVELS BELOW 850 MB (METEOSAT) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI. The GSI redefines report type 243 as EUMETSAT visible cloud drift at all levels.",243 -56,244,SATWND,"AVHRR/POES IR (LONG-WAVE) CLOUD DRIFT (ALL LEVELS) (NOAA, METOP) - u,v",The assimilation will obtain this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This will not be written into the PREPBUFR file.,244 -57,245,SATWND,"NESDIS IR (LONG-WAVE) CLOUD DRIFT (ALL LEVELS) (GOES) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",245 -58,246,SATWND,"NESDIS IMAGER WATER VAPOR (ALL LEVELS) - CLOUD TOP (GOES) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",246 -59,247,SATWND,"NESDIS IMAGER WATER VAPOR (ALL LEVELS) - DEEP LAYER (GOES) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file (it was never in the PREPBUFR file) (see Table 18).  Since it has both a missing obs error and is set to monitor in the convinfo file, it is now monitored by the GBL-GSI.",247 -60,248,SATWND,"NESDIS SOUNDER WATER VAPOR (ALL LEVELS) - CLOUD TOP (GOES) - u, v","If ever processed, the assimilation will obtain this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This will never be written into the PREPBUFR file.",248 -61,249,SATWND,"NESDIS SOUNDER WATER VAPOR (ALL LEVELS) - DEEP LAYER (GOES) - u, v","If ever processed, the assimilation will obtain this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This will never be written into the PREPBUFR file.",249 -62,250,SATWND,"JMA IMAGER WATER VAPOR (ALL LEVELS) - CLOUD TOP & DEEP LAYER (GMS, MTSAT, HIMAWARI) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",250 -63,251,SATWND,"NESDIS VISIBLE CLOUD DRIFT (ALL LEVELS) (GOES) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",251 -64,252,SATWND,"JMA IR (LONG-WAVE) AND VISIBLE CLOUD DRIFT AT LEVELS ABOVE 850 MB (GMS, MTSAT, HIMAWARI) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI. The GSI redefines report type 252 as JMA IR cloud drift at all levels.",252 -65,253,SATWND,"EUMETSAT IR (LONG-WAVE) AND VISIBLE CLOUD DRIFT AT LEVELS ABOVE 850 MB (METEOSAT) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI. The GSI redefines report type 253 as EUMETSAT IR cloud drift at all levels.",253 -66,254,SATWND,"EUMETSAT IMAGER WATER VAPOR (ALL LEVELS) - CLOUD TOP & DEEP LAYER (METEOSAT) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",254 -67,255,SATWND,NESDIS PICTURE TRIPLET CLOUD DRIFT (LOW LEVELS) (GOES),No longer produced by NESDIS,255 -68,256,SATWND,"INDIA IMAGER WATER VAPOR (ALL LEVELS) (INSAT, KALPANA) - u, v","If ever processed, the assimilation will obtain this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This will never be written into the PREPBUFR file.",256 -69,257,SATWND,"MODIS/POES IR (LONG-WAVE) CLOUD DRIFT (ALL LEVELS) (AQUA, TERRA) - u,v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",257 -70,258,SATWND,"MODIS/POES IMAGER WATER VAPOR (ALL LEVELS) - CLOUD TOP (AQUA, TERRA) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",258 -71,259,SATWND,"MODIS/POES IMAGER WATER VAPOR (ALL LEVELS) - DEEP LAYER (AQUA, TERRA) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",259 -72,260,SATWND,"VIIRS/POES IR (LONG-WAVE) CLOUD DRIFT (ALL LEVELS) (NPP) - u,v",The assimilation will obtain this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This will not be written into the PREPBUFR file.,260 -73,270,,"NACELLE - u,v",Not dumped,270 -74,271,,"TALL TOWER -u,v",Not dumped,271 -75,280 (R - U.S. & JMA SHIPS),SFCSHP,"SURFACE MARINE WITH REPORTED STATION PRESSURE (SHIP, BUOY, C-MAN, TIDE GAUGE) - u, v","u, v used by assimilation",280 -76,281 (R - WMO Res 40 SYNOPS),ADPSFC,"SURFACE LAND [SYNOPTIC (fixed and mobile), METAR] WITH REPORTED STATION PRESSURE - u, v","u, v (entire report) flagged for non-use by assimilation due to missing obs error u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI",281 -77,282,SFCSHP,"ATLAS BUOY - u, v (see % below)","u, v used by assimilation Reported station pressure and mean sea-level pressure BOTH missing. Station pressure is set to 1013 mb. Elevation is less than or equal to 7.5 meters.",282 -78,283,SPSSMI,"SSM/I SUPEROBED (1 DEGREE LAT/LON) NEURAL NET 3 WIND SPEED OVER OCEAN - u, v","u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI Only wspd available so direction initially set to ZERO in PREPBUFR file, direction calculated from analysis and u, v recomputed in program PREPOBS_OIQCBUFR. Reported station pressure and mean sea-level pressure BOTH missing. Station pressure is set to 1013 mb. Elevation is less than or equal to 7.5 meters. The SSM/I F-13 satellite went bad in November 2009 resulting in no data being processed upstream.  The empty dumps were turned off and the PREPOBS_PREPDATA parm cards were set to no longer process these data in October 2010.",283 -79,"284 (R - WMO Res 40 SYNOPS, U.S. & JMA SHIPS)","ADPSFC, SFCSHP","SURFACE MARINE (SHIP, BUOY, C-MAN, TIDE GAUGE) OR LAND [SYNOPTIC (fixed and mobile), METAR] WITH MISSING STATION PRESSURE - u, v","u, v (entire report) flagged for non-use by assimilation due to missing obs error u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI Altimeter setting is also missing (always the case for synoptic). Station pressure calculated from reported mean sea-level pressure and elevation via U.S. Standard Atmosphere approximation. Elevation is greater than 7.5 meters (if less than 7.5 meters, station pressure set equal to sea-level pressure and report type set to 281).",284 -80,285,QKSWND,"SUPEROBED (0.5 DEGREE LAT/LON) SCATTEROMETER WINDS OVER OCEAN (QUIKSCAT) - u, v",No longer produced,285 -81,286,ERS1DA,"SCATTEROMETER WINDS OVER OCEAN (ERS) - u, v",No longer produced,286 -82,287,ADPSFC,"SURFACE LAND (METAR) WITH MISSING STATION PRESSURE - u, v","u, v (entire report) flagged for non-use by assimilation due to missing obs error u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI Altimeter setting is reported (never the case for synoptic). Station pressure calculated from reported altimeter setting and elevation.",287 -83,288 R,MSONET,"SURFACE MESONET - u, v",Not dumped,288 -84,289,WDSATR,"SUPEROBED (1.0 DEGREE LAT/LON) SCATTEROMETER WINDS OVER OCEAN (WINDSAT) - u,v","u, v used by assimilation (currently not available due to format change in raw files)",289 -85,290,ASCATW,"NON-SUPEROBED SCATTEROMETER WINDS OVER OCEAN (ASCAT) - u,v (50 km resolution)","METOP-2(A) u, v used by assimilation METOP-1(B) u,v monitored by assimilation by switch in convinfo text file read by GBL-GSI",290 -86,291,,"NON-SUPEROBED SCATTEROMETER WINDS OVER OCEAN (OSCAT) - u,v","When available, the assimilation will obtain this directly from the oscatw dump file rather than from the PREPBUFR file (see Table 18).  This will not be written into the PREPBUFR file. This type will never be available. Instrument failed on 20 February 2014.",291 -87,292,ADPSFC,"SURFACE LAND SYNOPIC (fixed and mobile) WITH MISSING STATION PRESSURE AND MISSING SEA-LEVEL PRESSURE - u,v",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,292 -88,293,ADPSFC,"SURFACE LAND METAR WITH MISSING STATION PRESSURE, MISSING SEA-LEVEL PRESSURE AND MISSING ALTIMETER SETTING - u,v",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,293 -89,294,SFCSHP,"SURFACE MARINE (SHIP, BUOY, C-MAN, TIDE GAUGE) OR LAND (SYNOPTIC, METAR) WITH MISSING STATION PRESSURE AND MISSING SEA-LEVEL PRESSURE - u,v",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,294 -90,295,MSONET,"SURFACE MESONET WITH MISSING STATION PRESSURE AND MISSING ALTIMETER SETTING (SEA-LEVEL PRESSURE IS ALWAYS MISSING) - u,v",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,295 +,type,detailed_source,source,description,note,code +0,111,SYNDAT,SYNDAT,"SYNTHETIC (BOGUS) TROPICAL CYCLONE STORM CENTER (generated in SYNDAT_SYNDATA) - q, Pstn",Not created (switch not set in SYNDAT_SYNDATA parm cards),111 +1,112,NA,Empty,"PSEUDO MEAN SEA-LEVEL PRESSURE AT TROPICAL CYCLONE STORM CENTER (generated in GSI, does not appear in pre-analysis PREPBUFR files) - Pstn",Pstn used by assimilation When implemented. may appear in post-analysis PREPBUFR file,112 +2,120,ADPUPA,Radiosonde,"RAWINSONDE - Tv, q, Pstn, sst","Tv, q, Pstn used by assimilation sst monitored by assimilation by switch in convinfo text file read by GBL-GSI",120 +3,122,ADPUPA,Radiosonde,"CLASS SOUNDING - Tv, q, Pstn",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Entire report neither monitored nor assimilated - not in convinfo text file read by GBL-GSI,122 +4,126,RASSDA,Profiler,RASS [FROM NOAA PROFILER NETWORK (NPN) OR MULTI-AGENCY PROFILER (MAP) NETWORK] - Tv,Tv flagged for non-use by assimilation due to missing obs error Tv monitored by assimilation by switch in convinfo text file read by GBL-GSI NPN data are no longer produced after 13 September 2017. Multiu-Agency data are still available.,126 +5,130,AIRCFT,Aircraft,AIREP AND PIREP AIRCRAFT - Ts,Ts used by assimilation,130 +6,131 R,AIRCFT,Aircraft,"AMDAR AIRCRAFT - Ts, q (E-AMDAR only)",q (E-AMDAR only) not considered by assimilation Ts used by assimilation,131 +7,132,ADPUPA,Dropsonde,"FLIGHT-LEVEL RECONNAISSANCE AND PROFILE DROPSONDE - Tv, q, Pstn","q (non-U.S. drops, NASA global Hawk drops, all levels), Pstn (all drops) [and surface level Tv (all drops), surface level q (all drops)] flagged for non-use by assimilation by switch in PREPOBS_PREPDATA parm cards Tv (all drops, above surface; reccos), q [U.S. (NOAA Gulf Stream and P-3, and USAF) drops, above surface; reccos] used by assimilation q (non-U.S. drops, NASA global Hawk drops, all levels) and Tv (all drops, at surface) monitored by assimilation due to their being flagged by switch in PREPOBS_PREPDATA parm cards",132 +8,133 R,AIRCAR,Aircraft,"MDCRS ACARS AIRCRAFT - Ts, q","Ts, q used by assimilation",133 +9,134 R,AIRCFT,Aircraft,"TAMDAR AIRCRAFT - Ts, q","Ts, q monitored by assimilation by switch in convinfo text file read by GBL-GSI",134 +10,135 R,AIRCFT,Aircraft,CANADIAN AMDAR AIRCRAFT - Ts,Ts monitored by assimilation by switch in convinfo text file read by GBL-GSI,135 +11,150,SPSSMI,SPSSMI,SSM/I SUPEROBED (1 DEGREE LAT/LON) FNMOC (OPERATIONAL) RAIN RATE (DMSP) - rr,rr monitored by assimilation by switch in pcpinfo text file read by GBL-GSI Currently the assimilation obtains this directly from the spssmi dump file rather than from PREPBUFR file. The SSM/I F-13 satellite went bad in November 2009 resulting in no data being processed upstream.  The empty dumps were turned off and the PREPOBS_PREPDATA parm cards were set to no longer process these data in October 2010.,150 +12,151,GOESND,GOESND,"NESDIS 1x1 F-O-V CLOUD TOP PRESSURE, TEMPERATURE; CLOUD AMOUNT (GOES)",Not dumped,151 +13,152,SPSSMI,SPSSMI,SSM/I SUPEROBED (1 DEGREE LAT/LON) NEURAL NET-3 PRECIPITABLE WATER OVER OCEAN (DMSP) - PWt,PWt flagged for non-use by assimilation due to missing obs error PWt monitored by assimilation by switch in convinfo text file read by GBL-GSI The SSM/I F-13 satellite went bad in November 2009 resulting in no data being processed upstream.  The empty dumps were turned off and the PREPOBS_PREPDATA parm cards were set to no longer process these data in October 2010.,152 +14,153,GPSIPW,GPSIPW,GPS-INTEGRATED PRECIPITABLE WATER (GPS-IPW) - PWt,Currently only reports from ENI (mainly over U.S.) are encoded into PREPBUFR file. European GNSS reports are skipped. PWt flagged for non-use by assimilation due to missing obs error PWt monitored by assimilation by switch in convinfo text file read by GBL-GSI,153 +15,154,NA,Empty,reserved for GOES IMAGER SKY-COVER DATA used only in RTMA/URMA,,154 +16,156,GOESND,GOESND,NESDIS 1x1 F-O-V 4-LAYER PRECIPITABLE WATER OVER LAND - CLEAR (GOES) - PWl,Not dumped,156 +17,157,GOESND,GOESND,NESDIS 1x1 F-O-V 4-LAYER PRECIPITABLE WATER OVER LAND - CLOUDY (GOES) - PWl,Not dumped,157 +18,158,GOESND,GOESND,NESDIS 1x1 F-O-V 4-LAYER PRECIPITABLE WATER OVER OCEAN - CLEAR (GOES) - PWl,Not dumped,158 +19,159,GOESND,GOESND,NESDIS 1x1 F-O-V 4-LAYER PRECIPITABLE WATER OVER OCEAN - CLOUDY (GOES) - PWl,Not dumped,159 +20,164,GOESND,GOESND,NESDIS 1x1 F-O-V RADIANCES OVER LAND - CLEAR (GOES) - Tb,Not dumped,164 +21,165,GOESND,GOESND,NESDIS 1x1 F-O-V RADIANCES OVER LAND - CLOUDY (GOES) - Tb,Not dumped,165 +22,170,NA,Empty,"NACELLE - Tb, q",Not dumped,170 +23,171,NA,Empty,"TALL TOWER - Tb, q",Not dumped,171 +24,174,GOESND,GOESND,NESDIS 1x1 F-O-V RADIANCES OVER OCEAN - CLEAR (GOES) - Tb,Not dumped,174 +25,175,GOESND,GOESND,NESDIS 1x1 F-O-V RADIANCES OVER OCEAN - CLOUDY (GOES) - Tb,Not dumped,175 +26,180 (R - U.S. & JMA SHIPS),SFCSHP,Ocean_Surface,"SURFACE MARINE WITH REPORTED STATION PRESSURE (SHIP, BUOY, C-MAN, TIDE GAUGE) - Tv, q, Pstn, sst","Tv, q, Pstn used by assimilation sst monitored by assimilation by switch in convinfo text file read by GBL-GSI",180 +27,181 (R - WMO Res 40 SYNOPS),ADPSFC,Land_Surface,"SURFACE LAND [SYNOPTIC (fixed and mobile), METAR] WITH REPORTED STATION PRESSURE - Tv, q, Pstn, sst","Pstn used by assimilation Tv, q flagged for non-use by assimilation due to missing obs error Tv, q, sst monitored by assimilation by switch in convinfo text file read by GBL-GSI",181 +28,182,SFCSHP,Dropsonde,"SPLASH-LEVEL DROPSONDE OVER OCEAN - Tv, q, Pstn","Tv, q, Pstn used by assimilation",182 +29,"183 (R - WMO Res 40 SYNOPS, U.S. & JMA SHIPS)","ADPSFC, SFCSHP",Ocean_Surface,"SURFACE MARINE (SHIP, BUOY, C-MAN, TIDE GAUGE) OR LAND [SYNOPTIC (fixed and mobile), METAR] WITH MISSING STATION PRESSURE - Tv, q, Pstn, sst","Tv, q, Pstn (entire report) flagged for non-use by assimilation due to missing obs error Tv, q, Pstn, sst monitored by assimilation by switch in convinfo text file read by GBL-GSI Altimeter setting is also missing (always the case for synoptic). Station pressure calculated from reported mean sea-level pressure and elevation via U.S. Standard Atmosphere approximation. Elevation is greater than 7.5 meters (if less than 7.5 meters, station pressure set equal to sea-level pressure and report type set to 181).",183 +30,187,ADPSFC,Land_Surface,"SURFACE LAND (METAR) WITH MISSING STATION PRESSURE - Tv, q, Pstn, sst","Pstn used by assimilation Tv, q flagged for non-use by assimilation due to missing obs error Tv, q, sst monitored by assimilation by switch in convinfo text file read by GBL-GSI Altimeter setting is reported (never the case for synoptic). Station pressure calculated from reported altimeter setting and elevation.",187 +31,188 R,MSONET,MSONET,"SURFACE MESONET - Tv, q, Pstn",Not dumped,188 +32,191,SFCBOG,SFCBOG,AUSTRALIAN PAOB MEAN SEA-LEVEL PRESSURE BOGUS OVER OCEAN - Pstn,sfcbog dump file not read by PREPOBS_PREPDATA This data no longer produced after 17 August 2010,191 +33,192,ADPSFC,Land_Surface,"SURFACE LAND SYNOPIC (fixed and mobile) WITH MISSING STATION PRESSURE AND MISSING SEA-LEVEL PRESSURE - Tv, q, Pstn",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,192 +34,193,ADPSFC,Land_Surface,"SURFACE LAND METAR WITH MISSING STATION PRESSURE, MISSING SEA-LEVEL PRESSURE AND MISSING ALTIMETER SETTING - Tv, q, Pstn",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,193 +35,194,SFCSHP,Ocean_Surface,"SURFACE MARINE (SHIP, BUOY, C-MAN, TIDE GAUGE) OR LAND (SYNOPTIC, METAR) WITH MISSING STATION PRESSURE AND MISSING SEA-LEVEL PRESSURE - Tv, q, Pstn",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,194 +36,195,MSONET,MSONET,"SURFACE MESONET WITH MISSING STATION PRESSURE AND MISSING ALTIMETER SETTING (SEA-LEVEL PRESSURE IS ALWAYS MISSING) - Tv, q, Pstn",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,195 +37,210,SYNDAT,SYNDAT,"SYNTHETIC (BOGUS) TROPICAL CYCLONE - u, v","u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI Added to PREPBUFR file by later program SYNDAT_SYNDATA.",210 +38,220,ADPUPA,Radiosonde,"RAWINSONDE - u, v (all levels), z (winds-by-height levels)","u, v used by assimilation p (vertical coordinate) calculated from z on winds-by-height levels.",220 +39,221,ADPUPA,PIBAL,"PIBAL - u,v,z","u, v used by assimilation p (vertical coordinate) calculated from z.",221 +40,222,ADPUPA,Radiosonde,"CLASS SOUNDING - u, v",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Entire report neither monitored nor assimilated - not in convinfo text file read by GBL-GSI,222 +41,223,PROFLR,Profiler,"NOAA PROFILER NETWORK (NPN) WIND PROFILER - u, v, z","u, v used by assimilation p (vertical coordinate) calculated from z. These data are no longer produced after 13 September 2017.",223 +42,224,VADWND,NEXRAD_Wind,"NEXRAD VERTICAL AZIMUTH DISPLAY (VAD) from Radar Coded Message (subtype 1) - u, v, z","u, v used by assimilation p (vertical coordinate) calculated from z.",224 +43,227,PROFLR,Profiler,"MULTI-AGENCY PROFILER (MAP) AND ACOUSTIC SOUNDER (SODAR) - u, v, z",Entire report tossed by switch in PREPOBS_PREPDATA parm cards,227 +44,228,PROFLR,Profiler,"JAPANESE METEOROLOGICAL AGENCY (JMA) WIND PROFILER - u, v, z","u, v (entire report) flagged for non-use by assimilation due to missing obs error u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI",228 +45,229,PROFLR,PIBAL,"WIND PROFILER DECODED FROM PILOT (PIBAL) BULLETINS - u, v, z","u, v used by assimilation p (vertical coordinate) calculated from z.",229 +46,230,AIRCFT,Aircraft,"AIREP AND PIREP AIRCRAFT - u, v","u, v used by assimilation",230 +47,231 R,AIRCFT,Aircraft,"AMDAR AIRCRAFT - u, v","u, v used by assimilation",231 +48,232,ADPUPA,Dropsonde,"FLIGHT-LEVEL RECONNAISSANCE AND PROFILE DROPSONDE - u, v","u, v used by assimilation",232 +49,233 R,AIRCAR,Aircraft,"MDCRS ACARS AIRCRAFT - u, v","u, v used by assimilation",233 +50,234 R,AIRCFT,Aircraft,"TAMDAR AIRCRAFT - u, v","u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI",234 +51,235 R,AIRCFT,Aircraft,"CANADIAN AMDAR AIRCRAFT - u, v","u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI",235 +52,240,SATWND,GEO_Wind,"NESDIS IR (SHORT-WAVE) CLOUD DRIFT (ALL LEVELS) (GOES) - u, v",The assimilation will obtain this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This will not be written into the PREPBUFR file.,240 +53,241,SATWND,GEO_Wind,"INDIA IR (LONG-WAVE) AND VISIBLE CLOUD DRIFT (ALL LEVELS) (INSAT, KALPANA) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",241 +54,242,SATWND,GEO_Wind,"JMA IR (LONG-WAVE) AND VISIBLE CLOUD DRIFT AT LEVELS BELOW 850 MB (GMS, MTSAT, HIMAWARI) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI. The GSI redefines report type 242 as JMA visible cloud drift at all levels.",242 +55,243,SATWND,GEO_Wind,"EUMETSAT IR (LONG-WAVE) AND VISIBLE CLOUD DRIFT AT LEVELS BELOW 850 MB (METEOSAT) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI. The GSI redefines report type 243 as EUMETSAT visible cloud drift at all levels.",243 +56,244,SATWND,AVHRR_Wind,"AVHRR/POES IR (LONG-WAVE) CLOUD DRIFT (ALL LEVELS) (NOAA, METOP) - u,v",The assimilation will obtain this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This will not be written into the PREPBUFR file.,244 +57,245,SATWND,GEO_Wind,"NESDIS IR (LONG-WAVE) CLOUD DRIFT (ALL LEVELS) (GOES) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",245 +58,246,SATWND,GEO_Wind,"NESDIS IMAGER WATER VAPOR (ALL LEVELS) - CLOUD TOP (GOES) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",246 +59,247,SATWND,GEO_Wind,"NESDIS IMAGER WATER VAPOR (ALL LEVELS) - DEEP LAYER (GOES) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file (it was never in the PREPBUFR file) (see Table 18).  Since it has both a missing obs error and is set to monitor in the convinfo file, it is now monitored by the GBL-GSI.",247 +60,248,SATWND,GEO_Wind,"NESDIS SOUNDER WATER VAPOR (ALL LEVELS) - CLOUD TOP (GOES) - u, v","If ever processed, the assimilation will obtain this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This will never be written into the PREPBUFR file.",248 +61,249,SATWND,GEO_Wind,"NESDIS SOUNDER WATER VAPOR (ALL LEVELS) - DEEP LAYER (GOES) - u, v","If ever processed, the assimilation will obtain this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This will never be written into the PREPBUFR file.",249 +62,250,SATWND,GEO_Wind,"JMA IMAGER WATER VAPOR (ALL LEVELS) - CLOUD TOP & DEEP LAYER (GMS, MTSAT, HIMAWARI) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",250 +63,251,SATWND,GEO_Wind,"NESDIS VISIBLE CLOUD DRIFT (ALL LEVELS) (GOES) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",251 +64,252,SATWND,GEO_Wind,"JMA IR (LONG-WAVE) AND VISIBLE CLOUD DRIFT AT LEVELS ABOVE 850 MB (GMS, MTSAT, HIMAWARI) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI. The GSI redefines report type 252 as JMA IR cloud drift at all levels.",252 +65,253,SATWND,GEO_Wind,"EUMETSAT IR (LONG-WAVE) AND VISIBLE CLOUD DRIFT AT LEVELS ABOVE 850 MB (METEOSAT) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI. The GSI redefines report type 253 as EUMETSAT IR cloud drift at all levels.",253 +66,254,SATWND,GEO_Wind,"EUMETSAT IMAGER WATER VAPOR (ALL LEVELS) - CLOUD TOP & DEEP LAYER (METEOSAT) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",254 +67,255,SATWND,GEO_Wind,NESDIS PICTURE TRIPLET CLOUD DRIFT (LOW LEVELS) (GOES),No longer produced by NESDIS,255 +68,256,SATWND,GEO_Wind,"INDIA IMAGER WATER VAPOR (ALL LEVELS) (INSAT, KALPANA) - u, v","If ever processed, the assimilation will obtain this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This will never be written into the PREPBUFR file.",256 +69,257,SATWND,MODIS_Wind,"MODIS/POES IR (LONG-WAVE) CLOUD DRIFT (ALL LEVELS) (AQUA, TERRA) - u,v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",257 +70,258,SATWND,MODIS_Wind,"MODIS/POES IMAGER WATER VAPOR (ALL LEVELS) - CLOUD TOP (AQUA, TERRA) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",258 +71,259,SATWND,MODIS_Wind,"MODIS/POES IMAGER WATER VAPOR (ALL LEVELS) - DEEP LAYER (AQUA, TERRA) - u, v","Effective 5/22/2012, the assimilation obtains this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This is still written into the PREPBUFR file but is ignored by the GBL-GSI.",259 +72,260,SATWND,GEO_Wind,"VIIRS/POES IR (LONG-WAVE) CLOUD DRIFT (ALL LEVELS) (NPP) - u,v",The assimilation will obtain this directly from the satwnd dump file rather than from the PREPBUFR file (see Table 18).  This will not be written into the PREPBUFR file.,260 +73,270,NA,Empty,"NACELLE - u,v",Not dumped,270 +74,271,NA,Empty,"TALL TOWER -u,v",Not dumped,271 +75,280 (R - U.S. & JMA SHIPS),SFCSHP,Ocean_Surface,"SURFACE MARINE WITH REPORTED STATION PRESSURE (SHIP, BUOY, C-MAN, TIDE GAUGE) - u, v","u, v used by assimilation",280 +76,281 (R - WMO Res 40 SYNOPS),ADPSFC,Land_Surface,"SURFACE LAND [SYNOPTIC (fixed and mobile), METAR] WITH REPORTED STATION PRESSURE - u, v","u, v (entire report) flagged for non-use by assimilation due to missing obs error u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI",281 +77,282,SFCSHP,Ocean_Surface,"ATLAS BUOY - u, v (see % below)","u, v used by assimilation Reported station pressure and mean sea-level pressure BOTH missing. Station pressure is set to 1013 mb. Elevation is less than or equal to 7.5 meters.",282 +78,283,SPSSMI,SPSSMI,"SSM/I SUPEROBED (1 DEGREE LAT/LON) NEURAL NET 3 WIND SPEED OVER OCEAN - u, v","u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI Only wspd available so direction initially set to ZERO in PREPBUFR file, direction calculated from analysis and u, v recomputed in program PREPOBS_OIQCBUFR. Reported station pressure and mean sea-level pressure BOTH missing. Station pressure is set to 1013 mb. Elevation is less than or equal to 7.5 meters. The SSM/I F-13 satellite went bad in November 2009 resulting in no data being processed upstream.  The empty dumps were turned off and the PREPOBS_PREPDATA parm cards were set to no longer process these data in October 2010.",283 +79,"284 (R - WMO Res 40 SYNOPS, U.S. & JMA SHIPS)","ADPSFC, SFCSHP",Ocean_Surface,"SURFACE MARINE (SHIP, BUOY, C-MAN, TIDE GAUGE) OR LAND [SYNOPTIC (fixed and mobile), METAR] WITH MISSING STATION PRESSURE - u, v","u, v (entire report) flagged for non-use by assimilation due to missing obs error u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI Altimeter setting is also missing (always the case for synoptic). Station pressure calculated from reported mean sea-level pressure and elevation via U.S. Standard Atmosphere approximation. Elevation is greater than 7.5 meters (if less than 7.5 meters, station pressure set equal to sea-level pressure and report type set to 281).",284 +80,285,QKSWND,QKSWND,"SUPEROBED (0.5 DEGREE LAT/LON) SCATTEROMETER WINDS OVER OCEAN (QUIKSCAT) - u, v",No longer produced,285 +81,286,ERS1DA,ERS1DA,"SCATTEROMETER WINDS OVER OCEAN (ERS) - u, v",No longer produced,286 +82,287,ADPSFC,Land_Surface,"SURFACE LAND (METAR) WITH MISSING STATION PRESSURE - u, v","u, v (entire report) flagged for non-use by assimilation due to missing obs error u, v monitored by assimilation by switch in convinfo text file read by GBL-GSI Altimeter setting is reported (never the case for synoptic). Station pressure calculated from reported altimeter setting and elevation.",287 +83,288 R,MSONET,Land_Surface,"SURFACE MESONET - u, v",Not dumped,288 +84,289,WDSATR,WDSATR,"SUPEROBED (1.0 DEGREE LAT/LON) SCATTEROMETER WINDS OVER OCEAN (WINDSAT) - u,v","u, v used by assimilation (currently not available due to format change in raw files)",289 +85,290,ASCATW,ASCAT_Wind,"NON-SUPEROBED SCATTEROMETER WINDS OVER OCEAN (ASCAT) - u,v (50 km resolution)","METOP-2(A) u, v used by assimilation METOP-1(B) u,v monitored by assimilation by switch in convinfo text file read by GBL-GSI",290 +86,291,NA,Empty,"NON-SUPEROBED SCATTEROMETER WINDS OVER OCEAN (OSCAT) - u,v","When available, the assimilation will obtain this directly from the oscatw dump file rather than from the PREPBUFR file (see Table 18).  This will not be written into the PREPBUFR file. This type will never be available. Instrument failed on 20 February 2014.",291 +87,292,ADPSFC,Land_Surface,"SURFACE LAND SYNOPIC (fixed and mobile) WITH MISSING STATION PRESSURE AND MISSING SEA-LEVEL PRESSURE - u,v",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,292 +88,293,ADPSFC,Land_Surface,"SURFACE LAND METAR WITH MISSING STATION PRESSURE, MISSING SEA-LEVEL PRESSURE AND MISSING ALTIMETER SETTING - u,v",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,293 +89,294,SFCSHP,Ocean_Surface,"SURFACE MARINE (SHIP, BUOY, C-MAN, TIDE GAUGE) OR LAND (SYNOPTIC, METAR) WITH MISSING STATION PRESSURE AND MISSING SEA-LEVEL PRESSURE - u,v",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,294 +90,295,MSONET,Land_Surface,"SURFACE MESONET WITH MISSING STATION PRESSURE AND MISSING ALTIMETER SETTING (SEA-LEVEL PRESSURE IS ALWAYS MISSING) - u,v",Entire report tossed by switch in PREPOBS_PREPDATA parm cards Station pressure estimated from U.S. Standard Atmosphere approximation Pmsl and reported temperature and elevation.,295 +91,3,Metop-B GPSRO,GPSRO,GPSRO,empty,3 +92,4,Metop-A GPSRO,GPSRO,GPSRO,empty,4 +93,5,Metop-C GPSRO,GPSRO,GPSRO,empty,5 +94,41,CHAMP GPSRO,GPSRO,GPSRO,empty,41 +95,42,TerraSAR-X GPSRO,GPSRO,GPSRO,empty,42 +96,43,TanDEM-X GPSRO,GPSRO,GPSRO,empty,43 +97,44,PAZ GPSRO,GPSRO,GPSRO,empty,44 +98,421,Oceansat-2 GPSRO,GPSRO,GPSRO,empty,421 +99,440,Megha-Tropiques GPSRO,GPSRO,GPSRO,empty,440 +100,722,GRACE A GPSRO,GPSRO,GPSRO,empty,722 +101,723,GRACE B GPSRO,GPSRO,GPSRO,empty,723 +102,724,724 GPSRO,GPSRO,GPSRO,empty,724 +103,725,725 GPSRO,GPSRO,GPSRO,empty,725 +104,726,726 GPSRO,GPSRO,GPSRO,empty,726 +105,727,727 GPSRO,GPSRO,GPSRO,empty,727 +106,728,728 GPSRO,GPSRO,GPSRO,empty,728 +107,729,729 GPSRO,GPSRO,GPSRO,empty,729 +108,740,COSMIC-1 GPSRO,GPSRO,GPSRO,empty,740 +109,741,COSMIC-2 GPSRO,GPSRO,GPSRO,empty,741 +110,742,COSMIC-3 GPSRO,GPSRO,GPSRO,empty,742 +111,743,COSMIC-4 GPSRO,GPSRO,GPSRO,empty,743 +112,744,COSMIC-5 GPSRO,GPSRO,GPSRO,empty,744 +113,745,COSMIC-6 GPSRO,GPSRO,GPSRO,empty,745 +114,750,COSMIC-2 E1 GPSRO,GPSRO,GPSRO,empty,750 +115,751,COSMIC-2 E2 GPSRO,GPSRO,GPSRO,empty,751 +116,752,COSMIC-2 E3 GPSRO,GPSRO,GPSRO,empty,752 +117,753,COSMIC-2 E4 GPSRO,GPSRO,GPSRO,empty,753 +118,754,COSMIC-2 E5 GPSRO,GPSRO,GPSRO,empty,754 +119,755,COSMIC-2 E6 GPSRO,GPSRO,GPSRO,empty,755 +120,786,C/NOFS GPSRO,GPSRO,GPSRO,empty,786 +121,820,SAC-C GPSRO,GPSRO,GPSRO,empty,820 +122,821,821 GPSRO,GPSRO,GPSRO,empty,821 +123,825,KOMPSAT-5 GPSRO,GPSRO,GPSRO,empty,825 +124,40,Oersted GPSRO,GPSRO,GPSRO,empty,40 +125,66,Sentinel-6A GPSRO,GPSRO,GPSRO,empty,66 +126,67,Sentinel-6B GPSRO,GPSRO,GPSRO,empty,67 +127,265,GeoOptics CICERO OP1 GPSRO,GPSRO,GPSRO,empty,265 +128,266,GeoOptics CICERO OP2 GPSRO,GPSRO,GPSRO,empty,266 +129,267,PlanetiQ GNOMES-A GPSRO,GPSRO,GPSRO,empty,267 +130,268,PlanetiQ GNOMES-B GPSRO,GPSRO,GPSRO,empty,268 +131,269,Spire Lemur 3U CubeSat GPSRO,GPSRO,GPSRO,empty,269 +132,800,SUNSAT GPSRO,GPSRO,GPSRO,empty,800 +133,803,GRACE C (GRACE-FO) GPSRO,GPSRO,GPSRO,empty,803 +134,804,GRACE D (GRACE-FO) GPSRO,GPSRO,GPSRO,empty,804 + + diff --git a/src/EFSOI_Utilities/scripts/copyefsoifiles.sh b/src/EFSOI_Utilities/scripts/copyefsoifiles.sh new file mode 100755 index 00000000..466a7b7c --- /dev/null +++ b/src/EFSOI_Utilities/scripts/copyefsoifiles.sh @@ -0,0 +1,125 @@ +#!/bin/sh -x +################################################################################ +#### UNIX Script Documentation Block +# . . +# Script name: copyefsoifiles.sh +# Script description: copy files required for EFSOI from COMROT +# +# Author: Andrew Eichmann Org: NCEP/EMC Date: 2021-03-05 +# +# Abstract: This script copies the files required to run EFSOI executable +# from COMROT that has generated 24/30 hour forecasts to staging +# directory for testing purposes +# +# $Id$ +# +# Attributes: +# Language: POSIX shell +# Machine: Hera +# +################################################################################ + + + +COMROT= +STGDIR= + +EDATE=2019111818 +NENS=20 +NDATE=/scratch1/NCEPDEV/global/Fanglin.Yang/save/VRFY/vsdb/nwprod/util/exec/ndate + +PDATE=`${NDATE} -6 ${EDATE}` +VDATE=`${NDATE} +24 ${EDATE}` + +EDATECYC="${EDATE:8:2}" +PDATECYC="${PDATE:8:2}" +VDATECYC="${VDATE:8:2}" + +EDATEDIR=enkfgdas."${EDATE:0:8}" +PDATEDIR=enkfgdas."${PDATE:0:8}" +VDATEDIR=enkfgdas."${VDATE:0:8}" + + +echo $PDATE +echo $VDATE +echo $EDATECYC +echo $EDATEDIR + +# set up 30 hour forecast + +cd $STGDIR + +if [ -d "$PDATEDIR" ]; then + echo "$PDATEDIR exists." +else + echo "$PDATEDIR does not exist, creating it" + mkdir $PDATEDIR +fi + +cd $PDATEDIR + +if [ -d "$PDATECYC" ]; then + echo "$PDATECYC exists." +else + echo "$PDATECYC does not exist, creating it" + mkdir $PDATECYC +fi + +cd $PDATECYC + +cp -vi $COMROT/$PDATEDIR/$PDATECYC/gdas.t${PDATECYC}z.atmf030.ensmean.* . +cp -vi $COMROT/$PDATEDIR/$PDATECYC/gdas.t${PDATECYC}z.atmf006.ensmean.* . + + +# set up 24 hour forecast + +cd $STGDIR + +if [ -d "$EDATEDIR" ]; then + echo "$EDATEDIR exists." +else + echo "$EDATEDIR does not exist, creating it" + mkdir $EDATEDIR +fi + +cd $EDATEDIR + +if [ -d "$EDATECYC" ]; then + echo "$EDATECYC exists." +else + echo "$EDATECYC does not exist, creating it" + mkdir $EDATECYC +fi + +cd $EDATECYC + +cp -vi $COMROT/$EDATEDIR/$EDATECYC/gdas.t${EDATECYC}z.atmf024.ensmean.* . +cp -vi $COMROT/$EDATEDIR/$EDATECYC/gdas.t${EDATECYC}z.atmf006.ensmean.* . +cp -vi $COMROT/$EDATEDIR/$EDATECYC/gdas.t${EDATECYC}z.abias_int.ensmean . + + + +imem=1 +while [[ $imem -le $NENS ]]; do + member="mem"`printf %03i $imem` + + if [ -d "$member" ]; then + echo "$member exists." + else + echo "$member does not exist, creating it" + mkdir $member + fi + + cd $member + + cp -vi $COMROT/$EDATEDIR/$EDATECYC/$member/gdas.t${EDATECYC}z.atmf024.nemsio . + cp -vi $COMROT/$EDATEDIR/$EDATECYC/$member/gdas.t${EDATECYC}z.atmf024s.nemsio . + + cd .. + + (( imem = $imem + 1 )) +done + + + + diff --git a/src/EFSOI_Utilities/scripts/geoplot.py b/src/EFSOI_Utilities/scripts/geoplot.py new file mode 100644 index 00000000..54ddca36 --- /dev/null +++ b/src/EFSOI_Utilities/scripts/geoplot.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Thu Jul 7 14:23:41 2022 + +@author: mossland +""" +# Default imports +import numpy as np +import matplotlib.pyplot as plt +import cartopy.crs as ccrs +import pandas as pd +import pickle +import datetime +import os.path +import matplotlib.colors as colors +import matplotlib.cm as cmx + +CDATE = '2021121000' +exp='rodeo' +exp='baily' + +#datadir='/work2/noaa/stmp/aeichman/comrot/' + exp + '/osense/' +#datadir='/work/noaa/stmp/aeichman/comrot/' + exp + '/osense/' +rootdir = '/Users/mossland/Desktop/work/EFSOI/' + exp +indir = rootdir + '/consolidated/' + +filename = indir + 'osense_' + CDATE + '.pkl' +infile = open( filename ,'rb') +[ idata, osensedata ] = pickle.load(infile) +infile.close() + + +# drop message='NA'/message02='Empty' +# data not dumped/not used +indices=osensedata[osensedata['assimilated']==0].index +osensedata.drop(indices,inplace = True) + +source = 'Radiosonde' +source = 'AMSUA' +source = 'Ocean_Surface' +source = 'Aircraft' + +for source in osensedata.source.unique(): +#for source in ['Radiosonde','AMSUA','Ocean_Surface','Aircraft']: + + + source = str( source ) + print('doing ' + source ) + + lon = osensedata.loc[osensedata['source'] == source ]['lon'] + + lon = osensedata.loc[osensedata['source'] == source ]['lon'] + lat = osensedata.loc[osensedata['source'] == source ]['lat'] + vals = osensedata.loc[osensedata['source'] == source ]['osense_moist'] + +# vmin = vals.min() +# vmax = vals.max() +# maxval = max(abs(vmin),vmax) +# vmin=-maxval/10 +# vmax=maxval/10 + valmean = vals.mean() + valsstd = vals.std() +# vmin = valmean - valsstd +# vmax = valmean + valsstd + vmin = -valsstd + vmax = valsstd + + + print('vmin, vmax: ' , vmin,vmax) + rnbw = cm = plt.get_cmap('rainbow') + rnbw = cm = plt.get_cmap('RdBu') + rnbw = cm = plt.get_cmap('coolwarm') + cNorm = colors.Normalize(vmin=vmin, vmax=vmax) + scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=rnbw) + scalarMap.set_array(vals) + + colorVal = scalarMap.to_rgba(vals) + + title = source + ' ' + CDATE +', assimilated' + + + fig = plt.figure(figsize=(12,9)) + ax = fig.add_subplot(1, 1, 1, + projection=ccrs.PlateCarree(), + title=title) +# ax.set_extent([0, 360, -90, 90]) + ax.set_extent([-180, 180, -90, 90 ]) + ax.scatter(lon, lat,c=colorVal, marker='.',s=10) + #ax.set_title(title) + ax.coastlines() + ax.gridlines() + + + diff --git a/src/EFSOI_Utilities/scripts/getefsiofiles.sh b/src/EFSOI_Utilities/scripts/getefsiofiles.sh new file mode 100755 index 00000000..3c41b772 --- /dev/null +++ b/src/EFSOI_Utilities/scripts/getefsiofiles.sh @@ -0,0 +1,77 @@ +#!/bin/sh -x + + +# Author: Andrew Eichmann +# Purpose: to obtain from the HPSS archive the minimal set of files needed to +# run the gdaseupd task of GFS workflow, intended for EFSOI processing +# Script assumes that these files have been archived during a run; this will +# probably require modifications to the archiving script in workflow. + +# This will have to be changed to handle nc4 files + +EXP=nov2019c +#EDATE=2019111800 +#EDATE=2019111818 +EDATE=2019111712 +NMEM_ENKF=20 # total number of ensemble members +#NMEM_ENKF=0 +NMEM_EARCGRP=10 # number of ensemble members in archiving group +NDATE=/scratch1/NCEPDEV/global/Fanglin.Yang/save/VRFY/vsdb/nwprod/util/exec/ndate + +PDATE=`${NDATE} -6 ${EDATE}` +VDATE=`${NDATE} +24 ${EDATE}` + +EDATECYC="${EDATE:8:2}" +PDATECYC="${PDATE:8:2}" +VDATECYC="${VDATE:8:2}" + +EDATEDIR=enkfgdas."${EDATE:0:8}" +PDATEDIR=enkfgdas."${PDATE:0:8}" +VDATEDIR=enkfgdas."${VDATE:0:8}" + + +echo $PDATE +echo $VDATE +echo $EDATECYC +echo $EDATEDIR + +tarfile=enkfgdas.tar + + +hpssdir=/1year/NCEPDEV/emc-global/Andrew.Eichmann/HERA/scratch/${EXP}/${EDATE} + +hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/gdas.t${EDATECYC}z.cnvstat.ensmean +hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/gdas.t${EDATECYC}z.oznstat.ensmean +hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/gdas.t${EDATECYC}z.radstat.ensmean +hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/gdas.t${EDATECYC}z.abias_int.ensmean + +hpssdir=/1year/NCEPDEV/emc-global/Andrew.Eichmann/HERA/scratch/${EXP}/${PDATE} + +hpsstar get ${hpssdir}/${tarfile} ./${PDATEDIR}/${PDATECYC}/gdas.t${PDATECYC}z.atmf006.ensmean.nemsio + + + +imem=1 +while [[ $imem -le $NMEM_ENKF ]]; do + grpnum=`echo "( ($imem - 1) / ${NMEM_EARCGRP} ) + 1" | bc` + group="grp"`printf %02i $grpnum` + member="mem"`printf %03i $imem` + + echo $member + echo $group + + tarfile=enkfgdas_${group}.tar + hpssdir=/1year/NCEPDEV/emc-global/Andrew.Eichmann/HERA/scratch/${EXP}/${EDATE} + hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/${member}/gdas.t${EDATECYC}z.cnvstat + hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/${member}/gdas.t${EDATECYC}z.oznstat + hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/${member}/gdas.t${EDATECYC}z.radstat + hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/${member}/gdas.t${EDATECYC}z.atmanl.nemsio + + hpssdir=/1year/NCEPDEV/emc-global/Andrew.Eichmann/HERA/scratch/${EXP}/${PDATE} + hpsstar get ${hpssdir}/${tarfile} ./${PDATEDIR}/${PDATECYC}/${member}/gdas.t${PDATECYC}z.atmf006s.nemsio + + (( imem = $imem + 1 )) +done + + + diff --git a/src/EFSOI_Utilities/scripts/osense.py b/src/EFSOI_Utilities/scripts/osense.py index e4bad502..f56b13e0 100644 --- a/src/EFSOI_Utilities/scripts/osense.py +++ b/src/EFSOI_Utilities/scripts/osense.py @@ -2,135 +2,137 @@ from sys import exit, argv import pandas as pd -def read_osense( filename): - -#! Structure for observation sensitivity information output -#type obsense_header +# +# Structure for observation sensitivity information output +# type obsense_header # sequence -# integer(i_kind) :: idate ! Base date (initial date) -# integer(i_kind) :: obsnum ! Observation number (total) -# integer(i_kind) :: convnum ! Observation number (conventional) -# integer(i_kind) :: oznum ! Observation number (ozone) -# integer(i_kind) :: satnum ! Observation number (satellite) -# integer(i_kind) :: npred ! Number of predictors for bias correction -# integer(i_kind) :: nanals ! Number of members -#end type obsense_header +# integer(i_kind) :: idate ! Base date (initial date) +# integer(i_kind) :: obsnum ! Observation number (total) +# integer(i_kind) :: convnum ! Observation number (conventional) +# integer(i_kind) :: oznum ! Observation number (ozone) +# integer(i_kind) :: satnum ! Observation number (satellite) +# integer(i_kind) :: npred ! Number of predictors for bias correction +# integer(i_kind) :: nanals ! Number of members +# end type obsense_header # # where i_kind - generic specification kind for default integer # is a long integer # -#! Type definition for observation sensitivity information file -#type obsense_info +# Type definition for observation sensitivity information file +# type obsense_info # sequence -# real(r_single) :: obfit_prior ! Observation fit to the first guess -# real(r_single) :: obsprd_prior ! Spread of observation prior -# real(r_single) :: ensmean_obnobc ! Ensemble mean first guess (no bias correction) -# real(r_single) :: ensmean_ob ! Ensemble mean first guess (bias corrected) -# real(r_single) :: ob ! Observation value -# real(r_single) :: oberrvar ! Observation error variance -# real(r_single) :: lon ! Longitude -# real(r_single) :: lat ! Latitude -# real(r_single) :: pres ! Pressure -# real(r_single) :: time ! Observation time -# real(r_single) :: oberrvar_orig ! Original error variance -# integer(i_kind) :: stattype ! Observation type -# character(len=20) :: obtype ! Observation element / Satellite name -# integer(i_kind) :: indxsat ! Satellite index (channel) set to zero -# real(r_single) :: osense_kin ! Observation sensitivity (kinetic energy) [J/kg] -# real(r_single) :: osense_dry ! Observation sensitivity (Dry total energy) [J/kg] -# real(r_single) :: osense_moist ! Observation sensitivity (Moist total energy) [J/kg] -#end type obsense_info -# - - - -#if len(sys.argv) > 0: -#if len(argv) > 0: -# file = argv[1] - - - datacolumns = [ 'obfit_prior', - 'obsprd_prior', - 'ensmean_obnobc', - 'ensmean_ob', - 'ob', - 'oberrvar', - 'lon', - 'lat', - 'pres', - 'time', - 'oberrvar_orig', - 'stattype', - 'obtype', - 'indxsat', - 'osense_kin', - 'osense_dry', - 'osense_moist' ] - - - headerf=' 0: +#if len(argv) > 0: +# file = argv[1] + + + datacolumns = [ 'obfit_prior', + 'obsprd_prior', + 'ensmean_obnobc', + 'ensmean_ob', + 'ob', + 'oberrvar', + 'lon', + 'lat', + 'pres', + 'time', + 'oberrvar_orig', + 'stattype', + 'obtype', + 'indxsat', + 'osense_kin', + 'osense_dry', + 'osense_moist' ] + + + headerf=' Date: Mon, 25 Jul 2022 09:06:18 -0500 Subject: [PATCH 02/13] adding fix file with efsoi entry --- src/EFSOI_Utilities/fix/README_EFSOI_fix | 16 +++ .../fix/global_anavinfo.l127.txt | 116 ++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 src/EFSOI_Utilities/fix/README_EFSOI_fix create mode 100644 src/EFSOI_Utilities/fix/global_anavinfo.l127.txt diff --git a/src/EFSOI_Utilities/fix/README_EFSOI_fix b/src/EFSOI_Utilities/fix/README_EFSOI_fix new file mode 100644 index 00000000..6d8e68e7 --- /dev/null +++ b/src/EFSOI_Utilities/fix/README_EFSOI_fix @@ -0,0 +1,16 @@ + +directory: + +GSI/util/EFSOI_Utilities/fix + +file: +global_anavinfo_efsoi.l127.txt + +This is the EFSOI version of the GSI fix file global_anavinfo.l127.txt, +modified with an entry for EFSOI. This is located here until it can be +added to the GSI/fix directory in the code repository. It may have to be +updated if the default file is modified. The variable ANAVINFO in the +global-workflow config file config.efsoi should point to this file. + + + diff --git a/src/EFSOI_Utilities/fix/global_anavinfo.l127.txt b/src/EFSOI_Utilities/fix/global_anavinfo.l127.txt new file mode 100644 index 00000000..6363e450 --- /dev/null +++ b/src/EFSOI_Utilities/fix/global_anavinfo.l127.txt @@ -0,0 +1,116 @@ +correlated_observations:: +! isis method kreq kmult type cov_file +iasi_metop-c 2 0.0 1.0 sea Rcov_iasicsea +iasi_metop-b 2 0.0 1.0 sea Rcov_iasibsea +iasi_metop-c 2 0.0 1.0 land Rcov_iasicland +iasi_metop-b 2 0.0 1.0 land Rcov_iasibland +cris-fsr_n20 2 0.0 1.0 sea Rcov_crisn20 +cris-fsr_npp 2 0.0 1.0 sea Rcov_crisnpp +:: + +met_guess:: +!var level crtm_use desc orig_name + ps 1 -1 surface_pressure ps + z 1 -1 geopotential_height phis + u 127 2 zonal_wind u + v 127 2 meridional_wind v + div 127 -1 zonal_wind div + vor 127 -1 meridional_wind vor + tv 127 2 virtual_temperature tv + q 127 2 specific_humidity sphu + oz 127 2 ozone ozone + cw 127 10 cloud_condensate cw + ql 127 12 cloud_liquid ql + qi 127 12 cloud_ice qi +:: + +state_derivatives:: +!var level src + ps 1 met_guess + u 127 met_guess + v 127 met_guess + tv 127 met_guess + q 127 met_guess + oz 127 met_guess + cw 127 met_guess + prse 128 met_guess +:: + +state_tendencies:: +!var levels source + u 127 met_guess + v 127 met_guess + tv 127 met_guess + q 127 met_guess + cw 127 met_guess + oz 127 met_guess + prse 128 met_guess +:: + +state_vector:: +!var level itracer source funcof + u 127 0 met_guess u + v 127 0 met_guess v + tv 127 0 met_guess tv + tsen 127 0 met_guess tv,q + q 127 1 met_guess q + oz 127 1 met_guess oz + ql 127 1 met_guess ql + qi 127 1 met_guess qi + prse 128 0 met_guess prse + ps 1 0 met_guess prse + sst 1 0 met_guess sst +:: + + +state_vector_efsoi:: +!var level itracer source funcof + u 127 0 met_guess u + v 127 0 met_guess v + tv 127 0 met_guess tv + q 127 1 met_guess q + ps 1 0 met_guess prse +:: + + + +control_vector_enkf:: +!var level itracer as/tsfc_sdv an_amp0 source funcof + u 127 0 1.00 -1.0 state u,v + v 127 0 1.00 -1.0 state u,v + ps 1 0 1.20 -1.0 state prse +!pst 1 0 1.20 -1.0 state prse,u,v + tv 127 0 1.50 -1.0 state tv + q 127 1 1.50 -1.0 state q + oz 127 1 2.00 -1.0 state oz +!sst 1 0 1.00 -1.0 state sst +!cw 127 1 1.00 -1.0 state cw +!stl 1 0 3.00 -1.0 motley sst +!sti 1 0 3.00 -1.0 motley sst +:: + +control_vector:: +!var level itracer as/tsfc_sdv an_amp0 source funcof + sf 127 0 1.00 -1.0 state u,v + vp 127 0 1.00 -1.0 state u,v + ps 1 0 1.20 -1.0 state prse + t 127 0 1.50 -1.0 state tv + q 127 1 1.50 -1.0 state q + oz 127 1 2.00 -1.0 state oz + sst 1 0 1.00 -1.0 state sst + cw 127 1 1.00 -1.0 state cw + stl 1 0 3.00 -1.0 motley sst + sti 1 0 3.00 -1.0 motley sst +:: + +! Following table shows the use of all four prescribed trace gas data. +! To turn off any one of any combination of trace gas input, add "!" +! in the first column of that trace gas name. To use all default +! trace gas profiles, just delete the following seven lines. +chem_guess:: +!var level itracer crtm_use type orig_name +!ch4 64 1 2 n/a ch4 + co2 127 1 0 n/a co2 +!co 64 1 2 n/a co +!n2o 64 1 2 n/a n2o +:: From 43a84a8f47feb3e8e3c6dbc506005557ae2832b8 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 9 Aug 2022 16:21:00 -0400 Subject: [PATCH 03/13] added comments to osense reader, removed obsolete README --- src/EFSOI_Utilities/scripts/README_EFSOI | 122 ----------------------- src/EFSOI_Utilities/scripts/osense.py | 41 +++++++- 2 files changed, 36 insertions(+), 127 deletions(-) delete mode 100644 src/EFSOI_Utilities/scripts/README_EFSOI diff --git a/src/EFSOI_Utilities/scripts/README_EFSOI b/src/EFSOI_Utilities/scripts/README_EFSOI deleted file mode 100644 index 32f56104..00000000 --- a/src/EFSOI_Utilities/scripts/README_EFSOI +++ /dev/null @@ -1,122 +0,0 @@ -############################################################ -README file instructions for EFSOI calculations on the Theia -machines (04/12/2017). -############################################################ - ---------------------------------------------------------- -#################################### -# Overview of EFSOI functionality # -#################################### -The inputs required for EFSOI calculations are prepared -via the scripts in this directory (i.e util/EFSOI_utilities). -Note that these scripts in turn use ancillary scripts -to perform the ensemble of forecasts needed for EFSOI. -Special instructions to setup the scripts from the -sandbox project are provided by instruction -(2) below. ---------------------------------------------------------- - ----------------------------------------------------------- -################################################## -# Instructions for performing EFSOI calculations # -################################################## -1) Run a 3DEnVar, 4DEnVar or pure EnKF GFS experiment. When doing this - make sure to set fso_cycling to .true. in the exglobal_enkf_update.sh.ecf - script, and set $ARCHIVE_ENSEMBLE to YES in your para config file to output - EnKF products (FG files) necessary for the EFSOI calculations. Modify your - rlist file to add archival of files with the naming convention sanl_${CDATE}_nimemxxx - (where xxx refers to the ensemble member number) and osense_${CDATE}.dat. - The additional sanl* files correspond to the direct result of the EnKF update, - and represent a non-inflated set of EnKF analyses. If you do not see - archival of both the osense_${CDATE}.dat and sanl_${CDATE}_nimemxxx - files, something has gone wrong, in this case please contact David Groff - (david.groff@noaa.gov) as necessary. -2) Instructions relating to sandbox scripts that - perform the necessary forecasts: - # --------------------------------------------------------------------# - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!# - # Usage of scripts borrowed from Rahul Mahajan's version controlled # - # sandbox scripts # - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!# - - Create the following directory from your home directory. - $HOME/svn-work/GDAS - - cd to the newly created $HOME/svn-work/GDAS directory. - - Perform a checkout of the relevant scripts as follows: - svn checkout -r60977 https://svnemc.ncep.noaa.gov:/projects/sandbox/Rahul.Mahajan/trunk/GDAS/scripts - - Modify param_EFSO.j and advance_ensemble.j as necessary. - To see an example for the changes needed to make the scripts functional, - compare param_EFSO.j and advance_ensemble.j from - /scratch4/NCEPDEV/da/save/David.Groff/EFSOI/shared_scripts - with param_template.j and advance_ensemble.j in your checkout - of the relevant scripts. - ---List of changes for param_template.j --- - i) Comment out start_date and end_date - ii) Set PSLOT and ROTDIR - iii) Set FHOUT to 6 (i.e 6 hour output forecast intervals) - iv) Set FHMAX to 30 (i.e each cycle we need 30 hour forecasts) - v) Set FHCYC to 30. - vi) Set FHDFI and FHDFI_ENKF to 0. - vii) Turn off stochastic physics, set sppt, shum, skeb and vcamp to zero. - viii) Set BASEGFS to the directory relevant to the global_fcst version of your choice. - ix) Set HOMEDIR, PTMP to directories with adequate disk space (etc.). - ---List of changes for advance_ensemble.j --- - i) All instances of setting SUFOUT need to be with the following conventions: - setenv SUFOUT _nimem. - ii) Add the following to the top of the script. - set end_date = $2 - set begin_date = $2 - - Copy over job_monitor.py and advance_time from Dave Groff's - $HOME/bin to your $HOME/bin. - # --------------------------------------------------------------------# -3) Go to /scripts/EnKF/scripts_ncep from the root - of your working copy branch and perform the - following modifications to scripts in this - directory: - - efso_file_assembly.sh: - i. Set the SCRIPTS_DIR directory - to the /util/EFSO_utilities/scripts directory from - the root of your working copy branch. - ii. Set WDIR to a directory you have permissions to, - that provides adequate disk space (etc.) for - the required inputs for the EFSOI calculation - (eg. stmp4 on Theia). - iii. Under the documentation title "Parallel experiment context" - modify $HSI_DIR and $HFILE1 throug $HFILE7 to refer to - the directories where you are archiving EnKF products - from your parallel experiments. Note that the appropriate - dates get handled by the wrapper script, so just modify - the bits that do not relate to the date. - - efso_advance_ensemble.sh: - i. Change $WDIR to be consistent with $WDIR in your efso_file_assembly.sh (eg. stmp4 (etc.)). - ii. Change $ADVANCE_CURRENT_RUNDIR and $ADVANCE_PREVIOUS_RUNDIR to be - consistent with that specified in the script changes you performed to - the sandbox scripts in step (2). When doing this do not modify - the $PYMDH and $YMDH directory extension (the dates) that are currently - in the version you checked out (i.e the last directory in the path already - specified in the script you checked out). - iii. Set $ADVANCE_CURRENT_ROTDIR and $ADVANCE_PREVIOUS_ROTDIR to the directory - specified as ROTDIR in the param_EFSOI.j script you modified from step (2). - Again leave (do not delete) the $YMDH and $PYMDH already in the version - you checked out. - iv. You can generate your own version of getsigensmeanp_smooth.x, or for now - simply leave $EXEC_ENKF_STATS as it is versioned now (no changes necessary). - v. Modify the paths specified by the *ENSMEAN* variables to be consistent with - your $WDIR directory as you specified in efso_file_assembly.sh. - vi. Modify the paths specified by the *MEMBER* variables to be consistent with - your *ROTDIR* locations. - - efso_run.sh: - i. Modify $WDIR to be consistent with what you specified in efso_file_assembly.sh. - ii. Set $FIXDIR to the fix directory of your branch. - iii. Set $EFSOI_OUTPUT_DIR to a directory where you would like to accumulate EFSOI stats. - iv. Set $SCRIPTS_DIR to the /util/EFSOI_utilities/scripts from the root - of your branch. - v. Modify your specified account (eg. da-cpu) for submission as necessary. - -4) Execute the efso wrapper script from the command line to - perform EFSOI calculations for a period of time: -./efso_wrapper.sh Begin_Date Experiment_Name End_Date -(example) ./efso_wrapper.sh 2015120100 prfsoidjf 2015121000 - diff --git a/src/EFSOI_Utilities/scripts/osense.py b/src/EFSOI_Utilities/scripts/osense.py index f56b13e0..0fbea4ce 100644 --- a/src/EFSOI_Utilities/scripts/osense.py +++ b/src/EFSOI_Utilities/scripts/osense.py @@ -1,7 +1,29 @@ import struct -from sys import exit, argv import pandas as pd + +# osense.py + +# Written by Andrew Eichmann, NOAA/NWS/NCEP/EMC + +# ============================================================================= +# function: read_osense(filename) +# +# Reads osense file generated by enkf and efsoi executables in EFSOI +# implementation in NOAA-EMC/GSI and NOAA-EMC/GSI-Utils, and returns a pandas +# DataFrame with conventional and ozone data, a second DataFrame with satellite +# data, and the date and cycle read from the file. The conventional/ozone data +# and satellite data have different record formats and slightly different +# meanings to the columns, which get hashed out in later scripts. +# The osense file format expected by this fuction is determined by +# enkf_obs_sensitivity module in GSI/src/enkf. The relevant variable +# declarations in enkf_obs_sensitivity are reproduced below. Two variables +# from enkf that are in the osense file but ignored here are analobs and +# biaspreds - analobs is used by the +# ============================================================================= + + + # # Structure for observation sensitivity information output # type obsense_header @@ -51,6 +73,7 @@ def read_osense(filename): + # these names are the same as in enkf_obs_sensitivity, but needn't be datacolumns = ['obfit_prior', 'obsprd_prior', 'ensmean_obnobc', @@ -70,6 +93,7 @@ def read_osense(filename): 'osense_dry', 'osense_moist'] +# these determine the size of the records read and depend on the formatting headerf = ' Date: Tue, 9 Aug 2022 16:38:53 -0400 Subject: [PATCH 04/13] pep-8 check for osense.py, removed obsolete files --- src/EFSOI_Utilities/scripts/osense.py | 20 ++- src/EFSOI_Utilities/scripts/osense_old.py | 136 --------------------- src/EFSOI_Utilities/scripts/read_osense.py | 133 -------------------- 3 files changed, 8 insertions(+), 281 deletions(-) delete mode 100644 src/EFSOI_Utilities/scripts/osense_old.py delete mode 100644 src/EFSOI_Utilities/scripts/read_osense.py diff --git a/src/EFSOI_Utilities/scripts/osense.py b/src/EFSOI_Utilities/scripts/osense.py index 0fbea4ce..67a36e83 100644 --- a/src/EFSOI_Utilities/scripts/osense.py +++ b/src/EFSOI_Utilities/scripts/osense.py @@ -9,22 +9,19 @@ # ============================================================================= # function: read_osense(filename) # -# Reads osense file generated by enkf and efsoi executables in EFSOI +# Reads osense file generated by enkf and efsoi executables in EFSOI # implementation in NOAA-EMC/GSI and NOAA-EMC/GSI-Utils, and returns a pandas # DataFrame with conventional and ozone data, a second DataFrame with satellite # data, and the date and cycle read from the file. The conventional/ozone data # and satellite data have different record formats and slightly different -# meanings to the columns, which get hashed out in later scripts. -# The osense file format expected by this fuction is determined by -# enkf_obs_sensitivity module in GSI/src/enkf. The relevant variable +# meanings to the columns, which get hashed out in later scripts. +# The osense file format expected by this fuction is determined by +# enkf_obs_sensitivity module in GSI/src/enkf. The relevant variable # declarations in enkf_obs_sensitivity are reproduced below. Two variables -# from enkf that are in the osense file but ignored here are analobs and -# biaspreds - analobs is used by the +# from enkf that are in the osense file but ignored here are analobs and +# biaspreds - analobs is used by the # ============================================================================= - - -# # Structure for observation sensitivity information output # type obsense_header # sequence @@ -107,7 +104,7 @@ def read_osense(filename): with open(filename, 'rb') as fin: - # first read the header for info about the records + # first read the header for info about the records # this seems to be header info, read and ignored f = fin.read(4) @@ -123,7 +120,7 @@ def read_osense(filename): convnum + oznum, ', satnum = ', satnum, ' npred = ', npred, ', nanals = ', nanals) - # now read the conventional data records + # now read the conventional data records print('reading conventional data...') for i in range(0, convnum + oznum): @@ -166,4 +163,3 @@ def read_osense(filename): print(satdata.describe()) return((convdata, satdata, idate)) - diff --git a/src/EFSOI_Utilities/scripts/osense_old.py b/src/EFSOI_Utilities/scripts/osense_old.py deleted file mode 100644 index b5150c53..00000000 --- a/src/EFSOI_Utilities/scripts/osense_old.py +++ /dev/null @@ -1,136 +0,0 @@ -import struct -from sys import exit, argv -import pandas as pd - -def read_osense( filename): - -#! Structure for observation sensitivity information output -#type obsense_header -# sequence -# integer(i_kind) :: idate ! Base date (initial date) -# integer(i_kind) :: obsnum ! Observation number (total) -# integer(i_kind) :: convnum ! Observation number (conventional) -# integer(i_kind) :: oznum ! Observation number (ozone) -# integer(i_kind) :: satnum ! Observation number (satellite) -# integer(i_kind) :: npred ! Number of predictors for bias correction -# integer(i_kind) :: nanals ! Number of members -#end type obsense_header -# -# where i_kind - generic specification kind for default integer -# is a long integer - - -# -#! Type definition for observation sensitivity information file -#type obsense_info -# sequence -# real(r_single) :: obfit_prior ! Observation fit to the first guess -# real(r_single) :: obsprd_prior ! Spread of observation prior -# real(r_single) :: ensmean_obnobc ! Ensemble mean first guess (no bias correction) -# real(r_single) :: ensmean_ob ! Ensemble mean first guess (bias corrected) -# real(r_single) :: ob ! Observation value -# real(r_single) :: oberrvar ! Observation error variance -# real(r_single) :: lon ! Longitude -# real(r_single) :: lat ! Latitude -# real(r_single) :: pres ! Pressure -# real(r_single) :: time ! Observation time -# real(r_single) :: oberrvar_orig ! Original error variance -# integer(i_kind) :: stattype ! Observation type -# character(len=20) :: obtype ! Observation element / Satellite name -# integer(i_kind) :: indxsat ! Satellite index (channel) set to zero -# real(r_single) :: osense_kin ! Observation sensitivity (kinetic energy) [J/kg] -# real(r_single) :: osense_dry ! Observation sensitivity (Dry total energy) [J/kg] -# real(r_single) :: osense_moist ! Observation sensitivity (Moist total energy) [J/kg] -#end type obsense_info -# - - - -#if len(sys.argv) > 0: -#if len(argv) > 0: -# file = argv[1] - - - datacolumns = [ 'obfit_prior', - 'obsprd_prior', - 'ensmean_obnobc', - 'ensmean_ob', - 'ob', - 'oberrvar', - 'lon', - 'lat', - 'pres', - 'time', - 'oberrvar_orig', - 'stattype', - 'obtype', - 'indxsat', - 'osense_kin', - 'osense_dry', - 'osense_moist' ] - - - headerf=' 0: -if len(argv) > 0: - file = argv[1] - - -datacolumns = [ 'obfit_prior', - 'obsprd_prior', - 'ensmean_obnobc', - 'ensmean_ob', - 'ob', - 'oberrvar', - 'lon', - 'lat', - 'pres', - 'time', - 'oberrvar_orig', - 'stattype', - 'obtype', - 'indxsat', - 'osense_kin', - 'osense_dry', - 'osense_moist' ] - - - -headerf=' 3176 ): -# print('indxsat=',indxsat,' ',idate) - - print('read satellite data...') - -convdata = pd.DataFrame( convdatatmp, columns = datacolumns ) - -satdata = pd.DataFrame( satdatatmp, columns = datacolumns ) - -print(convdata.head()) -print(convdata.tail()) -print(satdata.head()) -print(satdata.tail()) -#print(lon, lat, pres, time,obtype) - From d0fc1ea59a636cc739eb2441a9d4c6f563ec2852 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Wed, 10 Aug 2022 16:36:29 -0400 Subject: [PATCH 05/13] improvements to osense library, added stripped down osense distiller --- src/EFSOI_Utilities/scripts/README_fv3gfs | 3 - src/EFSOI_Utilities/scripts/osense.py | 78 +++++++++++++++++++- src/EFSOI_Utilities/scripts/reduce_osense.py | 78 ++++++++++++++++++++ 3 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 src/EFSOI_Utilities/scripts/reduce_osense.py diff --git a/src/EFSOI_Utilities/scripts/README_fv3gfs b/src/EFSOI_Utilities/scripts/README_fv3gfs index a7e83112..b7643540 100644 --- a/src/EFSOI_Utilities/scripts/README_fv3gfs +++ b/src/EFSOI_Utilities/scripts/README_fv3gfs @@ -15,9 +15,6 @@ from osense file generated by EFSOI executable osense.py: contains routine to read osense file -osense_old.py: contains routine to read older osense file without assimilated flag --- try this if osense.py throws exception on reading - convdata_codes.csv: codes needed to organize convetional data sources, derived from webpage (with the help of tools in pandas): https://www.emc.ncep.noaa.gov/mmb/data_processing/prepbufr.doc/table_2.htm diff --git a/src/EFSOI_Utilities/scripts/osense.py b/src/EFSOI_Utilities/scripts/osense.py index 67a36e83..74d98d4f 100644 --- a/src/EFSOI_Utilities/scripts/osense.py +++ b/src/EFSOI_Utilities/scripts/osense.py @@ -19,7 +19,7 @@ # enkf_obs_sensitivity module in GSI/src/enkf. The relevant variable # declarations in enkf_obs_sensitivity are reproduced below. Two variables # from enkf that are in the osense file but ignored here are analobs and -# biaspreds - analobs is used by the +# biaspreds - analobs is used by the efsoi algorithm # ============================================================================= # Structure for observation sensitivity information output @@ -70,7 +70,8 @@ def read_osense(filename): - # these names are the same as in enkf_obs_sensitivity, but needn't be + # these names are the same as in enkf_obs_sensitivity, but needn't be, + # though later processing assumes them datacolumns = ['obfit_prior', 'obsprd_prior', 'ensmean_obnobc', @@ -118,7 +119,9 @@ def read_osense(filename): print('read header: idate = ', idate, ', convnum + oznum = ', convnum + oznum, ', satnum = ', satnum, ' npred = ', npred, - ', nanals = ', nanals) + ', nanals = ', nanals, ' obsnum = ', obsnum, + ', should equal convnum + oznum + satnum: ', + convnum + oznum + satnum) # now read the conventional data records @@ -163,3 +166,72 @@ def read_osense(filename): print(satdata.describe()) return((convdata, satdata, idate)) + + +def consolidate(convdata, satdata, sensors='Default', osensefields='Default'): + # takes convdata and satdata osense DataFrames and joins them in a single + # internally consistant DataFrame + + # these are satellite sensors to consolidate across plaforms + if sensors == 'Default': + sensors = ['airs', + 'amsr', + 'amsua', + 'atms', + 'avhrr', + 'cris', + 'iasi', + 'mhs', + 'saphir', + 'seviri', + 'ssmis'] + + # these are fields in the osense files and can be expanded or reduced + if osensefields == 'Default': + osensefields = ['source', + 'detailed_source', + 'indxsat', + 'osense_kin', + 'osense_dry', + 'osense_moist', + 'assimilated', + 'lat', + 'lon', + 'pres'] + + # the following is to provide "sensor-platform" in the detailed_source + # column and "sensor" in the source column + satdata['detailed_source'] = satdata['obtype'] + satdata['source'] = satdata['obtype'] + + for sensor in sensors: + mask = satdata.source.str.contains(sensor) + satdata.loc[mask, 'source'] = sensor.upper() + + # csv file should be in repo with osense.py; there should be a more + # generalizable way to tell it where + convcodes = pd.read_csv('convdata_codes.csv') + + # associate each data point with its source, by code + # this also effectively adds the columns 'source' and 'detailed_source' + convbycodes = pd.merge(convdata, convcodes, how='left', + left_on='stattype', + right_on='code') + + # drop message='NA'/message02='Empty' + # data not dumped/not used + indices = convbycodes[convbycodes['source'] == 'Empty'].index + convbycodes.drop(indices, inplace=True) + + # some stattypes, namely 700, have no corresponding code, and so have + # nans in the code, source, and detailed_source fields. This replaces those + # fields with the stattype + nanmask = convbycodes.code.isna() + nanstattypes = convbycodes.loc[nanmask, ['stattype']] + for nanstattype in nanstattypes.stattype.unique(): + convbycodes.loc[convbycodes['stattype'] == nanstattype, + ['source', 'detailed_source']] = nanstattype + + osensedata = pd.concat([satdata[osensefields], + convbycodes[osensefields]]) + return(osensedata) diff --git a/src/EFSOI_Utilities/scripts/reduce_osense.py b/src/EFSOI_Utilities/scripts/reduce_osense.py new file mode 100644 index 00000000..1ca05bcf --- /dev/null +++ b/src/EFSOI_Utilities/scripts/reduce_osense.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +from pickle import dump +from datetime import datetime +import os.path +import argparse +import sys +import pandas as pd +import osense + +parser = argparse.ArgumentParser() +parser.add_argument( + 'firstcycle', + help='first cycle to process, format YYYY-MM-DD-HH', + type=lambda s: datetime.strptime(s, '%Y-%m-%d-%H')) +parser.add_argument( + 'lastcycle', + help='last cycle to process, format YYYY-MM-DD-HH', + type=lambda s: datetime.strptime(s, '%Y-%m-%d-%H')) +parser.add_argument('indir', help='directory to read in osense files') +parser.add_argument('outdir', help='directory to save output') +args = parser.parse_args() + +firstcycle = args.firstcycle +lastcycle = args.lastcycle +indir = args.indir +outdir = args.outdir + +cycletimes = ['00', '06', '12', '18'] + +if lastcycle < firstcycle: + print('lastcycle', lastcycle, 'comes before firstcycle', + firstcycle, ', you probably don\'t want that') + sys.exit(1) + +if firstcycle.strftime("%H") not in cycletimes: + print('firstcycle', firstcycle, + 'needs to have an hour that is one of', cycletimes) + sys.exit(1) + +if lastcycle.strftime("%H") not in cycletimes: + print('lastcycle', lastcycle, 'needs to have an hour that is one of', cycletimes) + sys.exit(1) + +if not os.path.isdir(indir): + print('indir', indir, 'is not a directory') + sys.exit(1) + +if not os.path.isdir(outdir): + print('outdir', outdir, 'is not a directory') + sys.exit(1) + + +print('running from', firstcycle, 'to', lastcycle) + +# time between cycles +cycles = pd.date_range(firstcycle, lastcycle, freq='6H') + + +for cycle in cycles: + + CDATE = cycle.strftime("%Y%m%d%H") + PDY = cycle.strftime("%Y%m%d") + cyc = cycle.strftime("%H") + + filename = os.path.join(indir, 'osense_' + CDATE + '_final.dat') + + if not os.path.isfile(filename): + print('skipping ' + filename) + continue + + (convdata, satdata, idate) = osense.read_osense(filename) + + osensedata = osense.consolidate(convdata, satdata) + + outfilename = os.path.join(outdir, 'osense_' + CDATE + '.pkl') + print("saving file ", outfilename) + with open(outfilename, 'wb') as outfile: + dump([idate, osensedata], outfile) From 3de192f4a981ef0ed8700b8e60ac05e30b179845 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 11 Aug 2022 11:30:01 -0400 Subject: [PATCH 06/13] polishing up scripts and library --- src/EFSOI_Utilities/scripts/osense.py | 60 ++++++++++++++------ src/EFSOI_Utilities/scripts/quickstats.py | 56 ++++++++++++++++++ src/EFSOI_Utilities/scripts/reduce_osense.py | 39 ++----------- 3 files changed, 105 insertions(+), 50 deletions(-) create mode 100644 src/EFSOI_Utilities/scripts/quickstats.py diff --git a/src/EFSOI_Utilities/scripts/osense.py b/src/EFSOI_Utilities/scripts/osense.py index 74d98d4f..2df7b1aa 100644 --- a/src/EFSOI_Utilities/scripts/osense.py +++ b/src/EFSOI_Utilities/scripts/osense.py @@ -1,7 +1,7 @@ import struct +import sys import pandas as pd - # osense.py # Written by Andrew Eichmann, NOAA/NWS/NCEP/EMC @@ -117,11 +117,13 @@ def read_osense(filename): biaspredsf = '=' + 'f' * (npred + 1) biaspreds_size = struct.calcsize(biaspredsf) - print('read header: idate = ', idate, ', convnum + oznum = ', - convnum + oznum, ', satnum = ', satnum, ' npred = ', npred, - ', nanals = ', nanals, ' obsnum = ', obsnum, - ', should equal convnum + oznum + satnum: ', - convnum + oznum + satnum) + print('read header: idate =', idate) + print('convnum + oznum =', convnum + oznum) + print('satnum =', satnum) + print('npred =', npred) + print('nanals =', nanals) + print(' convnum + oznum + satnum: =', convnum + + oznum + satnum, ', should equal obsnum:', obsnum) # now read the conventional data records @@ -165,7 +167,7 @@ def read_osense(filename): print(convdata.describe()) print(satdata.describe()) - return((convdata, satdata, idate)) + return(convdata, satdata, idate) def consolidate(convdata, satdata, sensors='Default', osensefields='Default'): @@ -223,15 +225,41 @@ def consolidate(convdata, satdata, sensors='Default', osensefields='Default'): indices = convbycodes[convbycodes['source'] == 'Empty'].index convbycodes.drop(indices, inplace=True) - # some stattypes, namely 700, have no corresponding code, and so have - # nans in the code, source, and detailed_source fields. This replaces those - # fields with the stattype - nanmask = convbycodes.code.isna() - nanstattypes = convbycodes.loc[nanmask, ['stattype']] - for nanstattype in nanstattypes.stattype.unique(): - convbycodes.loc[convbycodes['stattype'] == nanstattype, - ['source', 'detailed_source']] = nanstattype +# this causes more trouble than it solves, but might be useful later + # # some stattypes, namely 700, have no corresponding code, and so have + # # nans in the code, source, and detailed_source fields. This replaces those + # # fields with the stattype + # nanmask = convbycodes.code.isna() + # nanstattypes = convbycodes.loc[nanmask, ['stattype']] + # for nanstattype in nanstattypes.stattype.unique(): + # convbycodes.loc[convbycodes['stattype'] == nanstattype, + # ['source', 'detailed_source']] = nanstattype osensedata = pd.concat([satdata[osensefields], convbycodes[osensefields]]) - return(osensedata) + return osensedata + + +def make_cycles(firstcycle, lastcycle): + + cycletimes = ['00', '06', '12', '18'] + + if lastcycle < firstcycle: + print('lastcycle', lastcycle, 'comes before firstcycle', + firstcycle, ', you probably don\'t want that') + sys.exit(1) + + if firstcycle.strftime("%H") not in cycletimes: + print('firstcycle', firstcycle, + 'needs to have an hour that is one of', cycletimes) + sys.exit(1) + + if lastcycle.strftime("%H") not in cycletimes: + print('lastcycle', lastcycle, + 'needs to have an hour that is one of', cycletimes) + sys.exit(1) + + # time between cycles + cycles = pd.date_range(firstcycle, lastcycle, freq='6H') + + return cycles diff --git a/src/EFSOI_Utilities/scripts/quickstats.py b/src/EFSOI_Utilities/scripts/quickstats.py new file mode 100644 index 00000000..e64cf624 --- /dev/null +++ b/src/EFSOI_Utilities/scripts/quickstats.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +from pickle import dump, load +from datetime import datetime +import os.path +import argparse +import osense + +parser = argparse.ArgumentParser() +parser.add_argument( + 'firstcycle', + help='first cycle to process, format YYYY-MM-DD-HH', + type=lambda s: datetime.strptime(s, '%Y-%m-%d-%H')) +parser.add_argument( + 'lastcycle', + help='last cycle to process, format YYYY-MM-DD-HH', + type=lambda s: datetime.strptime(s, '%Y-%m-%d-%H')) +parser.add_argument('indir', help='directory to read in reduced osense files') +parser.add_argument('outdir', help='directory to save stat files') +args = parser.parse_args() + +firstcycle = args.firstcycle +lastcycle = args.lastcycle +indir = args.indir +outdir = args.outdir + +grouping = 'source' +#grouping = 'detailed_source' +norms = ['osense_kin', 'osense_dry', 'osense_moist'] + +cycles = osense.make_cycles(firstcycle, lastcycle) + +print('running from', firstcycle, 'to', lastcycle) + +for cycle in cycles: + + CDATE = cycle.strftime("%Y%m%d%H") + + infilename = os.path.join(indir, 'osense_' + CDATE + '.pkl') + + if not os.path.isfile(infilename): + print(infilename + ' doesn\'t exist, skipping') + continue + + print('loading ' + infilename) + with open(infilename, 'rb') as infile: + [idate, osensedata] = load(infile) + + columns = [grouping] + norms + meanimpacts = osensedata[columns].groupby(grouping).mean() + sumimpacts = osensedata[columns].groupby(grouping).sum() + obcounts = osensedata[columns].groupby(grouping).count() + + outfilename = os.path.join(outdir, 'osensestats_' + CDATE + '.pkl') + print("saving file ", outfilename) + with open(outfilename, 'wb') as outfile: + dump([idate, meanimpacts, sumimpacts, obcounts], outfile) diff --git a/src/EFSOI_Utilities/scripts/reduce_osense.py b/src/EFSOI_Utilities/scripts/reduce_osense.py index 1ca05bcf..c121e20c 100644 --- a/src/EFSOI_Utilities/scripts/reduce_osense.py +++ b/src/EFSOI_Utilities/scripts/reduce_osense.py @@ -3,8 +3,8 @@ from datetime import datetime import os.path import argparse -import sys -import pandas as pd +#import sys +#import pandas as pd import osense parser = argparse.ArgumentParser() @@ -17,7 +17,7 @@ help='last cycle to process, format YYYY-MM-DD-HH', type=lambda s: datetime.strptime(s, '%Y-%m-%d-%H')) parser.add_argument('indir', help='directory to read in osense files') -parser.add_argument('outdir', help='directory to save output') +parser.add_argument('outdir', help='directory to save reduced osense files') args = parser.parse_args() firstcycle = args.firstcycle @@ -25,47 +25,18 @@ indir = args.indir outdir = args.outdir -cycletimes = ['00', '06', '12', '18'] - -if lastcycle < firstcycle: - print('lastcycle', lastcycle, 'comes before firstcycle', - firstcycle, ', you probably don\'t want that') - sys.exit(1) - -if firstcycle.strftime("%H") not in cycletimes: - print('firstcycle', firstcycle, - 'needs to have an hour that is one of', cycletimes) - sys.exit(1) - -if lastcycle.strftime("%H") not in cycletimes: - print('lastcycle', lastcycle, 'needs to have an hour that is one of', cycletimes) - sys.exit(1) - -if not os.path.isdir(indir): - print('indir', indir, 'is not a directory') - sys.exit(1) - -if not os.path.isdir(outdir): - print('outdir', outdir, 'is not a directory') - sys.exit(1) - - print('running from', firstcycle, 'to', lastcycle) -# time between cycles -cycles = pd.date_range(firstcycle, lastcycle, freq='6H') - +cycles = osense.make_cycles(firstcycle, lastcycle) for cycle in cycles: CDATE = cycle.strftime("%Y%m%d%H") - PDY = cycle.strftime("%Y%m%d") - cyc = cycle.strftime("%H") filename = os.path.join(indir, 'osense_' + CDATE + '_final.dat') if not os.path.isfile(filename): - print('skipping ' + filename) + print(filename + ' doesn\'t exist, skipping') continue (convdata, satdata, idate) = osense.read_osense(filename) From e6620b6eac9ad17c6e127bcac358f242f6c21761 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 11 Aug 2022 15:41:44 -0400 Subject: [PATCH 07/13] more polishing --- src/EFSOI_Utilities/scripts/allstats.py | 96 ++++++++++++++++++++ src/EFSOI_Utilities/scripts/plotstats.py | 71 +++++++++++++++ src/EFSOI_Utilities/scripts/quickstats.py | 4 +- src/EFSOI_Utilities/scripts/reduce_osense.py | 8 +- 4 files changed, 173 insertions(+), 6 deletions(-) create mode 100644 src/EFSOI_Utilities/scripts/allstats.py create mode 100644 src/EFSOI_Utilities/scripts/plotstats.py diff --git a/src/EFSOI_Utilities/scripts/allstats.py b/src/EFSOI_Utilities/scripts/allstats.py new file mode 100644 index 00000000..7304e1fc --- /dev/null +++ b/src/EFSOI_Utilities/scripts/allstats.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +from pickle import dump, load +from datetime import datetime +import os.path +import argparse +import pandas as pd +import osense + +parser = argparse.ArgumentParser() +parser.add_argument( + 'firstcycle', + help='first cycle to process, format YYYY-MM-DD-HH', + type=lambda s: datetime.strptime(s, '%Y-%m-%d-%H')) +parser.add_argument( + 'lastcycle', + help='last cycle to process, format YYYY-MM-DD-HH', + type=lambda s: datetime.strptime(s, '%Y-%m-%d-%H')) +parser.add_argument('indir', help='directory to read and write osense stat files') +args = parser.parse_args() + +firstcycle = args.firstcycle +lastcycle = args.lastcycle +indir = args.indir + + +norms = ['osense_kin', 'osense_dry', 'osense_moist'] + +# there's a better way to do this, maybe with xarrays +moist_count = pd.DataFrame() +moist_mean = pd.DataFrame() +moist_sum = pd.DataFrame() +dry_count = pd.DataFrame() +dry_mean = pd.DataFrame() +dry_sum = pd.DataFrame() +kin_count = pd.DataFrame() +kin_mean = pd.DataFrame() +kin_sum = pd.DataFrame() + +print('running from', firstcycle, 'to', lastcycle) + +cycles = osense.make_cycles(firstcycle, lastcycle) + +for cycle in cycles: + + CDATE = cycle.strftime("%Y%m%d%H") + + infilename = os.path.join(indir, 'osensestats_' + CDATE + '.pkl') + + if not os.path.isfile(infilename): + print(infilename + ' doesn\'t exist, skipping') + continue + + print('loading ' + infilename) + with open(infilename, 'rb') as infile: + [idate, meanimpacts, sumimpacts, obcounts] = load(infile) + + moist_count = pd.concat([moist_count, obcounts['osense_moist']], axis=1) + moist_mean = pd.concat([moist_mean, meanimpacts['osense_moist']], axis=1) + moist_sum = pd.concat([moist_sum, sumimpacts['osense_moist']], axis=1) + dry_count = pd.concat([dry_count, obcounts['osense_dry']], axis=1) + dry_mean = pd.concat([dry_mean, meanimpacts['osense_dry']], axis=1) + dry_sum = pd.concat([dry_sum, sumimpacts['osense_dry']], axis=1) + kin_count = pd.concat([kin_count, obcounts['osense_kin']], axis=1) + kin_mean = pd.concat([kin_mean, meanimpacts['osense_kin']], axis=1) + kin_sum = pd.concat([kin_sum, sumimpacts['osense_kin']], axis=1) + +columns = ['mean total impact', 'fractional impact', + 'mean num obs', 'mean impact per ob'] +moist = pd.DataFrame(columns=columns) +dry = pd.DataFrame(columns=columns) +kin = pd.DataFrame(columns=columns) + +# The abs operators are in here so that all of the fractional impacts are +# positive whether the mean impacts are negative or positive +moist['mean total impact'] = moist_sum.mean(axis=1) +moist_impact_sum = abs(moist['mean total impact']).sum() +moist['fractional impact'] = abs(moist['mean total impact'])/moist_impact_sum * 100 +moist['mean num obs'] = moist_count.mean(axis=1) +moist['mean impact per ob'] = moist['mean total impact']/moist['mean num obs'] + +dry['mean total impact'] = dry_sum.mean(axis=1) +dry_impact_sum = abs(dry['mean total impact']).sum() +dry['fractional impact'] = abs(dry['mean total impact'])/dry_impact_sum * 100 +dry['mean num obs'] = dry_count.mean(axis=1) +dry['mean impact per ob'] = dry['mean total impact']/dry['mean num obs'] + +kin['mean total impact'] = kin_sum.mean(axis=1) +kin_impact_sum = abs(kin['mean total impact']).sum() +kin['fractional impact'] = abs(kin['mean total impact'])/kin_impact_sum * 100 +kin['mean num obs'] = kin_count.mean(axis=1) +kin['mean impact per ob'] = kin['mean total impact']/kin['mean num obs'] + +outfilename = os.path.join(indir, 'osensestats_all.pkl') +print("saving file ", outfilename) +with open(outfilename, 'wb') as outfile: + dump([firstcycle, lastcycle, moist, dry, kin], outfile) diff --git a/src/EFSOI_Utilities/scripts/plotstats.py b/src/EFSOI_Utilities/scripts/plotstats.py new file mode 100644 index 00000000..c3a08f64 --- /dev/null +++ b/src/EFSOI_Utilities/scripts/plotstats.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +from pickle import load +from datetime import datetime +import os.path +import argparse +import pandas as pd +import osense + +parser = argparse.ArgumentParser() +parser.add_argument('indir', help='directory to read in osense stat files') +parser.add_argument('outdir', help='directory to save plots') +args = parser.parse_args() + +indir = args.indir +outdir = args.outdir + + +norms = ['osense_kin', 'osense_dry', 'osense_moist'] + +infilename = os.path.join(indir, 'osensestats_all.pkl') + +print('loading ' + infilename) +with open(infilename, 'rb') as infile: + [firstcycle, lastcycle,moist, dry, kin] = load(infile) + + +filename= os.path.join(outdir, 'meantotal.png') +title='mean total impact per cycle (J/kg)' +ax=moist['mean total impact'].sort_values(ascending=False).plot.barh( + xlabel='', + ylabel='J/kg', + title=title, + figsize=(10, 6) + ) +fig=ax.get_figure() +fig.savefig(filename,bbox_inches='tight') +fig.clear() + +filename= os.path.join(outdir, 'fractional.png') +title='fractional impact (%)' +ax=moist['fractional impact'].sort_values().plot.barh( + title=title, + figsize=(10, 6) + ) +fig=ax.get_figure() +fig.savefig(filename,bbox_inches='tight') +fig.clear() + +filename= os.path.join(outdir, 'obspercycle.png') +title='mean observations per cycle' +ax=moist['mean num obs'].sort_values().plot.barh( + title=title, + figsize=(10, 6) + ) +fig=ax.get_figure() +fig.savefig(filename,bbox_inches='tight') +fig.clear() + +filename= os.path.join(outdir, 'impactperobs.png') +title='impact per observation (J/kg)' +ax=moist['mean impact per ob'].sort_values(ascending=False).plot.barh( + xlabel='', + ylabel='J/kg', + title=title, + figsize=(10, 6) + ) +fig=ax.get_figure() +fig.savefig(filename,bbox_inches='tight') +fig.clear() + + diff --git a/src/EFSOI_Utilities/scripts/quickstats.py b/src/EFSOI_Utilities/scripts/quickstats.py index e64cf624..97e05807 100644 --- a/src/EFSOI_Utilities/scripts/quickstats.py +++ b/src/EFSOI_Utilities/scripts/quickstats.py @@ -43,7 +43,7 @@ print('loading ' + infilename) with open(infilename, 'rb') as infile: - [idate, osensedata] = load(infile) + [exp, cdate, osensedata] = load(infile) columns = [grouping] + norms meanimpacts = osensedata[columns].groupby(grouping).mean() @@ -53,4 +53,4 @@ outfilename = os.path.join(outdir, 'osensestats_' + CDATE + '.pkl') print("saving file ", outfilename) with open(outfilename, 'wb') as outfile: - dump([idate, meanimpacts, sumimpacts, obcounts], outfile) + dump([exp, cdate, meanimpacts, sumimpacts, obcounts], outfile) diff --git a/src/EFSOI_Utilities/scripts/reduce_osense.py b/src/EFSOI_Utilities/scripts/reduce_osense.py index c121e20c..361242d2 100644 --- a/src/EFSOI_Utilities/scripts/reduce_osense.py +++ b/src/EFSOI_Utilities/scripts/reduce_osense.py @@ -3,8 +3,6 @@ from datetime import datetime import os.path import argparse -#import sys -#import pandas as pd import osense parser = argparse.ArgumentParser() @@ -18,12 +16,14 @@ type=lambda s: datetime.strptime(s, '%Y-%m-%d-%H')) parser.add_argument('indir', help='directory to read in osense files') parser.add_argument('outdir', help='directory to save reduced osense files') +parser.add_argument('exp', help='experiment name') args = parser.parse_args() firstcycle = args.firstcycle lastcycle = args.lastcycle indir = args.indir outdir = args.outdir +exp = args.exp print('running from', firstcycle, 'to', lastcycle) @@ -39,11 +39,11 @@ print(filename + ' doesn\'t exist, skipping') continue - (convdata, satdata, idate) = osense.read_osense(filename) + (convdata, satdata, cdate) = osense.read_osense(filename) osensedata = osense.consolidate(convdata, satdata) outfilename = os.path.join(outdir, 'osense_' + CDATE + '.pkl') print("saving file ", outfilename) with open(outfilename, 'wb') as outfile: - dump([idate, osensedata], outfile) + dump([exp, cycle, osensedata], outfile) From adca6c5b201280cc511a0b75333ef9ac7e70304a Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 11 Aug 2022 16:14:05 -0400 Subject: [PATCH 08/13] more tweaking --- .../scripts/consolidate_osense.py | 121 ---------------- .../scripts/readcycsatconsolidate.py | 134 ------------------ 2 files changed, 255 deletions(-) delete mode 100755 src/EFSOI_Utilities/scripts/consolidate_osense.py delete mode 100755 src/EFSOI_Utilities/scripts/readcycsatconsolidate.py diff --git a/src/EFSOI_Utilities/scripts/consolidate_osense.py b/src/EFSOI_Utilities/scripts/consolidate_osense.py deleted file mode 100755 index 0e8e7b08..00000000 --- a/src/EFSOI_Utilities/scripts/consolidate_osense.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import osense -import pandas as pd -import pickle -import datetime -import os.path - -exp='rodeo' -#exp='baily' - -#datadir='/work2/noaa/stmp/aeichman/comrot/' + exp + '/osense/' -#datadir='/work/noaa/stmp/aeichman/comrot/' + exp + '/osense/' -#rootdir = '/Users/mossland/Desktop/work/EFSOI/' + exp -rootdir = '/work/noaa/da/aeichman/gsidev/EFSOI-dev-post/GSI/util/EFSOI_Utilities/scripts/' + exp -indir = '/work2/noaa/stmp/aeichman/comrot/rodeo/osense/' -outdir = rootdir + '/consolidated/' - -# baily -cycleinit = datetime.datetime( 2021, 12,10, 0 ) -cycleend = datetime.datetime( 2021, 12, 14, 18 ) - -# rodeo -cycleinit = datetime.datetime( 2020, 12,14, 12 ) -cycleinit = datetime.datetime( 2020, 12,14, 18 ) -#cycleend = datetime.datetime( 2021, 1, 13, 18 ) - - -# The following determines whether the conventional data points are aggregated -# by the column in convdata_codes.cvs "message02" (DETAILED = False ) or "message" -# (DETAILED = True). Generally message offers finer distinctions between the -# sources, though these are somewhat arbitrary. See convdata_codes.csv -DETAILED = True - -# these are satellite sensors to consolidate across plaforms -sensors = ['airs', - 'amsr', - 'amsua', - 'atms', - 'avhrr', - 'cris', - 'iasi', - 'mhs', - 'saphir', - 'seviri', - 'ssmis'] - -# these are fields in the osense files and can be expanded or reduced -osensefields = ['source', - 'detailed_source', - 'indxsat', - 'osense_kin', - 'osense_dry', - 'osense_moist', - 'assimilated', - 'lat', - 'lon', - 'pres'] - -# time between cycles -cycles = pd.date_range( cycleinit, cycleend, freq='6H' ) - - -for thiscycle in cycles: - - CDATE = thiscycle.strftime("%Y%m%d%H") - PDY = thiscycle.strftime("%Y%m%d") - cyc = thiscycle.strftime("%H") - - filename = indir + 'osense_' + CDATE + '_final.dat' - - if not os.path.isfile(filename): - print('skipping ' + filename) - continue - - ( convdata, satdata, idate )= osense.read_osense( filename ) - - satdata['detailed_source'] = satdata['obtype'] - satdata['source'] = satdata['obtype'] - - for sensor in sensors: - mask = satdata.source.str.contains(sensor) - satdata.loc[mask,'source'] = sensor.upper() - - - # now consolidate conventional data - - convcodes = pd.read_csv('convdata_codes.csv') - - # associate each data point with its source, by code - # this also effectively adds the columns 'source' and 'detailed_source' - convbycodes=pd.merge(convdata,convcodes,how='left', - left_on='stattype', - right_on='code') - - # drop message='NA'/message02='Empty' - # data not dumped/not used - indices=convbycodes[convbycodes['source']=='Empty'].index - convbycodes.drop(indices,inplace = True) - - # some stattypes, namely 700, have no corresponding code, and so have - # nans in the code, source, and detailed_source fields. This replaces those - # fields with the stattype - nanmask = convbycodes.code.isna() - nanstattypes = convbycodes.loc[nanmask,['stattype']] - for nanstattype in nanstattypes.stattype.unique(): - convbycodes.loc[convbycodes['stattype'] == nanstattype, \ - ['source','detailed_source']] = nanstattype - - - osensedata = pd.concat([ satdata[osensefields], convbycodes[osensefields] ] ) - - - outfilename = outdir + 'osense_' + CDATE + '.pkl' - outfile = open( outfilename, 'wb' ) - pickle.dump([ idate, osensedata ], outfile ) - outfile.close() - - - diff --git a/src/EFSOI_Utilities/scripts/readcycsatconsolidate.py b/src/EFSOI_Utilities/scripts/readcycsatconsolidate.py deleted file mode 100755 index 7c978e5f..00000000 --- a/src/EFSOI_Utilities/scripts/readcycsatconsolidate.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Aug 19 14:27:30 2021 - -@author: mossland -""" - -#import osense_old -import osense -import pandas as pd -import pickle -import datetime -import os.path - -#exp='desert' -#exp='narwal' -#exp='carrot' -#exp='vassal' -#exp='wombat' -exp='rodeo' -#datadir = '/Users/mossland/Desktop/work/EFSOI/carrot/' -#datadir='/work/noaa/stmp/aeichman/comrot/carrot/osense/' -datadir='/work2/noaa/stmp/aeichman/comrot/' + exp + '/osense/' -#datadir='/work/noaa/stmp/aeichman/comrot/' + exp + '/osense/' - -#cycleinit = datetime.datetime( 2021, 6, 9, 12 ) -#cycleend = datetime.datetime( 2021, 6, 11, 0 ) -cycleinit = datetime.datetime( 2020, 12,14, 12 ) -#cycleend = datetime.datetime( 2020, 12, 17, 12 ) -#cycleinit = datetime.datetime( 2020, 12,17, 18 ) -cycleend = datetime.datetime( 2021, 1, 12, 18 ) -#cycleinit= datetime.datetime( 2021, 1, 9, 12 ) -#cycleend= datetime.datetime( 2021, 1, 12, 18 ) - -# these are sensors to consolidate across plaforms -sensors = ['airs','amsr','amsua','atms','avhrr','cris','iasi','mhs','saphir','seviri','ssmis'] - -# these probably shouldn't be changed -cycledelta = datetime.timedelta( hours = 6 ) - - - - -thiscycle = cycleinit -while thiscycle <= cycleend: - - CDATE = thiscycle.strftime("%Y%m%d%H") - PDY = thiscycle.strftime("%Y%m%d") - cyc = thiscycle.strftime("%H") - - - filename = 'osense_' + CDATE + '_final.dat' - - if not os.path.isfile(datadir + filename): - print('skipping ' + datadir + filename) - thiscycle += cycledelta - continue - - #( convdata, satdata )= osense.read_osense('osense_2017091000.dat') - ( convdata, satdata, idate )= osense.read_osense(datadir + filename) - - print(convdata['assimilated'].value_counts()) - print(satdata['assimilated'].value_counts() ) - - satdata = satdata[satdata['assimilated'] == 1 ] - convdata = convdata[convdata['assimilated'] == 1 ] - - # creates a DataFrame with the mean of each satellite instrument - #meanbyobtype = satdata.groupby('obtype').mean() - - for sensor in sensors: - mask = satdata.obtype.str.contains(sensor) - satdata.loc[mask,'obtype'] = sensor.upper() - - - satmeanbyobtype=satdata[['obtype','osense_kin','osense_dry','osense_moist']].groupby('obtype').mean() - #cmeanbyobtype=convdata[['stattype','osense_kin','osense_dry','osense_moist']].groupby('stattype').mean() - - - convcodes = pd.read_csv('convdata_codes.csv') - - # associate each data point with its source, by code - # it would be more efficient to take the mean of the observation sensitivities by - # code/stattype, but this way the mean is by the message column in the codes - # (ADPUPA, AIRCRAFT, etc) to consolidate for simpler graphing. Taking the mean - # by code/stattype would break it down by data source more - convbycodes=pd.merge(convdata,convcodes,how='left',left_on='stattype', right_on='code') - - # drop message='NA'/message02='Empty' - indices=convbycodes[convbycodes['message02']=='Empty'].index - convbycodes.drop(indices,inplace = True) - - #convgrouping='message02' - convgrouping='message' - - convmeanbymsg2=convbycodes[[convgrouping,'osense_kin','osense_dry','osense_moist']].groupby(convgrouping).mean() - - meanimpacts=pd.concat([satmeanbyobtype,convmeanbymsg2]) - - - satsumbyobtype=satdata[['obtype','osense_kin','osense_dry','osense_moist']].groupby('obtype').sum() - convsumbymsg2=convbycodes[[convgrouping,'osense_kin','osense_dry','osense_moist']].groupby(convgrouping).sum() - - sumimpacts=pd.concat([satsumbyobtype,convsumbymsg2]) - - - satcntbyobtype=satdata[['obtype','osense_kin','osense_dry','osense_moist']].groupby('obtype').count() - convcntbymsg2=convbycodes[[convgrouping,'osense_kin','osense_dry','osense_moist']].groupby(convgrouping).count() - cntimpacts=pd.concat([satcntbyobtype,convcntbymsg2]) - - moist = pd.concat([cntimpacts['osense_moist'],sumimpacts['osense_moist'],meanimpacts['osense_moist']], axis=1 ) - moist.columns = ['count', 'sum','mean'] - dry = pd.concat([cntimpacts['osense_dry'],sumimpacts['osense_dry'],meanimpacts['osense_dry']], axis=1 ) - dry.columns = ['count', 'sum','mean'] - kin = pd.concat([cntimpacts['osense_kin'],sumimpacts['osense_kin'],meanimpacts['osense_kin']], axis=1 ) - kin.columns = ['count', 'sum','mean'] - - - - - - - outfilename = 'stat_' + CDATE + '.pkl' - #outfile = open('./pickle2/'+ outfilename,'wb') - outfile = open('./' + exp + '/'+ outfilename,'wb') - pickle.dump([moist,dry,kin],outfile) - outfile.close() - - thiscycle += cycledelta - - - - From b4b0a258d3191974b3c9a70168d07854ddd5e2a9 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 11 Aug 2022 16:15:20 -0400 Subject: [PATCH 09/13] added exp and date info to plots --- src/EFSOI_Utilities/scripts/allstats.py | 5 +++-- src/EFSOI_Utilities/scripts/plotstats.py | 24 +++++++++++++++--------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/EFSOI_Utilities/scripts/allstats.py b/src/EFSOI_Utilities/scripts/allstats.py index 7304e1fc..df568d21 100644 --- a/src/EFSOI_Utilities/scripts/allstats.py +++ b/src/EFSOI_Utilities/scripts/allstats.py @@ -52,7 +52,7 @@ print('loading ' + infilename) with open(infilename, 'rb') as infile: - [idate, meanimpacts, sumimpacts, obcounts] = load(infile) + [exp, cdate, meanimpacts, sumimpacts, obcounts] = load(infile) moist_count = pd.concat([moist_count, obcounts['osense_moist']], axis=1) moist_mean = pd.concat([moist_mean, meanimpacts['osense_moist']], axis=1) @@ -70,6 +70,7 @@ dry = pd.DataFrame(columns=columns) kin = pd.DataFrame(columns=columns) + # The abs operators are in here so that all of the fractional impacts are # positive whether the mean impacts are negative or positive moist['mean total impact'] = moist_sum.mean(axis=1) @@ -93,4 +94,4 @@ outfilename = os.path.join(indir, 'osensestats_all.pkl') print("saving file ", outfilename) with open(outfilename, 'wb') as outfile: - dump([firstcycle, lastcycle, moist, dry, kin], outfile) + dump([exp, firstcycle, lastcycle, moist, dry, kin], outfile) diff --git a/src/EFSOI_Utilities/scripts/plotstats.py b/src/EFSOI_Utilities/scripts/plotstats.py index c3a08f64..b355a9ad 100644 --- a/src/EFSOI_Utilities/scripts/plotstats.py +++ b/src/EFSOI_Utilities/scripts/plotstats.py @@ -21,11 +21,13 @@ print('loading ' + infilename) with open(infilename, 'rb') as infile: - [firstcycle, lastcycle,moist, dry, kin] = load(infile) + [exp,firstcycle, lastcycle,moist, dry, kin] = load(infile) +start = firstcycle.strftime("%Y-%m-%d") +end = lastcycle.strftime("%Y-%m-%d") -filename= os.path.join(outdir, 'meantotal.png') -title='mean total impact per cycle (J/kg)' +filename= os.path.join(outdir, exp + '_meantotal.png') +title='mean total impact per cycle (J/kg), ' + exp + ' ' + start + ' to ' + end ax=moist['mean total impact'].sort_values(ascending=False).plot.barh( xlabel='', ylabel='J/kg', @@ -33,31 +35,34 @@ figsize=(10, 6) ) fig=ax.get_figure() +print('saving' + filename) fig.savefig(filename,bbox_inches='tight') fig.clear() -filename= os.path.join(outdir, 'fractional.png') -title='fractional impact (%)' +filename= os.path.join(outdir, exp + '_fractional.png') +title='fractional impact (%), ' + exp + ' ' + start + ' to ' + end ax=moist['fractional impact'].sort_values().plot.barh( title=title, figsize=(10, 6) ) fig=ax.get_figure() +print('saving' + filename) fig.savefig(filename,bbox_inches='tight') fig.clear() -filename= os.path.join(outdir, 'obspercycle.png') -title='mean observations per cycle' +filename= os.path.join(outdir, exp + '_obspercycle.png') +title='mean observations per cycle, ' + exp + ' ' + start + ' to ' + end ax=moist['mean num obs'].sort_values().plot.barh( title=title, figsize=(10, 6) ) fig=ax.get_figure() +print('saving' + filename) fig.savefig(filename,bbox_inches='tight') fig.clear() -filename= os.path.join(outdir, 'impactperobs.png') -title='impact per observation (J/kg)' +filename= os.path.join(outdir, exp + '_impactperobs.png') +title='impact per observation (J/kg), ' + exp ax=moist['mean impact per ob'].sort_values(ascending=False).plot.barh( xlabel='', ylabel='J/kg', @@ -65,6 +70,7 @@ figsize=(10, 6) ) fig=ax.get_figure() +print('saving' + filename) fig.savefig(filename,bbox_inches='tight') fig.clear() From e33d3309a5c68c5e7412613e22a99ca2d8bc1d11 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 11 Aug 2022 16:44:58 -0400 Subject: [PATCH 10/13] more of the same --- .../scripts/README_EFSOI_scripts | 47 ++++++++++++++++ src/EFSOI_Utilities/scripts/README_fv3gfs | 22 -------- src/EFSOI_Utilities/scripts/allstats.py | 8 +-- src/EFSOI_Utilities/scripts/plotstats.py | 54 +++++++++---------- 4 files changed, 78 insertions(+), 53 deletions(-) create mode 100644 src/EFSOI_Utilities/scripts/README_EFSOI_scripts delete mode 100644 src/EFSOI_Utilities/scripts/README_fv3gfs diff --git a/src/EFSOI_Utilities/scripts/README_EFSOI_scripts b/src/EFSOI_Utilities/scripts/README_EFSOI_scripts new file mode 100644 index 00000000..6ff2ce77 --- /dev/null +++ b/src/EFSOI_Utilities/scripts/README_EFSOI_scripts @@ -0,0 +1,47 @@ + +This file, README_EFSOI_scripts, describes the files in GSI-utils/src/EFSOI_Utilities/scripts +under the branch EFSOI. They are used for working with data from EFSOI in GDAS/global-workflow + + +reduce_osense.py: Takes the osense data files and reduces them to about 10% the size for +easier handling, saving as python pickle files. +Usage: reduce_osense.py firstcycle lastcycle indir outdir exp +firstcycle, lastcycle: cycles of first and last expected osense files, format YYYY-MM-DD-HH +indir: location of osense files +outdir: where pickle files will be saved +exp: experiment name + +quickstats.py: Takes output from reduce_osense.py and makes per-cycle stats for plotting. +Usage: quickstats.py firstcycle lastcycle indir outdir +firstcycle, lastcycle: cycles of first and last expected osense files, format YYYY-MM-DD-HH +indir: location of output from reduce_osense.py +outdir: where pickle files with statistics will be saved + +allstats.py: Takes output from quickstats.py and makes all-experiment obs impact stats +Usage: allstats.py firstcycle lastcycle indir +firstcycle, lastcycle: cycles of first and last expected osense files, format YYYY-MM-DD-HH +indir: location of output from quickstats.py, also output directory + +quickstats.py: Takes output from allstats.py and makes all-experiment obs impact plots +Usage: quickstats.py indir outdir +indir: location of output from allstats.py +outdir: location of saved plots + +convdata_codes.csv: codes needed to organize convetional data sources, derived from +webpage (with the help of tools in pandas): +https://www.emc.ncep.noaa.gov/mmb/data_processing/prepbufr.doc/table_2.htm +May need to be periodically updated + +copyefsoifiles.sh: copies files from experiment to EFSOI staging directory after +gdasepos has run + +display_osense.py: reads and plots sensitivity data for single cycle after reading +from osense file generated by EFSOI executable + +getefsiofiles.sh: obtains files necessary for gdaseupd task in global workflow, +the modified version of which is the first step in the EFSOI process + +osense.py: contains routine to read osense file and others + +geoplot.py +osense2nc.py: converts osense files to netcdf files diff --git a/src/EFSOI_Utilities/scripts/README_fv3gfs b/src/EFSOI_Utilities/scripts/README_fv3gfs deleted file mode 100644 index b7643540..00000000 --- a/src/EFSOI_Utilities/scripts/README_fv3gfs +++ /dev/null @@ -1,22 +0,0 @@ - -This file, README_fv3gfs, describes the files in GSI/util/EFSOI_Utilities/scripts -under the branch EXP-efso_fv3. These files are expected to be merged into master -shortly (as of 2020-07-30) - - -getefsiofiles.sh: obtains files necessary for gdaseupd task in global workflow, -the modified version of which is the first step in the EFSOI process - -copyefsoifiles.sh: copies files from experiment to EFSOI staging directory after -gdasepos has run - -display_osense.py: reads and plots sensitivity data for single cycle after reading -from osense file generated by EFSOI executable - -osense.py: contains routine to read osense file - -convdata_codes.csv: codes needed to organize convetional data sources, derived from -webpage (with the help of tools in pandas): -https://www.emc.ncep.noaa.gov/mmb/data_processing/prepbufr.doc/table_2.htm -May need to be periodically updated - diff --git a/src/EFSOI_Utilities/scripts/allstats.py b/src/EFSOI_Utilities/scripts/allstats.py index df568d21..e125f0d1 100644 --- a/src/EFSOI_Utilities/scripts/allstats.py +++ b/src/EFSOI_Utilities/scripts/allstats.py @@ -15,7 +15,8 @@ 'lastcycle', help='last cycle to process, format YYYY-MM-DD-HH', type=lambda s: datetime.strptime(s, '%Y-%m-%d-%H')) -parser.add_argument('indir', help='directory to read and write osense stat files') +parser.add_argument( + 'indir', help='directory to read and write osense stat files') args = parser.parse_args() firstcycle = args.firstcycle @@ -71,11 +72,12 @@ kin = pd.DataFrame(columns=columns) -# The abs operators are in here so that all of the fractional impacts are +# The abs operators are in here so that all of the fractional impacts are # positive whether the mean impacts are negative or positive moist['mean total impact'] = moist_sum.mean(axis=1) moist_impact_sum = abs(moist['mean total impact']).sum() -moist['fractional impact'] = abs(moist['mean total impact'])/moist_impact_sum * 100 +moist['fractional impact'] = abs( + moist['mean total impact'])/moist_impact_sum * 100 moist['mean num obs'] = moist_count.mean(axis=1) moist['mean impact per ob'] = moist['mean total impact']/moist['mean num obs'] diff --git a/src/EFSOI_Utilities/scripts/plotstats.py b/src/EFSOI_Utilities/scripts/plotstats.py index b355a9ad..fd4394cc 100644 --- a/src/EFSOI_Utilities/scripts/plotstats.py +++ b/src/EFSOI_Utilities/scripts/plotstats.py @@ -4,7 +4,6 @@ import os.path import argparse import pandas as pd -import osense parser = argparse.ArgumentParser() parser.add_argument('indir', help='directory to read in osense stat files') @@ -21,57 +20,56 @@ print('loading ' + infilename) with open(infilename, 'rb') as infile: - [exp,firstcycle, lastcycle,moist, dry, kin] = load(infile) + [exp, firstcycle, lastcycle, moist, dry, kin] = load(infile) start = firstcycle.strftime("%Y-%m-%d") end = lastcycle.strftime("%Y-%m-%d") -filename= os.path.join(outdir, exp + '_meantotal.png') -title='mean total impact per cycle (J/kg), ' + exp + ' ' + start + ' to ' + end -ax=moist['mean total impact'].sort_values(ascending=False).plot.barh( +filename = os.path.join(outdir, exp + '_meantotal.png') +title = 'mean total impact per cycle (J/kg), ' + \ + exp + ' ' + start + ' to ' + end +ax = moist['mean total impact'].sort_values(ascending=False).plot.barh( xlabel='', ylabel='J/kg', title=title, figsize=(10, 6) - ) -fig=ax.get_figure() +) +fig = ax.get_figure() print('saving' + filename) -fig.savefig(filename,bbox_inches='tight') +fig.savefig(filename, bbox_inches='tight') fig.clear() -filename= os.path.join(outdir, exp + '_fractional.png') -title='fractional impact (%), ' + exp + ' ' + start + ' to ' + end -ax=moist['fractional impact'].sort_values().plot.barh( +filename = os.path.join(outdir, exp + '_fractional.png') +title = 'fractional impact (%), ' + exp + ' ' + start + ' to ' + end +ax = moist['fractional impact'].sort_values().plot.barh( title=title, figsize=(10, 6) - ) -fig=ax.get_figure() +) +fig = ax.get_figure() print('saving' + filename) -fig.savefig(filename,bbox_inches='tight') +fig.savefig(filename, bbox_inches='tight') fig.clear() -filename= os.path.join(outdir, exp + '_obspercycle.png') -title='mean observations per cycle, ' + exp + ' ' + start + ' to ' + end -ax=moist['mean num obs'].sort_values().plot.barh( +filename = os.path.join(outdir, exp + '_obspercycle.png') +title = 'mean observations per cycle, ' + exp + ' ' + start + ' to ' + end +ax = moist['mean num obs'].sort_values().plot.barh( title=title, figsize=(10, 6) - ) -fig=ax.get_figure() +) +fig = ax.get_figure() print('saving' + filename) -fig.savefig(filename,bbox_inches='tight') +fig.savefig(filename, bbox_inches='tight') fig.clear() -filename= os.path.join(outdir, exp + '_impactperobs.png') -title='impact per observation (J/kg), ' + exp -ax=moist['mean impact per ob'].sort_values(ascending=False).plot.barh( +filename = os.path.join(outdir, exp + '_impactperobs.png') +title = 'impact per observation (J/kg), ' + exp +ax = moist['mean impact per ob'].sort_values(ascending=False).plot.barh( xlabel='', ylabel='J/kg', title=title, figsize=(10, 6) - ) -fig=ax.get_figure() +) +fig = ax.get_figure() print('saving' + filename) -fig.savefig(filename,bbox_inches='tight') +fig.savefig(filename, bbox_inches='tight') fig.clear() - - From 3d00560b6ee9ee1a311a823d21cfcd86cd693f7c Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 11 Aug 2022 16:57:59 -0400 Subject: [PATCH 11/13] tweaks --- src/EFSOI_Utilities/scripts/plotstats.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/EFSOI_Utilities/scripts/plotstats.py b/src/EFSOI_Utilities/scripts/plotstats.py index fd4394cc..71a25b8c 100644 --- a/src/EFSOI_Utilities/scripts/plotstats.py +++ b/src/EFSOI_Utilities/scripts/plotstats.py @@ -13,9 +13,6 @@ indir = args.indir outdir = args.outdir - -norms = ['osense_kin', 'osense_dry', 'osense_moist'] - infilename = os.path.join(indir, 'osensestats_all.pkl') print('loading ' + infilename) @@ -62,7 +59,7 @@ fig.clear() filename = os.path.join(outdir, exp + '_impactperobs.png') -title = 'impact per observation (J/kg), ' + exp +title = 'impact per observation (J/kg), ' + exp + ' ' + start + ' to ' + end ax = moist['mean impact per ob'].sort_values(ascending=False).plot.barh( xlabel='', ylabel='J/kg', From 0d43a3aa4e09f63cd55bba757065b197000e6f75 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 11 Aug 2022 17:06:35 -0400 Subject: [PATCH 12/13] added matplotlib.use('Agg') for plotting on Orion --- src/EFSOI_Utilities/scripts/plotstats.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/EFSOI_Utilities/scripts/plotstats.py b/src/EFSOI_Utilities/scripts/plotstats.py index 71a25b8c..98964ca2 100644 --- a/src/EFSOI_Utilities/scripts/plotstats.py +++ b/src/EFSOI_Utilities/scripts/plotstats.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +import matplotlib # Matplotlib to make graphics +matplotlib.use('Agg') # Need this to generate figs when not running an Xserver (e.g. via PBS/LSF) from pickle import load from datetime import datetime import os.path From cfe1fa67ff02b4acc8999f56606403f3e7991aee Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 11 Aug 2022 17:18:05 -0400 Subject: [PATCH 13/13] cleaning up for PR --- src/EFSOI_Utilities/fix/README_EFSOI_fix | 16 --- .../fix/global_anavinfo.l127.txt | 116 ---------------- src/EFSOI_Utilities/scripts/copyefsoifiles.sh | 125 ------------------ src/EFSOI_Utilities/scripts/display_osense.py | 49 ------- src/EFSOI_Utilities/scripts/geoplot.py | 95 ------------- src/EFSOI_Utilities/scripts/getefsiofiles.sh | 77 ----------- src/EFSOI_Utilities/scripts/osense2nc.py | 88 ------------ 7 files changed, 566 deletions(-) delete mode 100644 src/EFSOI_Utilities/fix/README_EFSOI_fix delete mode 100644 src/EFSOI_Utilities/fix/global_anavinfo.l127.txt delete mode 100755 src/EFSOI_Utilities/scripts/copyefsoifiles.sh delete mode 100644 src/EFSOI_Utilities/scripts/display_osense.py delete mode 100644 src/EFSOI_Utilities/scripts/geoplot.py delete mode 100755 src/EFSOI_Utilities/scripts/getefsiofiles.sh delete mode 100644 src/EFSOI_Utilities/scripts/osense2nc.py diff --git a/src/EFSOI_Utilities/fix/README_EFSOI_fix b/src/EFSOI_Utilities/fix/README_EFSOI_fix deleted file mode 100644 index 6d8e68e7..00000000 --- a/src/EFSOI_Utilities/fix/README_EFSOI_fix +++ /dev/null @@ -1,16 +0,0 @@ - -directory: - -GSI/util/EFSOI_Utilities/fix - -file: -global_anavinfo_efsoi.l127.txt - -This is the EFSOI version of the GSI fix file global_anavinfo.l127.txt, -modified with an entry for EFSOI. This is located here until it can be -added to the GSI/fix directory in the code repository. It may have to be -updated if the default file is modified. The variable ANAVINFO in the -global-workflow config file config.efsoi should point to this file. - - - diff --git a/src/EFSOI_Utilities/fix/global_anavinfo.l127.txt b/src/EFSOI_Utilities/fix/global_anavinfo.l127.txt deleted file mode 100644 index 6363e450..00000000 --- a/src/EFSOI_Utilities/fix/global_anavinfo.l127.txt +++ /dev/null @@ -1,116 +0,0 @@ -correlated_observations:: -! isis method kreq kmult type cov_file -iasi_metop-c 2 0.0 1.0 sea Rcov_iasicsea -iasi_metop-b 2 0.0 1.0 sea Rcov_iasibsea -iasi_metop-c 2 0.0 1.0 land Rcov_iasicland -iasi_metop-b 2 0.0 1.0 land Rcov_iasibland -cris-fsr_n20 2 0.0 1.0 sea Rcov_crisn20 -cris-fsr_npp 2 0.0 1.0 sea Rcov_crisnpp -:: - -met_guess:: -!var level crtm_use desc orig_name - ps 1 -1 surface_pressure ps - z 1 -1 geopotential_height phis - u 127 2 zonal_wind u - v 127 2 meridional_wind v - div 127 -1 zonal_wind div - vor 127 -1 meridional_wind vor - tv 127 2 virtual_temperature tv - q 127 2 specific_humidity sphu - oz 127 2 ozone ozone - cw 127 10 cloud_condensate cw - ql 127 12 cloud_liquid ql - qi 127 12 cloud_ice qi -:: - -state_derivatives:: -!var level src - ps 1 met_guess - u 127 met_guess - v 127 met_guess - tv 127 met_guess - q 127 met_guess - oz 127 met_guess - cw 127 met_guess - prse 128 met_guess -:: - -state_tendencies:: -!var levels source - u 127 met_guess - v 127 met_guess - tv 127 met_guess - q 127 met_guess - cw 127 met_guess - oz 127 met_guess - prse 128 met_guess -:: - -state_vector:: -!var level itracer source funcof - u 127 0 met_guess u - v 127 0 met_guess v - tv 127 0 met_guess tv - tsen 127 0 met_guess tv,q - q 127 1 met_guess q - oz 127 1 met_guess oz - ql 127 1 met_guess ql - qi 127 1 met_guess qi - prse 128 0 met_guess prse - ps 1 0 met_guess prse - sst 1 0 met_guess sst -:: - - -state_vector_efsoi:: -!var level itracer source funcof - u 127 0 met_guess u - v 127 0 met_guess v - tv 127 0 met_guess tv - q 127 1 met_guess q - ps 1 0 met_guess prse -:: - - - -control_vector_enkf:: -!var level itracer as/tsfc_sdv an_amp0 source funcof - u 127 0 1.00 -1.0 state u,v - v 127 0 1.00 -1.0 state u,v - ps 1 0 1.20 -1.0 state prse -!pst 1 0 1.20 -1.0 state prse,u,v - tv 127 0 1.50 -1.0 state tv - q 127 1 1.50 -1.0 state q - oz 127 1 2.00 -1.0 state oz -!sst 1 0 1.00 -1.0 state sst -!cw 127 1 1.00 -1.0 state cw -!stl 1 0 3.00 -1.0 motley sst -!sti 1 0 3.00 -1.0 motley sst -:: - -control_vector:: -!var level itracer as/tsfc_sdv an_amp0 source funcof - sf 127 0 1.00 -1.0 state u,v - vp 127 0 1.00 -1.0 state u,v - ps 1 0 1.20 -1.0 state prse - t 127 0 1.50 -1.0 state tv - q 127 1 1.50 -1.0 state q - oz 127 1 2.00 -1.0 state oz - sst 1 0 1.00 -1.0 state sst - cw 127 1 1.00 -1.0 state cw - stl 1 0 3.00 -1.0 motley sst - sti 1 0 3.00 -1.0 motley sst -:: - -! Following table shows the use of all four prescribed trace gas data. -! To turn off any one of any combination of trace gas input, add "!" -! in the first column of that trace gas name. To use all default -! trace gas profiles, just delete the following seven lines. -chem_guess:: -!var level itracer crtm_use type orig_name -!ch4 64 1 2 n/a ch4 - co2 127 1 0 n/a co2 -!co 64 1 2 n/a co -!n2o 64 1 2 n/a n2o -:: diff --git a/src/EFSOI_Utilities/scripts/copyefsoifiles.sh b/src/EFSOI_Utilities/scripts/copyefsoifiles.sh deleted file mode 100755 index 466a7b7c..00000000 --- a/src/EFSOI_Utilities/scripts/copyefsoifiles.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/sh -x -################################################################################ -#### UNIX Script Documentation Block -# . . -# Script name: copyefsoifiles.sh -# Script description: copy files required for EFSOI from COMROT -# -# Author: Andrew Eichmann Org: NCEP/EMC Date: 2021-03-05 -# -# Abstract: This script copies the files required to run EFSOI executable -# from COMROT that has generated 24/30 hour forecasts to staging -# directory for testing purposes -# -# $Id$ -# -# Attributes: -# Language: POSIX shell -# Machine: Hera -# -################################################################################ - - - -COMROT= -STGDIR= - -EDATE=2019111818 -NENS=20 -NDATE=/scratch1/NCEPDEV/global/Fanglin.Yang/save/VRFY/vsdb/nwprod/util/exec/ndate - -PDATE=`${NDATE} -6 ${EDATE}` -VDATE=`${NDATE} +24 ${EDATE}` - -EDATECYC="${EDATE:8:2}" -PDATECYC="${PDATE:8:2}" -VDATECYC="${VDATE:8:2}" - -EDATEDIR=enkfgdas."${EDATE:0:8}" -PDATEDIR=enkfgdas."${PDATE:0:8}" -VDATEDIR=enkfgdas."${VDATE:0:8}" - - -echo $PDATE -echo $VDATE -echo $EDATECYC -echo $EDATEDIR - -# set up 30 hour forecast - -cd $STGDIR - -if [ -d "$PDATEDIR" ]; then - echo "$PDATEDIR exists." -else - echo "$PDATEDIR does not exist, creating it" - mkdir $PDATEDIR -fi - -cd $PDATEDIR - -if [ -d "$PDATECYC" ]; then - echo "$PDATECYC exists." -else - echo "$PDATECYC does not exist, creating it" - mkdir $PDATECYC -fi - -cd $PDATECYC - -cp -vi $COMROT/$PDATEDIR/$PDATECYC/gdas.t${PDATECYC}z.atmf030.ensmean.* . -cp -vi $COMROT/$PDATEDIR/$PDATECYC/gdas.t${PDATECYC}z.atmf006.ensmean.* . - - -# set up 24 hour forecast - -cd $STGDIR - -if [ -d "$EDATEDIR" ]; then - echo "$EDATEDIR exists." -else - echo "$EDATEDIR does not exist, creating it" - mkdir $EDATEDIR -fi - -cd $EDATEDIR - -if [ -d "$EDATECYC" ]; then - echo "$EDATECYC exists." -else - echo "$EDATECYC does not exist, creating it" - mkdir $EDATECYC -fi - -cd $EDATECYC - -cp -vi $COMROT/$EDATEDIR/$EDATECYC/gdas.t${EDATECYC}z.atmf024.ensmean.* . -cp -vi $COMROT/$EDATEDIR/$EDATECYC/gdas.t${EDATECYC}z.atmf006.ensmean.* . -cp -vi $COMROT/$EDATEDIR/$EDATECYC/gdas.t${EDATECYC}z.abias_int.ensmean . - - - -imem=1 -while [[ $imem -le $NENS ]]; do - member="mem"`printf %03i $imem` - - if [ -d "$member" ]; then - echo "$member exists." - else - echo "$member does not exist, creating it" - mkdir $member - fi - - cd $member - - cp -vi $COMROT/$EDATEDIR/$EDATECYC/$member/gdas.t${EDATECYC}z.atmf024.nemsio . - cp -vi $COMROT/$EDATEDIR/$EDATECYC/$member/gdas.t${EDATECYC}z.atmf024s.nemsio . - - cd .. - - (( imem = $imem + 1 )) -done - - - - diff --git a/src/EFSOI_Utilities/scripts/display_osense.py b/src/EFSOI_Utilities/scripts/display_osense.py deleted file mode 100644 index ae6f6b01..00000000 --- a/src/EFSOI_Utilities/scripts/display_osense.py +++ /dev/null @@ -1,49 +0,0 @@ -import osense -import pandas as pd - - -( convdata, satdata )= osense.read_osense('osense_2017091000.dat.out') - -# creates a DataFrame with the mean of each satellite instrument -#meanbyobtype = satdata.groupby('obtype').mean() - -meanbyobtype=satdata[['obtype','osense_kin','osense_dry','osense_moist']].groupby('obtype').mean() -#cmeanbyobtype=convdata[['stattype','osense_kin','osense_dry','osense_moist']].groupby('stattype').mean() - -convcodes = pd.read_csv('convdata_codes.csv') - -# associate each data point with its source, by code -# it would be more efficient to take the mean of the observation sensitivities by -# code/stattype, but this way the mean is by the message column in the codes -# (ADPUPA, AIRCRAFT, etc) to consolidate for simpler graphing. Taking the mean -# by code/stattype would break it down by data source more -convbycodes=pd.merge(convdata,convcodes,how='left',left_on='stattype', right_on='code') - -convmean=convbycodes[['message','osense_kin','osense_dry','osense_moist']].groupby('message').mean() -alltheobtypes=pd.concat([meanbyobtype,convmean]) - -figuresize = (10,6) - -o_dry=alltheobtypes['osense_dry'].sort_values(ascending=False) -dry_plot=o_dry.plot.barh(title = '2017091000 mean obs sensitivity, dry', figsize=figuresize); -fig=dry_plot.get_figure() -fig.savefig('obsense_dry.png',bbox_inches='tight') - -fig.clear() - -o_moist=alltheobtypes['osense_moist'].sort_values(ascending=False) -moist_plot=o_moist.plot.barh(title = '2017091000 mean obs sensitivity, moist', figsize=figuresize); -fig=moist_plot.get_figure() -fig.savefig('obsense_moist.png',bbox_inches='tight') - -fig.clear() - -o_kin=alltheobtypes['osense_kin'].sort_values(ascending=False) -kin_plot=o_kin.plot.barh(title = '2017091000 mean obs sensitivity, kinetic', figsize=figuresize); -fig=kin_plot.get_figure() -fig.savefig('obsense_kin.png',bbox_inches='tight') - -fig.clear() - - - diff --git a/src/EFSOI_Utilities/scripts/geoplot.py b/src/EFSOI_Utilities/scripts/geoplot.py deleted file mode 100644 index 54ddca36..00000000 --- a/src/EFSOI_Utilities/scripts/geoplot.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Thu Jul 7 14:23:41 2022 - -@author: mossland -""" -# Default imports -import numpy as np -import matplotlib.pyplot as plt -import cartopy.crs as ccrs -import pandas as pd -import pickle -import datetime -import os.path -import matplotlib.colors as colors -import matplotlib.cm as cmx - -CDATE = '2021121000' -exp='rodeo' -exp='baily' - -#datadir='/work2/noaa/stmp/aeichman/comrot/' + exp + '/osense/' -#datadir='/work/noaa/stmp/aeichman/comrot/' + exp + '/osense/' -rootdir = '/Users/mossland/Desktop/work/EFSOI/' + exp -indir = rootdir + '/consolidated/' - -filename = indir + 'osense_' + CDATE + '.pkl' -infile = open( filename ,'rb') -[ idata, osensedata ] = pickle.load(infile) -infile.close() - - -# drop message='NA'/message02='Empty' -# data not dumped/not used -indices=osensedata[osensedata['assimilated']==0].index -osensedata.drop(indices,inplace = True) - -source = 'Radiosonde' -source = 'AMSUA' -source = 'Ocean_Surface' -source = 'Aircraft' - -for source in osensedata.source.unique(): -#for source in ['Radiosonde','AMSUA','Ocean_Surface','Aircraft']: - - - source = str( source ) - print('doing ' + source ) - - lon = osensedata.loc[osensedata['source'] == source ]['lon'] - - lon = osensedata.loc[osensedata['source'] == source ]['lon'] - lat = osensedata.loc[osensedata['source'] == source ]['lat'] - vals = osensedata.loc[osensedata['source'] == source ]['osense_moist'] - -# vmin = vals.min() -# vmax = vals.max() -# maxval = max(abs(vmin),vmax) -# vmin=-maxval/10 -# vmax=maxval/10 - valmean = vals.mean() - valsstd = vals.std() -# vmin = valmean - valsstd -# vmax = valmean + valsstd - vmin = -valsstd - vmax = valsstd - - - print('vmin, vmax: ' , vmin,vmax) - rnbw = cm = plt.get_cmap('rainbow') - rnbw = cm = plt.get_cmap('RdBu') - rnbw = cm = plt.get_cmap('coolwarm') - cNorm = colors.Normalize(vmin=vmin, vmax=vmax) - scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=rnbw) - scalarMap.set_array(vals) - - colorVal = scalarMap.to_rgba(vals) - - title = source + ' ' + CDATE +', assimilated' - - - fig = plt.figure(figsize=(12,9)) - ax = fig.add_subplot(1, 1, 1, - projection=ccrs.PlateCarree(), - title=title) -# ax.set_extent([0, 360, -90, 90]) - ax.set_extent([-180, 180, -90, 90 ]) - ax.scatter(lon, lat,c=colorVal, marker='.',s=10) - #ax.set_title(title) - ax.coastlines() - ax.gridlines() - - - diff --git a/src/EFSOI_Utilities/scripts/getefsiofiles.sh b/src/EFSOI_Utilities/scripts/getefsiofiles.sh deleted file mode 100755 index 3c41b772..00000000 --- a/src/EFSOI_Utilities/scripts/getefsiofiles.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/sh -x - - -# Author: Andrew Eichmann -# Purpose: to obtain from the HPSS archive the minimal set of files needed to -# run the gdaseupd task of GFS workflow, intended for EFSOI processing -# Script assumes that these files have been archived during a run; this will -# probably require modifications to the archiving script in workflow. - -# This will have to be changed to handle nc4 files - -EXP=nov2019c -#EDATE=2019111800 -#EDATE=2019111818 -EDATE=2019111712 -NMEM_ENKF=20 # total number of ensemble members -#NMEM_ENKF=0 -NMEM_EARCGRP=10 # number of ensemble members in archiving group -NDATE=/scratch1/NCEPDEV/global/Fanglin.Yang/save/VRFY/vsdb/nwprod/util/exec/ndate - -PDATE=`${NDATE} -6 ${EDATE}` -VDATE=`${NDATE} +24 ${EDATE}` - -EDATECYC="${EDATE:8:2}" -PDATECYC="${PDATE:8:2}" -VDATECYC="${VDATE:8:2}" - -EDATEDIR=enkfgdas."${EDATE:0:8}" -PDATEDIR=enkfgdas."${PDATE:0:8}" -VDATEDIR=enkfgdas."${VDATE:0:8}" - - -echo $PDATE -echo $VDATE -echo $EDATECYC -echo $EDATEDIR - -tarfile=enkfgdas.tar - - -hpssdir=/1year/NCEPDEV/emc-global/Andrew.Eichmann/HERA/scratch/${EXP}/${EDATE} - -hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/gdas.t${EDATECYC}z.cnvstat.ensmean -hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/gdas.t${EDATECYC}z.oznstat.ensmean -hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/gdas.t${EDATECYC}z.radstat.ensmean -hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/gdas.t${EDATECYC}z.abias_int.ensmean - -hpssdir=/1year/NCEPDEV/emc-global/Andrew.Eichmann/HERA/scratch/${EXP}/${PDATE} - -hpsstar get ${hpssdir}/${tarfile} ./${PDATEDIR}/${PDATECYC}/gdas.t${PDATECYC}z.atmf006.ensmean.nemsio - - - -imem=1 -while [[ $imem -le $NMEM_ENKF ]]; do - grpnum=`echo "( ($imem - 1) / ${NMEM_EARCGRP} ) + 1" | bc` - group="grp"`printf %02i $grpnum` - member="mem"`printf %03i $imem` - - echo $member - echo $group - - tarfile=enkfgdas_${group}.tar - hpssdir=/1year/NCEPDEV/emc-global/Andrew.Eichmann/HERA/scratch/${EXP}/${EDATE} - hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/${member}/gdas.t${EDATECYC}z.cnvstat - hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/${member}/gdas.t${EDATECYC}z.oznstat - hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/${member}/gdas.t${EDATECYC}z.radstat - hpsstar get ${hpssdir}/${tarfile} ./${EDATEDIR}/${EDATECYC}/${member}/gdas.t${EDATECYC}z.atmanl.nemsio - - hpssdir=/1year/NCEPDEV/emc-global/Andrew.Eichmann/HERA/scratch/${EXP}/${PDATE} - hpsstar get ${hpssdir}/${tarfile} ./${PDATEDIR}/${PDATECYC}/${member}/gdas.t${PDATECYC}z.atmf006s.nemsio - - (( imem = $imem + 1 )) -done - - - diff --git a/src/EFSOI_Utilities/scripts/osense2nc.py b/src/EFSOI_Utilities/scripts/osense2nc.py deleted file mode 100644 index 22e08ece..00000000 --- a/src/EFSOI_Utilities/scripts/osense2nc.py +++ /dev/null @@ -1,88 +0,0 @@ -from netCDF4 import Dataset # Note: python is case-sensitive! -import netCDF4 as nc -import numpy as np -import osense - -# this is determined in the observation sensitivty module in the enkf code - better would -# be to set it dynamically from the the maximum string length -strlen=20 -#osensefile='/scratch1/NCEPDEV/stmp4/Andrew.Eichmann/testtest/osense/osense_2019111918_final.dat' -dir='/work2/noaa/stmp/aeichman/comrot/EFOSIsandbox/osense/' -filename='osense_2020122418_final.dat' -osensefile=dir+filename - -columns = [ ( 'obfit_prior' , 'Observation fit to the first guess'), - ( 'obsprd_prior' , 'Spread of observation prior'), - ( 'ensmean_obnobc' , 'Ensemble mean first guess (no bias correction)'), - ( 'ensmean_ob' , 'Ensemble mean first guess (bias corrected)'), - ( 'ob' , 'Observation value'), - ( 'oberrvar' , 'Observation error variance'), - ( 'lon' , 'Longitude'), - ( 'lat' , 'Latitude'), - ( 'pres' , 'Pressure'), - ( 'time' , 'Observation time'), - ( 'oberrvar_orig' , 'Original error variance'), - ( 'osense_kin' , 'Observation sensitivity (kinetic energy) [J/kg]'), - ( 'osense_dry' , 'Observation sensitivity (Dry total energy) [J/kg]'), - ( 'osense_moist' , 'Observation sensitivity (Moist total energy) [J/kg]') ] - -(convdata, satdata, idate )= osense.read_osense( osensefile) - - - - -def fill_nc( dataset, ncfilname ): - - - obnum_in = dataset.shape[0] - try: ncfile.close() # just to be safe, make sure dataset is not already open. - except: pass - ncfile = Dataset(ncfilename,mode='w',format='NETCDF4') - - _ = ncfile.createDimension('nobs', obnum_in) - _ = ncfile.createDimension('nchars',strlen) - - for dim in ncfile.dimensions.items(): - print(dim) - - nobs = ncfile.createVariable('nobs', np.int32, ('nobs',)) - nobs.long_name = 'number of observations' - nobs[:] = list(range(1,obnum_in+1)) - - # netcdf isn't crazy about strings as data, so there's this - obtype = ncfile.createVariable('obtype', 'S1', ('nobs','nchars')) - obtypestr=np.array(dataset[ 'obtype' ],dtype='S20') - obtype[:] = nc.stringtochar(obtypestr ) - obtype.long_name = 'Observation element / Platform, instrument ' - - stattype = ncfile.createVariable('stattype', np.int32, ('nobs')) - stattype.long_name = 'Conventional PREPBUFR code/Observation type' - stattype[:] = dataset[ 'stattype' ].to_numpy() - - indxsat = ncfile.createVariable('indxsat', np.int32, ('nobs')) - indxsat.long_name = 'Satellite index (channel) set to zero' - indxsat[:] = dataset[ 'indxsat' ].to_numpy() - - for column in columns: - - varname = column[0] - ncvar = ncfile.createVariable(varname, np.float32, ('nobs')) - ncvar.long_name = column[1] - ncvar[:] = dataset[ varname ].to_numpy() - - return ncfile - - -ncfilename = 'osense_' + str(idate) + '_sat.nc' -ncfile = fill_nc( satdata, ncfilename ) -ncfile.title='My satellite osense data' -print(ncfile) -ncfile.close(); print('Dataset is closed!') - -ncfilename = 'osense_' + str(idate) + '_conv.nc' -ncfile = fill_nc( convdata, ncfilename ) -ncfile.title='My conventional and ozone osense data' -print(ncfile) -ncfile.close(); print('Dataset is closed!') - -