Skip to content

Commit

Permalink
Merge branch 'viirs_sst' into develop
Browse files Browse the repository at this point in the history 
Conflicts:
	py/polar2grid/polar2grid/awips/awips_grids.conf
	py/polar2grid_core/polar2grid/core/rescale_configs/rescale.8bit.conf
  • Loading branch information
@djhoese
djhoese committed Mar 11, 2014
2 parents a1b113b + ec34bde commit e2e83e7
Show file tree
Hide file tree
Showing 8 changed files with 87 additions and 20 deletions.
6 changes: 6 additions & 0 deletions py/polar2grid/polar2grid/awips/awips_grids.conf
View file
Expand Up @@ -13,6 +13,8 @@ npp, viirs, i_nav, i_enh, 05, btemp_enh, 211e,55779603,11.0 um,
npp, viirs, dnb_nav, dnb, none, radiance, 211e,55779604,DNB,SSEC,NPP-VIIRS,grid211e.ncml,SSEC_AWIPS_VIIRS-ECONUS_1KM_SVDNB_%Y%m%d_%H%M.55779604
npp, viirs, dnb_nav, dnb, new, radiance, 211e,55779606,DNB,SSEC,NPP-VIIRS,grid211e.ncml,SSEC_AWIPS_VIIRS-ECONUS_1KM_SVDNB_TRANS_%Y%m%d_%H%M.55779606
npp, viirs, i_nav, i, fog, fog, 211e,55779605,Fog,SSEC,NPP-VIIRS,grid211e.ncml,SSEC_AWIPS_VIIRS-ECONUS_1KM_SVIFOG_%Y%m%d_%H%M.55779605
# VIIRS SST
npp, viirs, m_nav, sea_surface_temp, None, btemp, 211e,7370,SST,SSEC,MODIS,grid211e.ncml,SSEC_AWIPS_MODIS-EAST_1KM_SST_%Y%m%d_%H%M.7370
# MODIS bands for grid 211e
# Aqua MODIS
aqua, modis, geo_250m_nav, visible, 01, reflectance, 211e, 7340, 0.64 um, SSEC, MODIS, grid211e.ncml, SSEC_AWIPS_MODIS_EAST_1KM_Visible_%Y%m%d_%H%M.7340
Expand Down Expand Up @@ -81,6 +83,7 @@ npp, viirs, dnb_nav, dnb, none, radiance, 211w,55779612,DNB,SSEC
npp, viirs, dnb_nav, dnb, new, radiance, 211w,55779614,DNB,SSEC,NPP-VIIRS,grid211w.ncml,SSEC_AWIPS_VIIRS-WCONUS_1KM_SVDNB_TRANS_%Y%m%d_%H%M.55779614
npp, viirs, i_nav, i, fog, fog, 211w,55779613,Fog,SSEC,NPP-VIIRS,grid211w.ncml,SSEC_AWIPS_VIIRS-WCONUS_1KM_SVIFOG_%Y%m%d_%H%M.55779613

npp, viirs, m_nav, sea_surface_temp, None, btemp, 211w,7371,SST,SSEC,MODIS,grid211w.ncml,SSEC_AWIPS_MODIS_WEST_1KM_SST_%Y%m%d_%H%M.7371
# MODIS bands for this grid
# Aqua MODIS
aqua, modis, geo_250m_nav, visible, 01, reflectance, 211w, 7350, 0.64 um, SSEC, MODIS, grid211w.ncml, SSEC_AWIPS_MODIS_WEST_1KM_Visible_%Y%m%d_%H%M.7350
Expand Down Expand Up @@ -140,6 +143,8 @@ npp, viirs, i_nav, i_enh, 05, btemp_enh, 203,55779619,11.0 um,S
npp, viirs, dnb_nav, dnb, none, radiance, 203,55779620,DNB,SSEC,NPP-VIIRS,grid203.ncml,SSEC_AWIPS_VIIRS-AK_1KM_SVDNB_%Y%m%d_%H%M.55779620
npp, viirs, dnb_nav, dnb, new, radiance, 203,55779622,DNB,SSEC,NPP-VIIRS,grid203.ncml,SSEC_AWIPS_VIIRS-AK_1KM_SVDNB_TRANS_%Y%m%d_%H%M.55779622
npp, viirs, i_nav, i, fog, fog, 203,55779621,Fog,SSEC,NPP-VIIRS,grid203.ncml,SSEC_AWIPS_VIIRS-AK_1KM_SVIFOG_%Y%m%d_%H%M.55779621
# Not Valid - There is no MODIS SST for grid 203 to mimic:
npp, viirs, m_nav, sea_surface_temp, None, btemp, 203,7372,SST,SSEC,MODIS,grid203.ncml,SSEC_AWIPS_MODIS_AK_1KM_SST_%Y%m%d_%H%M.7372

#
### Grid 204 - Hawaii ###
Expand All @@ -153,6 +158,7 @@ npp, viirs, i_nav, i_enh, 05, btemp_enh, 204,55779627,11.0 um,S
npp, viirs, dnb_nav, dnb, none, radiance, 204,55779628,DNB,SSEC,NPP-VIIRS,grid204.ncml,SSEC_AWIPS_VIIRS-HI_1KM_SVDNB_%Y%m%d_%H%M.55779628
npp, viirs, dnb_nav, dnb, new, radiance, 204,55779630,DNB,SSEC,NPP-VIIRS,grid204.ncml,SSEC_AWIPS_VIIRS-HI_1KM_SVDNB_TRANS_%Y%m%d_%H%M.55779630
npp, viirs, i_nav, i, fog, fog, 204,55779629,Fog,SSEC,NPP-VIIRS,grid204.ncml,SSEC_AWIPS_VIIRS-HI_1KM_SVIFOG_%Y%m%d_%H%M.55779629
npp, viirs, m_nav, sea_surface_temp, None, btemp, 204,55777371,SST,SSEC,MODIS,grid204.ncml,SSEC_AWIPS_MODIS_HI_1KM_SST_%Y%m%d_%H%M.55777371

# Aqua MODIS
aqua, modis, geo_250m_nav, visible, 01, reflectance, 204, 55777350, 0.64 um, SSEC, MODIS, grid204.ncml, SSEC_AWIPS_MODIS_HI_1KM_Visible_%Y%m%d_%H%M.55777350
Expand Down
4 changes: 2 additions & 2 deletions py/polar2grid_core/polar2grid/core/constants.py
View file
Expand Up @@ -113,12 +113,12 @@
BID_14 = "14"
BID_15 = "15"
BID_16 = "16"
BID_FOG = "fog"
BID_NEW = "new"
BID_20 = "20"
BID_26 = "26"
BID_27 = "27"
BID_31 = "31"
BID_FOG = "fog"
BID_NEW = "new"

# Data kinds
DKIND_LATITUDE = "latitude"
Expand Down
2 changes: 2 additions & 0 deletions py/polar2grid_core/polar2grid/core/rescale_configs/rescale.16bit.conf
View file
Expand Up @@ -28,6 +28,8 @@ npp, viirs, m_nav, m, 13, btemp, btemp, 242.0,
npp, viirs, m_nav, m, 14, btemp, btemp, 242.0, 169620, 514, 107426, 257,0,65535
npp, viirs, m_nav, m, 15, btemp, btemp, 242.0, 169620, 514, 107426, 257,0,65535
npp, viirs, m_nav, m, 16, btemp, btemp, 242.0, 169620, 514, 107426, 257,0,65535
# SST
npp, viirs, m_nav, sea_surface_temp, None, btemp, linear, 1542.0, -421162.3

# modis related data; most of these handle similarly to viirs
# aqua data
Expand Down
14 changes: 8 additions & 6 deletions py/polar2grid_core/polar2grid/core/rescale_configs/rescale.8bit.conf
View file
Expand Up @@ -25,11 +25,13 @@ npp, viirs, m_nav, m, 08, reflectance, sqrt, 100.0, 25
npp, viirs, m_nav, m, 09, reflectance, sqrt, 100.0, 25.5
npp, viirs, m_nav, m, 10, reflectance, sqrt, 100.0, 25.5
npp, viirs, m_nav, m, 11, reflectance, sqrt, 100.0, 25.5
npp, viirs, m_nav, m, 12, btemp, btemp, 242.0, 660, 2, 418, 1
npp, viirs, m_nav, m, 13, btemp, btemp, 242.0, 660, 2, 418, 1
npp, viirs, m_nav, m, 14, btemp, btemp, 242.0, 660, 2, 418, 1
npp, viirs, m_nav, m, 15, btemp, btemp, 242.0, 660, 2, 418, 1
npp, viirs, m_nav, m, 16, btemp, btemp, 242.0, 660, 2, 418, 1
npp, viirs, m_nav, m, 12, btemp, btemp, 242.0, 660, 2, 418, 1,0,255
npp, viirs, m_nav, m, 13, btemp, btemp, 242.0, 660, 2, 418, 1,0,255
npp, viirs, m_nav, m, 14, btemp, btemp, 242.0, 660, 2, 418, 1,0,255
npp, viirs, m_nav, m, 15, btemp, btemp, 242.0, 660, 2, 418, 1,0,255
npp, viirs, m_nav, m, 16, btemp, btemp, 242.0, 660, 2, 418, 1,0,255
# SST
npp, viirs, m_nav, sea_surface_temp, None, btemp, linear, 6.0, -1603.9

# modis related data; most of these handle similarly to viirs
# aqua data
Expand Down Expand Up @@ -77,7 +79,7 @@ terra, modis, mod06_nav, cloud_top_temperature, none, btemp,
terra, modis, mod07_nav, total_precipitable_water, none, distance, linear, 30.0, 40.0

# TODO, this is guesswork
terra, modis, geo_1000m_nav, inversion_strength, none, btemp, btemp_c, 242.0, 660, 2, 418, 1
terra, modis, geo_1000m_nav, inversion_strength, none, btemp, btemp_c, 242.0, 660, 2, 418, 1,0,255
terra, modis, geo_1000m_nav, inversion_depth, none, distance, raw
terra, modis, geo_1000m_nav, ice_concentration, none, percent, raw

Expand Down
2 changes: 2 additions & 0 deletions py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.16bit.conf
View file
Expand Up @@ -28,6 +28,8 @@ npp, viirs, m_nav, m, 13, btemp, btemp, 242.0,
npp, viirs, m_nav, m, 14, btemp, btemp, 242.0, 169620, 514, 107422.854, 256.987,0,65534
npp, viirs, m_nav, m, 15, btemp, btemp, 242.0, 169620, 514, 107422.854, 256.987,0,65534
npp, viirs, m_nav, m, 16, btemp, btemp, 242.0, 169620, 514, 107422.854, 256.987,0,65534
# SST
npp, viirs, m_nav, sea_surface_temp, None, btemp, linear, 1542.0, -421162.3

# modis related data; most of these handle similarly to viirs
# aqua data
Expand Down
2 changes: 2 additions & 0 deletions py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.8bit.conf
View file
Expand Up @@ -28,6 +28,8 @@ npp, viirs, m_nav, m, 13, btemp, btemp, 242.0,660
npp, viirs, m_nav, m, 14, btemp, btemp, 242.0,660,2,414.854,0.987,0,254
npp, viirs, m_nav, m, 15, btemp, btemp, 242.0,660,2,414.854,0.987,0,254
npp, viirs, m_nav, m, 16, btemp, btemp, 242.0,660,2,414.854,0.987,0,254
# SST
npp, viirs, m_nav, sea_surface_temp, None, btemp, linear, 6.0, -1603.9

# modis related data; most of these handle similarly to viirs
# aqua data
Expand Down
75 changes: 64 additions & 11 deletions py/polar2grid_viirs/polar2grid/viirs/viirs_guidebook.py
View file
Expand Up @@ -68,13 +68,15 @@
K_RADIANCE_FACTORS = "RadianceFactorsVar"
K_REFLECTANCE_FACTORS = "ReflectanceFactorsVar"
K_BTEMP_FACTORS = "BrightnessTemperatureFactorsVar"
K_SST_FACTORS = "SeaSurfaceTemperatureFactorsVar"
K_STARTTIME = "StartTimeVar"
K_AGGR_STARTTIME = "AggrStartTimeVar"
K_AGGR_STARTDATE = "AggrStartDateVar"
K_AGGR_ENDTIME = "AggrEndTimeVar"
K_AGGR_ENDDATE = "AggrEndDateVar"
K_MODESCAN = "ModeScanVar"
K_MODEGRAN = "ModeGranVar"
K_QF1 = "QF1Var"
K_QF3 = "QF3Var"
K_LAT_G_RING = "LatGRingAttr"
K_LON_G_RING = "LonGRingAttr"
Expand All @@ -89,20 +91,25 @@
BKIND_M : MBAND_NAV_UID,
BKIND_I : IBAND_NAV_UID,
BKIND_DNB : DNB_NAV_UID,
BKIND_SST : MBAND_NAV_UID,
}

# Non-TC vs Terrain Corrected
GEO_GUIDE = {
BKIND_M : ('GMODO','GMTCO'),
BKIND_I : ('GIMGO','GITCO'),
BKIND_DNB : ('GDNBO','GDNBO'),
BKIND_SST : ('GMODO','GMTCO'),
}

FACTORS_GUIDE = {
DKIND_REFLECTANCE : K_REFLECTANCE_FACTORS,
DKIND_RADIANCE : K_RADIANCE_FACTORS,
DKIND_BTEMP : K_BTEMP_FACTORS
}
(BKIND_M, DKIND_REFLECTANCE) : K_REFLECTANCE_FACTORS,
(BKIND_I, DKIND_REFLECTANCE) : K_REFLECTANCE_FACTORS,
(BKIND_DNB, DKIND_RADIANCE) : K_RADIANCE_FACTORS,
(BKIND_M, DKIND_BTEMP) : K_BTEMP_FACTORS,
(BKIND_I, DKIND_BTEMP) : K_BTEMP_FACTORS,
(BKIND_SST, DKIND_BTEMP) : K_SST_FACTORS,
}

GEO_FILE_GUIDE = {
r'GITCO.*' : {
Expand Down Expand Up @@ -214,6 +221,20 @@
K_QF3: '/All_Data/VIIRS-%(file_kind)s%(int(file_band))d-SDR_All/QF3_SCAN_RDR',
K_GEO_REF: r'%(GEO_GUIDE[kind])s_%(sat)s_d%(date)s_t%(file_start_time_str)s_e%(file_end_time_str)s_b%(orbit)s_*_%(site)s_%(domain)s.h5',
K_NAVIGATION: r'%%(geo_kind)s_%(sat)s_d%(date)s_t%(file_start_time_str)s_e%(file_end_time_str)s_b%(orbit)s_*_%(site)s_%(domain)s.h5' },
r'V(?P<file_kind>SST)O.*': {
K_RADIANCE: None,
K_REFLECTANCE: None,
K_BTEMP: '/All_Data/VIIRS-SST-EDR_All/SkinSST',
K_SST_FACTORS: '/All_Data/VIIRS-SST-EDR_All/SkinSSTFactors',
K_RADIANCE_FACTORS: None,
K_REFLECTANCE_FACTORS: None,
K_BTEMP_FACTORS: None,
K_MODESCAN: None,
K_MODEGRAN: None,
K_QF3: '/All_Data/VIIRS-SST-EDR_All/QF3_VIIRSSSTEDR',
K_QF1: '/All_Data/VIIRS-SST-EDR_All/QF1_VIIRSSSTEDR',
K_GEO_REF: r'%(GEO_GUIDE[kind])s_%(sat)s_d%(date)s_t%(file_start_time_str)s_e%(file_end_time_str)s_b%(orbit)s_*_%(site)s_%(domain)s.h5',
K_NAVIGATION: r'%%(geo_kind)s_%(sat)s_d%(date)s_t%(file_start_time_str)s_e%(file_end_time_str)s_b%(orbit)s_*_%(site)s_%(domain)s.h5' },
r'SVDNB.*': { K_RADIANCE: '/All_Data/VIIRS-DNB-SDR_All/Radiance',
K_REFLECTANCE: None,
K_BTEMP: None,
Expand All @@ -235,13 +256,15 @@
"GITCO" : 32,
"GIMGO" : 32,
BKIND_DNB : 16,
"GDNBO" : 16
"GDNBO" : 16,
BKIND_SST : 16,
}

COLS_PER_ROW = {
BKIND_M : 3200,
BKIND_I : 6400,
BKIND_DNB : 4064
BKIND_DNB : 4064,
BKIND_SST : 3200,
}

DATA_KINDS = {
Expand All @@ -267,7 +290,8 @@
(BKIND_I,BID_04) : DKIND_BTEMP,
(BKIND_I,BID_05) : DKIND_BTEMP,
(BKIND_I,BID_06) : DKIND_REFLECTANCE,
(BKIND_DNB,NOT_APPLICABLE) : DKIND_RADIANCE
(BKIND_DNB,NOT_APPLICABLE) : DKIND_RADIANCE,
(BKIND_SST,NOT_APPLICABLE) : DKIND_BTEMP,
}

ENHANCED_IR_BAND_KIND = {
Expand Down Expand Up @@ -302,7 +326,8 @@
(BKIND_I,BID_04) : K_BTEMP,
(BKIND_I,BID_05) : K_BTEMP,
(BKIND_I,BID_06) : K_REFLECTANCE,
(BKIND_DNB,NOT_APPLICABLE) : K_RADIANCE
(BKIND_DNB,NOT_APPLICABLE) : K_RADIANCE,
(BKIND_SST,NOT_APPLICABLE) : K_BTEMP,
}

band2const = {
Expand Down Expand Up @@ -424,6 +449,7 @@ def sort_files_by_nav_uid(filepaths):

for fp in filepaths:
fn = os.path.split(fp)[-1]
# FIXME: This sucks:
if fn.startswith("SVI") and fn.endswith(".h5"):
nav_uid = NAV_SET_GUIDE[BKIND_I]
if fn[3:5] in nav_dict[nav_uid]:
Expand All @@ -438,6 +464,9 @@ def sort_files_by_nav_uid(filepaths):
nav_uid = NAV_SET_GUIDE[BKIND_DNB]
nav_dict[nav_uid][NOT_APPLICABLE].append(fp)
continue
if fn.startswith("VSSTO") and fn.endswith(".h5"):
nav_uid = NAV_SET_GUIDE[BKIND_SST]
nav_dict[nav_uid][NOT_APPLICABLE].append(fp)

# Ignore the file that we don't understand

Expand All @@ -464,6 +493,7 @@ def sort_files_by_nav_uid(filepaths):
del nav_dict[nav_uid]
continue

print nav_dict
return nav_dict


Expand Down Expand Up @@ -595,6 +625,9 @@ def file_info(fn):
minfo["kind"] = BKIND_M
elif minfo["file_kind"] == "I":
minfo["kind"] = BKIND_I
elif minfo["file_kind"] == "SST":
minfo["kind"] = BKIND_SST
minfo["band"] = NOT_APPLICABLE
else:
log.error("Band kind not known '%s'" % minfo["kind"])
raise ValueError("Band kind not known '%s'" % minfo["kind"])
Expand All @@ -618,7 +651,7 @@ def file_info(fn):
finfo["data_kind"] = DATA_KINDS[dkind]
finfo["rows_per_scan"] = ROWS_PER_SCAN[finfo["kind"]]
finfo["cols_per_row"] = COLS_PER_ROW[finfo["kind"]]
finfo["factors"] = FACTORS_GUIDE[finfo["data_kind"]]
finfo["factors"] = FACTORS_GUIDE[(finfo["kind"], finfo["data_kind"])]

# Convert time information to datetime objects
get_datetimes(finfo)
Expand All @@ -630,6 +663,7 @@ def file_info(fn):

# Geonav file exists
file_glob = finfo[K_NAVIGATION] % {"geo_kind":GEO_GUIDE[finfo["kind"]][1]}
print finfo["kind"], GEO_GUIDE[finfo["kind"]], file_glob, fn
geo_glob = os.path.join(base_path, file_glob)
try:
# First try terrain corrected
Expand Down Expand Up @@ -697,6 +731,23 @@ def read_file_info(finfo, extra_mask=None, fill_value=-999, dtype=np.float32):
qf3_data = h5v[:]
del h5v

# Get QF1 data for VIIRS SST
# qf1_mask is True if the data is poor quality
qf1_mask = None
if K_QF1 in finfo:
log.info("Extracting QF1 flag for data quality checks...")
qf1_var_path = finfo[K_QF1]
h5v = h5path(hp, qf1_var_path, finfo["img_path"], required=False)
if h5v is None:
log.info("QF1 data not found in file")
else:
# 00 = Not retrieved
# 01 = Excluded
# 10 = Degraded
# 11 = High Quality
qf1_mask = (h5v[:, :] & 3) < 2
del h5v

# Get scaling function (also reads scaling factors from hdf)
factvar = h5path(hp, factors_var_path, finfo["img_path"], required=False) # FUTURE: make this more elegant please
if factvar is None:
Expand Down Expand Up @@ -734,11 +785,13 @@ def scaler(image_data, scaling_factors=factor_list):
# Calculate mask
mask = MISSING_GUIDE[data_kind][not needs_scaling](image_data) if data_kind in MISSING_GUIDE else None
if extra_mask is not None:
mask = mask | extra_mask
mask |= extra_mask
if qf1_mask is not None:
mask |= qf1_mask

# Scale image data
image_data,scaling_mask = scaler(image_data)
mask = mask | scaling_mask
mask |= scaling_mask

# Create scan_quality array
scan_quality = np.nonzero(np.repeat(qf3_data > 0, finfo["rows_per_scan"]))
Expand Down
2 changes: 1 addition & 1 deletion py/polar2grid_viirs/polar2grid/viirs/viirs_imager_to_swath.py
View file
Expand Up @@ -613,7 +613,7 @@ def parse_datetimes_from_filepaths(cls, filepaths):
filenames = [ os.path.split(fp)[-1] for fp in filepaths ]

# Clear out files we don't understand
filenames = [ fn for fn in filenames if fn.startswith("SV") and fn.endswith(".h5") ]
filenames = [ fn for fn in filenames if (fn.startswith("SV") or fn.startswith("VSST")) and fn.endswith(".h5") ]

# SVI01_npp_d20120225_t1801245_e1802487_b01708_c20120226002130255476_noaa_ops.h5
dt_list = [ datetime.strptime(fn[10:27], "d%Y%m%d_t%H%M%S") for
Expand Down

0 comments on commit e2e83e7

Please sign in to comment.