Skip to content

Commit

Permalink
configuring new nav groups for geocat2awips and setting up all the in…
Browse files Browse the repository at this point in the history 
…itial grid and scaling configurations
  • Loading branch information
@evas-ssec
evas-ssec committed Oct 18, 2013
1 parent 33d37ed commit 1d68c59
Show file tree
Hide file tree
Showing 6 changed files with 227 additions and 71 deletions.
112 changes: 78 additions & 34 deletions py/polar2grid/polar2grid/awips/awips_grids.conf
View file

Large diffs are not rendered by default.

7 changes: 4 additions & 3 deletions py/polar2grid/polar2grid/geocat2awips.py
View file
Expand Up @@ -18,7 +18,7 @@
from polar2grid.core.glue_utils import *
from polar2grid.core.time_utils import utc_now
from .grids.grids import create_grid_jobs, Cartographer
from polar2grid.modis import FILE_CONTENTS_GUIDE
from polar2grid.modis import FILE_CONTENTS_GUIDE, find_nav_uid_from_filename
from polar2grid.modis import Geo_Frontend
from .awips import Backend
import remap
Expand Down Expand Up @@ -253,17 +253,18 @@ def run_glue(filepaths,
for filepath in all_used:
log.debug("Processing file %s" % filepath)
filename = os.path.split(filepath)[1]
nav_uid = find_nav_uid_from_filename(filename)

try:
if multiprocess:
temp_processes = Process(target=_process_data_sets,
args = ([filepath], "geo_nav"), # TODO, this is the wrong way to match file and nav_uid
args = ([filepath], nav_uid),
kwargs = kwargs
)
temp_processes.start()
processes_to_wait_for.append(temp_processes)
else:
stat = _process_data_sets([filepath], "geo_nav", **kwargs) # TODO, this is the wrong way to match file and nav_uid
stat = _process_data_sets([filepath], nav_uid, **kwargs)
exit_status = exit_status or stat
except StandardError:
log.error("Could not process file %s" % filepath, exc_info=True)
Expand Down
3 changes: 3 additions & 0 deletions py/polar2grid_core/polar2grid/core/constants.py
View file
Expand Up @@ -67,6 +67,7 @@
SAT_TERRA = "terra"
SAT_AQUA = "aqua"
SAT_GOES13 = "goes13"
SAT_GOES14 = "goes14"
SAT_GOES15 = "goes15"
SAT_METEO9 = "meteosat9"
SAT_MTSAT2 = "mtsat2"
Expand Down Expand Up @@ -180,6 +181,8 @@
GEO_250M_NAV_UID = "geo_250m_nav" # the 250m navigation group
MOD06_NAV_UID = "mod06_nav" # the mod06 navigation group
MOD07_NAV_UID = "mod07_nav" # the mod07 navigation group
AK_NAV_UID = "ak_nav" # the Alaska navigation group
CONUS_NAV_UID = "conus_nav" # the CONUS navigation group

# Grid Constants
GRIDS_ANY = "any_grid"
Expand Down
87 changes: 63 additions & 24 deletions py/polar2grid_core/polar2grid/core/rescale_configs/rescale.8bit.conf
View file
Expand Up @@ -66,35 +66,74 @@ terra, modis, geo_nav, ndvi, none, contiguous_ind
terra, modis, mod06_nav, cloud_top_temperature, none, btemp, btemp_lin, 300.26, 173.16, 10, 250
terra, modis, mod07_nav, total_precipitable_water, none, distance, linear, 30.0, 40.0

# TODO, these are temporary entries for testing the geocat products
aqua, modis, geo_nav, ifr, fog, percent, linear, 2.55, 0.0
aqua, modis, geo_nav, lifr, fog, percent, linear, 2.55, 0.0
aqua, modis, geo_nav, mvfr, fog, percent, linear, 2.55, 0.0
aqua, modis, geo_nav, cloud_thickness, none, distance, linear, 0.17, 0.0
aqua, modis, geo_nav, cloud_phase, none, category, linear, 30.0, 30.0
aqua, modis, geo_nav, ash_height, none, distance, linear, 12.75, 0.0
aqua, modis, geo_nav, ash_mass_loading, none, mass_loading, linear, 5.1, 0.0
aqua, modis, geo_nav, ash_effective_radius, none, distance, linear, 15.875, 0.0
aqua, modis, geo_nav, ash_btd_11_12, none, btemp, linear, 25.5, 127.5
aqua, modis, geo_nav, ash_11, none, btemp, irw, 242.0, -2.0, 660.0, -1.0, 418.0
aqua, modis, geo_nav, ash_visible, none, reflectance, sqrt, 100.0, 2.55
aqua, modis, geo_nav, so2_loading, none, mass_loading, raw
aqua, modis, geo_nav, so2_mask, none, category, raw
# TODO more temporary entries for testing geocat products from goes
goes15, goes_imager, geo_nav, ifr, fog, percent, linear, 2.55, 0.0
goes15, goes_imager, geo_nav, lifr, fog, percent, linear, 2.55, 0.0
goes15, goes_imager, geo_nav, mvfr, fog, percent, linear, 2.55, 0.0
goes15, goes_imager, geo_nav, cloud_thickness, none, distance, linear, 0.17, 0.0
goes15, goes_imager, geo_nav, cloud_phase, none, category, linear, 30.0, 30.0
# TODO, this is guesswork
terra, modis, geo_nav, inversion_strength, none, btemp, btemp_c, 242.0, 660, 2, 418, 1
terra, modis, geo_nav, inversion_depth, none, distance, raw
terra, modis, geo_nav, ice_concentration, none, percent, raw

# Geocat product entries

# GOES-13 via GEOCAT
goes13, goes_imager, geo_nav, ifr, fog, percent, linear, 2.55, 0.0
goes13, goes_imager, geo_nav, lifr, fog, percent, linear, 2.55, 0.0
goes13, goes_imager, geo_nav, mvfr, fog, percent, linear, 2.55, 0.0
goes13, goes_imager, geo_nav, cloud_thickness, none, distance, linear, 0.17, 0.0
goes13, goes_imager, geo_nav, cloud_phase, none, category, linear, 30.0, 30.0

# GOES-14 via GEOCAT (this is currently configured to behave like GOES-13)
goes14, goes_imager, geo_nav, ifr, fog, percent, linear, 2.55, 0.0
goes14, goes_imager, geo_nav, lifr, fog, percent, linear, 2.55, 0.0
goes14, goes_imager, geo_nav, mvfr, fog, percent, linear, 2.55, 0.0
goes14, goes_imager, geo_nav, cloud_thickness, none, distance, linear, 0.17, 0.0
goes14, goes_imager, geo_nav, cloud_phase, none, category, linear, 30.0, 30.0

# GOES-15 via GEOCAT
goes15, goes_imager, geo_nav, ifr, fog, percent, linear, 2.55, 0.0
goes15, goes_imager, geo_nav, lifr, fog, percent, linear, 2.55, 0.0
goes15, goes_imager, geo_nav, mvfr, fog, percent, linear, 2.55, 0.0
goes15, goes_imager, geo_nav, cloud_thickness, none, distance, linear, 0.17, 0.0
goes15, goes_imager, geo_nav, cloud_phase, none, category, linear, 30.0, 30.0

# Aqua MODIS via Geocat
aqua, modis, ak_nav, ifr, fog, percent, linear, 2.55, 0.0
aqua, modis, ak_nav, lifr, fog, percent, linear, 2.55, 0.0
aqua, modis, ak_nav, mvfr, fog, percent, linear, 2.55, 0.0
aqua, modis, ak_nav, cloud_thickness, none, distance, linear, 0.17, 0.0
aqua, modis, ak_nav, cloud_phase, none, category, linear, 30.0, 30.0
aqua, modis, ak_nav, ash_height, none, distance, linear, 12.75, 0.0
aqua, modis, ak_nav, ash_mass_loading, none, mass_loading, linear, 5.1, 0.0
aqua, modis, ak_nav, ash_effective_radius, none, distance, linear, 15.875, 0.0
aqua, modis, ak_nav, ash_btd_11_12, none, btemp, linear, 25.5, 127.5
aqua, modis, ak_nav, ash_11, none, btemp, irw, 242.0, -2.0, 660.0, -1.0, 418.0
aqua, modis, ak_nav, ash_visible, none, reflectance, sqrt, 100.0, 2.55
aqua, modis, conus_nav, ifr, fog, percent, linear, 2.55, 0.0
aqua, modis, conus_nav, lifr, fog, percent, linear, 2.55, 0.0
aqua, modis, conus_nav, mvfr, fog, percent, linear, 2.55, 0.0
aqua, modis, conus_nav, cloud_thickness, none, distance, linear, 0.17, 0.0
aqua, modis, conus_nav, cloud_phase, none, category, linear, 30.0, 30.0

# Terra MODIS via Geocat
terra, modis, ak_nav, ifr, fog, percent, linear, 2.55, 0.0
terra, modis, ak_nav, lifr, fog, percent, linear, 2.55, 0.0
terra, modis, ak_nav, mvfr, fog, percent, linear, 2.55, 0.0
terra, modis, ak_nav, cloud_thickness, none, distance, linear, 0.17, 0.0
terra, modis, ak_nav, cloud_phase, none, category, linear, 30.0, 30.0
terra, modis, ak_nav, ash_height, none, distance, linear, 12.75, 0.0
terra, modis, ak_nav, ash_mass_loading, none, mass_loading, linear, 5.1, 0.0
terra, modis, ak_nav, ash_effective_radius, none, distance, linear, 15.875, 0.0
terra, modis, ak_nav, ash_btd_11_12, none, btemp, linear, 25.5, 127.5
terra, modis, ak_nav, ash_11, none, btemp, irw, 242.0, -2.0, 660.0, -1.0, 418.0
terra, modis, ak_nav, ash_visible, none, reflectance, sqrt, 100.0, 2.55
terra, modis, conus_nav, ifr, fog, percent, linear, 2.55, 0.0
terra, modis, conus_nav, lifr, fog, percent, linear, 2.55, 0.0
terra, modis, conus_nav, mvfr, fog, percent, linear, 2.55, 0.0
terra, modis, conus_nav, cloud_thickness, none, distance, linear, 0.17, 0.0
terra, modis, conus_nav, cloud_phase, none, category, linear, 30.0, 30.0

# TODO, these are guesswork
#aqua, modis, geo_nav, so2_loading, none, mass_loading, raw
#aqua, modis, geo_nav, so2_mask, none, category, raw



# TODO, this is guesswork
terra, modis, geo_nav, inversion_strength, none, btemp, btemp_c, 242.0, 660, 2, 418, 1
terra, modis, geo_nav, inversion_depth, none, distance, raw
terra, modis, geo_nav, ice_concentration, none, percent, raw

2 changes: 1 addition & 1 deletion py/polar2grid_modis/polar2grid/modis/__init__.py
View file
Expand Up @@ -45,5 +45,5 @@
from .modis_to_swath import Frontend
from .modis_guidebook import GEO_FILE_GROUPING, parse_datetime_from_filename
from .geocat_to_swath import Frontend as Geo_Frontend
from .geocat_guidebook import FILE_CONTENTS_GUIDE
from .geocat_guidebook import FILE_CONTENTS_GUIDE, find_nav_uid_from_filename

87 changes: 78 additions & 9 deletions py/polar2grid_modis/polar2grid/modis/geocat_guidebook.py
View file
Expand Up @@ -41,7 +41,7 @@
DEFAULT_SCALE_FACTOR_NAME = 'scale_factor'
DEFAULT_SCALE_METHOD_NAME = 'scaling_method'

# TODO, these patterns are guesses, need to get confirmation from Corey or Justin
# FUTURE, these patterns may need to be more flexible depending on future input
# fog variable patterns
IFR_FOG_PROB_VAR_PATTERN = r'.*?_IFR_fog_probability'
LIFR_FOG_PROB_VAR_PATTERN = r'.*?_LIFR_fog_probability'
Expand All @@ -53,36 +53,44 @@
ASH_HEIGHT_VAR_PATTERN = r'.*?_ash_top_height'
ASH_MASS_LOADING_VAR_PATTERN = r'.*?_ash_mass_loading'
ASH_EFF_RADIUS_VAR_PATTERN = r'.*?_ash_effective_radius'
ASH_BTD_11_12_UM_VAR_PATTERN = r'btd1112' # TODO, will this pattern work?
ASH_11_UM_VAR_PATTERN = r'channel_14_brightness_temperature' # TODO, will this pattern work?
ASH_VISIBLE_VAR_PATTERN = r'channel_2_reflectance' # TODO, will this pattern work?
ASH_BTD_11_12_UM_VAR_PATTERN = r'btd1112'
ASH_11_UM_VAR_PATTERN = r'channel_14_brightness_temperature'
ASH_VISIBLE_VAR_PATTERN = r'channel_2_reflectance'
# so2 variable patterns
SO2_LOADING_VAR_PATTERN = r'.*?_So2_Loading'
SO2_MASK_VAR_PATTERN = r'.*?_so2_mask'

# this is true for the 1km data, FUTURE: when we get to other kinds, this will need to be more sophisicated
MODIS_ROWS_PER_SCAN = 10
#AVHRR_ROWS_PER_SCAN = 1 # TODO William confirmed that this is 1, but fornav won't accept less than 2 (TODO, is this line needed at all?)
#AVHRR_ROWS_PER_SCAN = 1 # FUTURE William confirmed that this is 1, but fornav won't accept less than 2 (so if avhrr input is ever needed, this will need to be reconsidered)
MTSAT_ROWS_PER_SCAN = 2 # confirmed by William
SEVIRI_ROWS_PER_SCAN = 2 # confirmed by William
GOES_ROWS_PER_SCAN = 2 # confirmed by William
# TODO, need additional values for other cases: this should cover Aqua and Terra, but we also expect Goes-12, Goes-15, SNPP (VIIRS), Meteosat-9 (SEVIRI), and MTSAT-2
# FUTURE, do we need additional values for other cases? this should cover Aqua and Terra, but we also expect Goes-12, Goes-15, SNPP (VIIRS), Meteosat-9 (SEVIRI), and MTSAT-2

# more general rows per scan dictionary
ROWS_PER_SCAN = {
(SAT_AQUA, INST_MODIS) : MODIS_ROWS_PER_SCAN,
(SAT_TERRA, INST_MODIS) : MODIS_ROWS_PER_SCAN,
(SAT_GOES13, INST_GIMAGER) : GOES_ROWS_PER_SCAN,
(SAT_GOES14, INST_GIMAGER) : GOES_ROWS_PER_SCAN,
(SAT_GOES15, INST_GIMAGER) : GOES_ROWS_PER_SCAN,
# TODO, add the other satellites that we'll need to know this information for
# FUTURE, add the other satellites that we need to know this information for
}

# a regular expression that will match geocat files
GEOCAT_FILE_PATTERN = r'geocatL2\..*?\.\d\d\d\d\d\d\d\.\d\d\d\d\d\d\.hdf'
# a regular expression that will match geocat files that are in the general geo_nav group
#GEOCAT_FILE_PATTERN = r'geocatL2\..*?\.\d\d\d\d\d\d\d\.\d\d\d\d\d\d\.hdf'
GEOCAT_FILE_PATTERN = r'geocatL2\..*?-(?!(Alaska|CONUS)).*?\.\d\d\d\d\d\d\d\.\d\d\d\d\d\d\.hdf'
# a regular expression that will match Alaska specific files
ALASKA_FILE_PATTERN = r'geocatL2\..*?(Alaska).*?\.\d\d\d\d\d\d\d\.\d\d\d\d\d\d\.hdf'
# a regular expression that will match CONUS specific files
CONUS_FILE_PATTERN = r'geocatL2\..*?(CONUS).*?\.\d\d\d\d\d\d\d\.\d\d\d\d\d\d\.hdf'

# not sure if this will work this way in the long run
GEO_FILE_GROUPING = {
GEO_NAV_UID: [GEOCAT_FILE_PATTERN],
AK_NAV_UID: [ALASKA_FILE_PATTERN],
CONUS_NAV_UID: [CONUS_FILE_PATTERN],
}

# a mapping between regular expressions to match files and their band_kind and band_id contents
Expand All @@ -102,6 +110,44 @@
BKIND_ASH11: [NOT_APPLICABLE], # has no attrs
BKIND_ASHV: [NOT_APPLICABLE], # has no attrs

BKIND_SO2L: [NOT_APPLICABLE], # not present in current test files
BKIND_SO2M: [NOT_APPLICABLE], # not present in current test files
},

ALASKA_FILE_PATTERN: {
BKIND_IFR: [BID_FOG],
BKIND_LIFR: [BID_FOG],
BKIND_MVFR: [BID_FOG],

BKIND_CLDT: [NOT_APPLICABLE],
BKIND_CLDP: [NOT_APPLICABLE],

BKIND_ASHH: [NOT_APPLICABLE],
BKIND_ASHM: [NOT_APPLICABLE],
BKIND_ASHE: [NOT_APPLICABLE],
BKIND_ASHB: [NOT_APPLICABLE], # has no attrs
BKIND_ASH11: [NOT_APPLICABLE], # has no attrs
BKIND_ASHV: [NOT_APPLICABLE], # has no attrs

BKIND_SO2L: [NOT_APPLICABLE], # not present in current test files
BKIND_SO2M: [NOT_APPLICABLE], # not present in current test files
},

CONUS_FILE_PATTERN: {
BKIND_IFR: [BID_FOG],
BKIND_LIFR: [BID_FOG],
BKIND_MVFR: [BID_FOG],

BKIND_CLDT: [NOT_APPLICABLE],
BKIND_CLDP: [NOT_APPLICABLE],

BKIND_ASHH: [NOT_APPLICABLE],
BKIND_ASHM: [NOT_APPLICABLE],
BKIND_ASHE: [NOT_APPLICABLE],
BKIND_ASHB: [NOT_APPLICABLE], # has no attrs
BKIND_ASH11: [NOT_APPLICABLE], # has no attrs
BKIND_ASHV: [NOT_APPLICABLE], # has no attrs

BKIND_SO2L: [NOT_APPLICABLE], # not present in current test files
BKIND_SO2M: [NOT_APPLICABLE], # not present in current test files
},
Expand Down Expand Up @@ -190,6 +236,26 @@
(BKIND_SO2M, NOT_APPLICABLE): (DEFAULT_SCALE_FACTOR_NAME, DEFAULT_ADD_OFFSET_NAME, DEFAULT_SCALE_METHOD_NAME),
}

def find_nav_uid_from_filename (file_name) :
"""
Given the file name, figure out which navigation group it goes in and return the
appropriate nav_uid.
:param file_name:
:return:
"""

matched_uid = None
for nav_uid in GEO_FILE_GROUPING.keys() :
for file_pattern in GEO_FILE_GROUPING[nav_uid] :
if re.match(file_pattern, file_name) is not None :
matched_uid = nav_uid

if matched_uid is None :
LOG.warn("Unable to match file " + file_name + " to a navigation group.")

return matched_uid

def parse_datetime_from_filename (file_name_string) :
"""parse the given file_name_string and create an appropriate datetime object
that represents the datetime indicated by the file name; if the file name does
Expand Down Expand Up @@ -224,6 +290,9 @@ def get_satellite_from_filename (data_file_name_string) :
elif data_file_name_string.find("GOES-13") >= 0 :
satellite_to_return = SAT_GOES13
instrument_to_return = INST_GIMAGER
elif data_file_name_string.find("GOES-14") >= 0 :
satellite_to_return = SAT_GOES14
instrument_to_return = INST_GIMAGER
elif data_file_name_string.find("GOES-15") >= 0 :
satellite_to_return = SAT_GOES15
instrument_to_return = INST_GIMAGER
Expand Down

0 comments on commit 1d68c59

Please sign in to comment.