Permalink
Browse files

refactoring for more generalized data sets instead of hard coded day …

…and night
  • Loading branch information...
1 parent f11cbd3 commit 0a2f26aaa258b4d6c919f0621d1606e198ab02a8 Eva Schiffer committed Feb 21, 2014
Showing with 231 additions and 158 deletions.
  1. +26 −0 stg/constants.py
  2. +2 −2 stg/general_guidebook.py
  3. +25 −10 stg/io_manager.py
  4. +56 −0 stg/modis_io.py
  5. +7 −14 stg/space_gridding.py
  6. +115 −132 stg/space_time_gridding.py
View
@@ -42,3 +42,29 @@
LAT_KEY = "latitude"
DAY_MASK_KEY = "day_mask"
NIGHT_MASK_KEY = "night_mask"
+
+# these are suffixes used for temporary files
+DAY_TEMP_SUFFIX = "_daytemp"
+NIGHT_TEMP_SUFFIX = "_nighttemp"
+DAY_DENSITY_TEMP_SUFFIX = "_daydensitytemp"
+NIGHT_DENSITY_TEMP_SUFFIX = "_nightdensitytemp"
+DAY_NOBS_TEMP_SUFFIX = "_daynobstemp"
+NIGHT_NOBS_TEMP_SUFFIX = "_nightnobstemp"
+
+# these are suffixes used for the final, packed files
+DAY_SUFFIX = "_dayfinal"
+NIGHT_SUFFIX = "_nightfinal"
+DAY_NOBS_SUFFIX = "_daynobsfinal"
+NIGHT_NOBS_SUFFIX = "_nightnobsfinal"
+
+# keys for keeping track of data sets
+DAY_SET_KEY = "day"
+NIGHT_SET_KEY = "night"
+DAILY_SET = "daily"
+SET_MASK_KEY = "mask"
+SET_TEMP_DENSITY_SUFF_KEY = "temp density suffix"
+SET_TEMP_NOBS_SUFF_KEY = "temp nobs suffix"
+SET_TEMP_DATA_SUFF_KEY = "temp data suffix"
+SET_FINAL_DATA_SUFF_KEY = "output data suffix"
+SET_FINAL_NOBS_SUFF_KEY = "output nobs suffix"
+
View
@@ -72,13 +72,13 @@ def get_variable_names (file_name_string, user_requested_names=[ ]) :
return var_names
-def get_expected_files_per_day (file_name_string) :
+def get_expected_files_per_day (satellite_constant) :
"""given a file name, determine the expected number of files per day for that instrument
"""
number_to_return = None
- if modis_guidebook.is_MODIS_file(file_name_string) :
+ if satellite_constant == INST_MODIS :
number_to_return = modis_guidebook.EXPECTED_FILES_PER_DAY
# FUTURE, needs a statment for ctp
View
@@ -34,21 +34,11 @@
LOG = logging.getLogger(__name__)
# these are suffixes used for temporary files
-DAY_TEMP_SUFFIX = "_daytemp"
-NIGHT_TEMP_SUFFIX = "_nighttemp"
-DAY_DENSITY_TEMP_SUFFIX = "_daydensitytemp"
-NIGHT_DENSITY_TEMP_SUFFIX = "_nightdensitytemp"
-DAY_NOBS_TEMP_SUFFIX = "_daynobstemp"
-NIGHT_NOBS_TEMP_SUFFIX = "_nightnobstemp"
EXPECTED_TEMP_SUFFIXES = [DAY_TEMP_SUFFIX, NIGHT_TEMP_SUFFIX,
DAY_DENSITY_TEMP_SUFFIX, NIGHT_DENSITY_TEMP_SUFFIX,
DAY_NOBS_TEMP_SUFFIX, NIGHT_NOBS_TEMP_SUFFIX]
# these are suffixes used for the final, packed files
-DAY_SUFFIX = "_dayfinal"
-NIGHT_SUFFIX = "_nightfinal"
-DAY_NOBS_SUFFIX = "_daynobsfinal"
-NIGHT_NOBS_SUFFIX = "_nightnobsfinal"
EXPECTED_FINAL_SUFFIXES = [DAY_SUFFIX, NIGHT_SUFFIX,
DAY_NOBS_SUFFIX, NIGHT_NOBS_SUFFIX]
@@ -106,6 +96,31 @@ def load_aux_data (file_path, minimum_scan_angle, file_object=None) :
return file_object, temp_aux_data
+def get_expected_abstract_sets (satellite_constant) :
+
+ expected_data_sets = { }
+
+ if satellite_constant == INST_MODIS :
+ expected_data_sets = modis_io.get_abstract_data_sets ()
+ # FUTURE, needs a statment for ctp
+
+ return expected_data_sets
+
+def get_expected_data_sets_from_aux_data (satellite_constant, aux_data) :
+ """given aux data in the form returned by load_aux_data and the grid degrees constant, return the data sets to be processed
+
+ Each data set is defined by a constant name, a mask to select that set, it's expected suffixes for temporary density/nobs/data
+ and it's expected suffix for the final output data/nobs
+ """
+
+ expected_data_sets = { }
+
+ if satellite_constant == INST_MODIS :
+ expected_data_sets = modis_io.determine_data_sets(aux_data)
+ # FUTURE, needs a statment for ctp
+
+ return expected_data_sets
+
def load_variable_from_file (variable_name, file_path=None, file_object=None,
data_type_for_output=numpy.float32) :
"""
View
@@ -174,6 +174,62 @@ def unscale_data (data, fill_mask=None, scale_factor=None, offset=None) :
return to_return
+def get_abstract_data_sets () :
+
+ sets_to_return = { }
+
+ # build the day set
+ sets_to_return[DAY_SET_KEY] = { }
+
+ # set all the suffixes
+ sets_to_return[DAY_SET_KEY][SET_TEMP_DENSITY_SUFF_KEY] = DAY_DENSITY_TEMP_SUFFIX
+ sets_to_return[DAY_SET_KEY][SET_TEMP_NOBS_SUFF_KEY] = DAY_NOBS_TEMP_SUFFIX
+ sets_to_return[DAY_SET_KEY][SET_TEMP_DATA_SUFF_KEY] = DAY_TEMP_SUFFIX
+ # the final output suffixes
+ sets_to_return[DAY_SET_KEY][SET_FINAL_DATA_SUFF_KEY] = DAY_SUFFIX
+ sets_to_return[DAY_SET_KEY][SET_FINAL_NOBS_SUFF_KEY] = DAY_NOBS_SUFFIX
+
+ # build the night set
+ sets_to_return[NIGHT_SET_KEY] = { }
+
+ # set all the suffixes
+ sets_to_return[NIGHT_SET_KEY][SET_TEMP_DENSITY_SUFF_KEY] = NIGHT_DENSITY_TEMP_SUFFIX
+ sets_to_return[NIGHT_SET_KEY][SET_TEMP_NOBS_SUFF_KEY] = NIGHT_NOBS_TEMP_SUFFIX
+ sets_to_return[NIGHT_SET_KEY][SET_TEMP_DATA_SUFF_KEY] = NIGHT_TEMP_SUFFIX
+ # the final output suffixes
+ sets_to_return[NIGHT_SET_KEY][SET_FINAL_DATA_SUFF_KEY] = NIGHT_SUFFIX
+ sets_to_return[NIGHT_SET_KEY][SET_FINAL_NOBS_SUFF_KEY] = NIGHT_NOBS_SUFFIX
+
+ return sets_to_return
+
+def determine_data_sets(aux_data) :
+ """separate modis data into day and night sets
+
+ Each data set is defined by a constant name, a mask to select that set, the lon and lat data for that set,
+ it's expected suffixes for temporary density/nobs/data, and it's expected suffix for the final output data/nobs
+ """
+
+ sets_to_return = get_abstract_data_sets( )
+
+ # build the day set
+
+ # set the mask
+ sets_to_return[DAY_SET_KEY][SET_MASK_KEY] = aux_data[DAY_MASK_KEY]
+ # set the lon and lat data
+ sets_to_return[DAY_SET_KEY][LON_KEY] = aux_data[LON_KEY]
+ sets_to_return[DAY_SET_KEY][LAT_KEY] = aux_data[LAT_KEY]
+
+ # build the night set
+
+ # set the mask
+ sets_to_return[NIGHT_SET_KEY][SET_MASK_KEY] = aux_data[NIGHT_MASK_KEY]
+ # set the lon and lat data
+ sets_to_return[NIGHT_SET_KEY][LON_KEY] = aux_data[LON_KEY]
+ sets_to_return[NIGHT_SET_KEY][LAT_KEY] = aux_data[LAT_KEY]
+
+ # return the final sets
+ return sets_to_return
+
# FUTURE, will this be used for other satellites? should it move up to the io_manager?
def satellite_zenith_angle_to_scan_angle (sat_zenith_data) :
"""
View
@@ -23,24 +23,17 @@
LOG = logging.getLogger(__name__)
-def calculate_index_from_nav_data (aux_data, grid_degrees) :
+def calculate_index_from_nav_data (lat_data, lon_data, grid_degrees) :
+ """given the lon/lat data, calculate where the elements will be space gridded
"""
- given the aux data, use the navigation and masks to calculate
- where the elements will be space gridded
- """
-
- night_lon_temp = aux_data[LON_KEY][aux_data[NIGHT_MASK_KEY]]
- night_lat_temp = aux_data[LAT_KEY][aux_data[NIGHT_MASK_KEY]]
- # figure out where the day/night indexes will fall
- day_lon_index = numpy.round((aux_data[LON_KEY][aux_data[DAY_MASK_KEY]] + 180.0) / grid_degrees) % (360.0 / grid_degrees)
- day_lat_index = numpy.round((aux_data[LAT_KEY][aux_data[DAY_MASK_KEY]] + 90.0) / grid_degrees) % (180.0 / grid_degrees)
- night_lon_index = numpy.round((aux_data[LON_KEY][aux_data[NIGHT_MASK_KEY]] + 180.0) / grid_degrees) % (360.0 / grid_degrees)
- night_lat_index = numpy.round((aux_data[LAT_KEY][aux_data[NIGHT_MASK_KEY]] + 90.0) / grid_degrees) % (180.0 / grid_degrees)
+ # calculate the indexes for lon and lat
+ lon_index = numpy.round((lon_data + 180.0) / grid_degrees) % (360.0 / grid_degrees)
+ lat_index = numpy.round((lat_data + 90.0) / grid_degrees) % (180.0 / grid_degrees)
- return day_lon_index, day_lat_index, night_lon_index, night_lat_index
+ return lat_index, lon_index
-def space_grid_data (grid_lon_size, grid_lat_size, data, lon_indexes, lat_indexes ) :
+def space_grid_data (grid_lat_size, grid_lon_size, data, lat_indexes, lon_indexes ) :
"""
given lon/lat indexes, data, and the grid size, sort the data into a space grid
Oops, something went wrong.

0 comments on commit 0a2f26a

Please sign in to comment.