Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

fixing lon/lat reversal and moving stem name construction into a func…

…tion
  • Loading branch information...
commit 2058fdb0eb33536a3ac19804de12cde2179396d7 1 parent 776f28a
Eva Schiffer authored
View
31 stg/io_manager.py
@@ -55,6 +55,9 @@
DAY_SUFFIX, NIGHT_SUFFIX,
DAY_NOBS_SUFFIX, NIGHT_NOBS_SUFFIX]
+# the strftime format for date stamping our files
+DATE_STAMP_FORMAT = "%Y%m%d"
+
def open_file (file_path) :
"""
given a file path that is a modis file, open it
@@ -118,6 +121,34 @@ def save_data_to_file (stem_name, grid_shape, output_path, data_array, data_type
data_array.astype(data_type).tofile(temp_file_obj)
temp_file_obj.close()
+def build_name_stem (variable_name, date_time=None, satellite=None, algorithm=None, suffix=None) :
+ """given information on what's in the file, build a file stem
+ if there's extra info like the date time, satellite, algorithm name, or a suffix
+ include that in the file stem as well
+
+ the name format is:
+ satellite_algorithm_datestamp_variablename_suffix
+ """
+
+ # the basic stem name is just the variable
+ stem_name = variable_name
+
+ # if we have a date time, add a time stamp at the beginning
+ stem_name = date_time.strftime(DATE_STAMP_FORMAT) + "_" + stem_name if date_time is not None else stem_name
+
+ # if we have an algorithm prefix add that
+ stem_name = algorithm + "_" + stem_name if algorithm is not None else stem_name
+
+ # if we have a satellite, add that to the beginning
+ stem_name = satellite + "_" + stem_name if satellite is not None else stem_name
+
+ # if we have a suffix, add that too
+ stem_name = stem_name + suffix if suffix is not None else stem_name
+
+ return stem_name
+
+ # date_stamp + "_" + var_name + suffix
+
def main():
import optparse
from pprint import pprint
View
14 stg/modis_io.py
@@ -69,7 +69,7 @@ def load_aux_data (file_path, minimum_scan_angle, file_object=None) :
scan_angle_data_temp = satellite_zenith_angle_to_scan_angle(sat_zenith_data_temp)
# build the day and night masks
- ok_scan_angle = scan_angle_data_temp <= minimum_scan_angle
+ ok_scan_angle = scan_angle_data_temp <= minimum_scan_angle
aux_data_sets[DAY_MASK_KEY] = (solar_zenith_data_temp < DAY_NIGHT_LINE_DEGREES) & ok_scan_angle
aux_data_sets[NIGHT_MASK_KEY] = (solar_zenith_data_temp >= DAY_NIGHT_LINE_DEGREES) & ok_scan_angle
@@ -98,15 +98,14 @@ def load_variable_from_file (variable_name, file_path=None, file_object=None,
# defaults
scale_factor = 1.0
add_offset = 0.0
- data_type = None
- scaling_method = None
+ data_type = None
# get the variable object and use it to
# get our raw data and scaling info
variable_object = file_object.select(variable_name)
- raw_data_copy = variable_object[:]
- raw_data_copy = raw_data_copy.astype(data_type_for_output) if data_type_for_output is not None else raw_data_copy
- temp_attrs = variable_object.attributes()
+ raw_data_copy = variable_object[:]
+ raw_data_copy = raw_data_copy.astype(data_type_for_output) if data_type_for_output is not None else raw_data_copy
+ temp_attrs = variable_object.attributes()
try :
scale_factor, scale_factor_error, add_offset, add_offset_error, data_type = SDS.getcal(variable_object)
except HDF4Error:
@@ -144,8 +143,7 @@ def load_variable_from_file (variable_name, file_path=None, file_object=None,
return file_object, scaled_data_copy
def unscale_data (data, fill_mask=None, scale_factor=None, offset=None) :
- """
- unscale the given data
+ """unscale the given data
data is modified in place and fill values will not be changed
if a scale factor or offset is given as None (or not given) it will not be applied
View
19 stg/plot_tools.py
@@ -37,9 +37,6 @@
DEFAULT_RANGE_OFFSET = 0.00000000000000000000001
DEFAULT_AXIS = [-180, 180, -90, 90]
-EXP_MAX_LON_INDEX = 360
-EXP_MAX_LAT_INDEX = 180
-
def plot_mapped(data, baseMapInstance, title,
vmin=None, vmax=None,
boundingAxis=DEFAULT_AXIS,
@@ -50,18 +47,22 @@ def plot_mapped(data, baseMapInstance, title,
temp_mask = None
if not data is None :
# TODO, there's probably a better way to show 3D data
- data = numpy.nansum(data, axis=0) if len(data.shape) > 2 else data
+ if len(data.shape) > 2 :
+ # calculate a rough mean
+ data = numpy.nansum(data, axis=0) / numpy.sum(numpy.isfinite(data), axis=0)
+ #data = data[0]
temp_mask = numpy.isnan(data) if fillValue is numpy.nan else data == fillValue
data = numpy.ma.masked_where(temp_mask, data)
# build our lon/lat from the data shape
- lon_row = numpy.linspace(DEFAULT_LON_RANGE[0], DEFAULT_LON_RANGE[1], data.shape[1])
- lon_data = numpy.tile(lon_row, (data.shape[0], 1))
+ lon_row = numpy.linspace(DEFAULT_LON_RANGE[0], DEFAULT_LON_RANGE[1], data.shape[0])
+ lon_data = numpy.tile(lon_row, (data.shape[1], 1))
+ lon_data = numpy.transpose(lon_data)
- lat_row = numpy.linspace(DEFAULT_LAT_RANGE[0], DEFAULT_LAT_RANGE[1], data.shape[0])
- lat_data = numpy.tile(lat_row, (data.shape[1], 1))
- lat_data = numpy.transpose(lat_data)
+ lat_row = numpy.linspace(DEFAULT_LAT_RANGE[0], DEFAULT_LAT_RANGE[1], data.shape[1])
+ lat_data = numpy.tile(lat_row, (data.shape[0], 1))
+ #lat_data = numpy.transpose(lat_data)
# build the plot
figure = plt.figure()
View
22 stg/space_gridding.py
@@ -28,11 +28,14 @@ def calculate_index_from_nav_data (aux_data, grid_degrees) :
where the elements will be space gridded
"""
+ night_lon_temp = aux_data[LON_KEY][aux_data[NIGHT_MASK_KEY]]
+ night_lat_temp = aux_data[LAT_KEY][aux_data[NIGHT_MASK_KEY]]
+
# figure out where the day/night indexes will fall
- day_lon_index = numpy.round((aux_data[LON_KEY][aux_data[DAY_MASK_KEY]] + 90.0) / grid_degrees)
- day_lat_index = numpy.round((aux_data[LAT_KEY][aux_data[DAY_MASK_KEY]] + 180.0) / grid_degrees)
- night_lon_index = numpy.round((aux_data[LON_KEY][aux_data[NIGHT_MASK_KEY]] + 90.0) / grid_degrees)
- night_lat_index = numpy.round((aux_data[LAT_KEY][aux_data[NIGHT_MASK_KEY]] + 180.0) / grid_degrees)
+ day_lon_index = numpy.round((aux_data[LON_KEY][aux_data[DAY_MASK_KEY]] + 180.0) / grid_degrees)
+ day_lat_index = numpy.round((aux_data[LAT_KEY][aux_data[DAY_MASK_KEY]] + 90.0) / grid_degrees)
+ night_lon_index = numpy.round((aux_data[LON_KEY][aux_data[NIGHT_MASK_KEY]] + 180.0) / grid_degrees)
+ night_lat_index = numpy.round((aux_data[LAT_KEY][aux_data[NIGHT_MASK_KEY]] + 90.0) / grid_degrees)
return day_lon_index, day_lat_index, night_lon_index, night_lat_index
@@ -52,9 +55,9 @@ def space_grid_data (grid_lon_size, grid_lat_size, data, lon_indexes, lat_indexe
# FUTURE, I do not like this looping solution, figure out how to do this in native numpy ops
density_map = numpy.zeros(space_grid_shape)
nobs_map = numpy.zeros(space_grid_shape)
- for index in range(len(data)) :
+ for index in range(data.size) :
nobs_map[lon_indexes[index], lat_indexes[index]] += 1
- if not numpy.isnan(data[index]) :
+ if numpy.isfinite(data[index]) :
density_map[lon_indexes[index], lat_indexes[index]] += 1
max_depth = numpy.max(density_map)
@@ -66,8 +69,8 @@ def space_grid_data (grid_lon_size, grid_lat_size, data, lon_indexes, lat_indexe
# put the variable data into the space grid
# FUTURE, I do not like this looping solution, figure out how to do this in native numpy ops
- for index in range(len(data)) :
- if not numpy.isnan(data[index]) :
+ for index in range(data.size) :
+ if numpy.isfinite(data[index]) :
depth = temp_depth[lon_indexes[index], lat_indexes[index]]
space_grid[depth, lon_indexes[index], lat_indexes[index]] = data[index]
temp_depth[ lon_indexes[index], lat_indexes[index]] += 1
@@ -89,8 +92,7 @@ def pack_space_grid (data_array, density_array) :
max_depth = numpy.max(numpy.sum(density_array, axis=0))
# create the final data array at the right depth
- final_data = numpy.ones((max_depth, data_array.shape[1], data_array.shape[2]), dtype=data_array.dtype)
- final_data = final_data * numpy.nan
+ final_data = numpy.ones((max_depth, data_array.shape[1], data_array.shape[2]), dtype=data_array.dtype) * numpy.nan
LOG.debug(" original data shape: " + str(data_array.shape))
LOG.debug(" final data shape: " + str(final_data.shape))
View
122 stg/space_time_gridding.py
@@ -128,7 +128,7 @@ def main():
The following functions represent available menu selections
"""
- def space_day(*args):
+ def space_day(*args) :
"""grid one day of input files in space
given an input directory that contains appropriate files,
grid them in space and put the resulting gridded files
@@ -146,8 +146,8 @@ def space_day(*args):
grid_degrees = float(options.gridDegrees)
# determine the grid size in number of elements
- grid_lat_size = int(math.ceil(360.0 / grid_degrees))
- grid_lon_size = int(math.ceil(180.0 / grid_degrees))
+ grid_lon_size = int(math.ceil(360.0 / grid_degrees))
+ grid_lat_size = int(math.ceil(180.0 / grid_degrees))
space_grid_shape = (grid_lon_size, grid_lat_size) # TODO, is this the correct order?
# look through our files and figure out what variables we expect from them
@@ -157,19 +157,24 @@ def space_day(*args):
date_time_temp = None
for file_name in sorted(possible_files) :
expected_vars[file_name] = general_guidebook.get_variable_names (file_name, user_requested_names=desired_variables)
- all_vars.update(expected_vars[file_name])
- date_time_temp = general_guidebook.parse_datetime_from_filename(file_name) if date_time_temp is None else date_time_temp
-
- # figure out what our date stamp will look like
- date_stamp = date_time_temp.strftime("%Y%m%d")
+ # if this file has no variables, remove it from our files for consideration
+ if len(expected_vars[file_name]) <= 0 :
+ del expected_vars[file_name]
+ possible_files.remove(file_name)
+ # otherwise, add the variables we found to our list of all variables and try to get a time from the file
+ else :
+ all_vars.update(expected_vars[file_name])
+ date_time_temp = general_guidebook.parse_datetime_from_filename(file_name) if date_time_temp is None else date_time_temp
# check to make sure our intermediate file names don't exist already
for var_name in all_vars :
for suffix in io_manager.ALL_EXPECTED_SUFFIXES :
- temp_name = fbf.filename(date_stamp + "_" + var_name + suffix, TEMP_DATA_TYPE, shape=(space_grid_shape))
+ # TODO, pull satellite and algorithm too
+ temp_stem = io_manager.build_name_stem(var_name, date_time=date_time_temp, satellite=None, algorithm=None, suffix=suffix)
+ temp_name = fbf.filename(temp_stem, TEMP_DATA_TYPE, shape=(space_grid_shape))
if os.path.exists(os.path.join(output_path, temp_name)) :
- LOG.warn ("Cannot process files because intermediate files exist in the output directory.")
+ LOG.warn ("Cannot process files because matching temporary or output files exist in the output directory.")
return
# loop to deal with data from each of the files
@@ -212,19 +217,34 @@ def space_day(*args):
# save the space grids and density info for this variable and it's density map to files
- io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.DAY_TEMP_SUFFIX, space_grid_shape, output_path,
- day_space_grid, TEMP_DATA_TYPE )
- io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.DAY_DENSITY_TEMP_SUFFIX, space_grid_shape, output_path,
- day_density_map, TEMP_DATA_TYPE)
- io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.DAY_NOBS_TEMP_SUFFIX, space_grid_shape, output_path,
- day_nobs, TEMP_DATA_TYPE)
+ # day related files
+ io_manager.save_data_to_file(io_manager.build_name_stem (variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.DAY_TEMP_SUFFIX),
+ space_grid_shape, output_path, day_space_grid, TEMP_DATA_TYPE)
+ io_manager.save_data_to_file(io_manager.build_name_stem (variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.DAY_DENSITY_TEMP_SUFFIX),
+ space_grid_shape, output_path, day_density_map, TEMP_DATA_TYPE)
+ io_manager.save_data_to_file(io_manager.build_name_stem (variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.DAY_NOBS_TEMP_SUFFIX),
+ space_grid_shape, output_path, day_nobs, TEMP_DATA_TYPE)
- io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.NIGHT_TEMP_SUFFIX, space_grid_shape, output_path,
- night_space_grid, TEMP_DATA_TYPE )
- io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.NIGHT_DENSITY_TEMP_SUFFIX, space_grid_shape, output_path,
- night_density_map, TEMP_DATA_TYPE)
- io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.NIGHT_NOBS_TEMP_SUFFIX, space_grid_shape, output_path,
- night_nobs, TEMP_DATA_TYPE)
+ # night related files
+
+ io_manager.save_data_to_file(io_manager.build_name_stem (variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.NIGHT_TEMP_SUFFIX),
+ space_grid_shape, output_path, night_space_grid, TEMP_DATA_TYPE)
+ io_manager.save_data_to_file(io_manager.build_name_stem (variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.NIGHT_DENSITY_TEMP_SUFFIX),
+ space_grid_shape, output_path, night_density_map, TEMP_DATA_TYPE)
+ io_manager.save_data_to_file(io_manager.build_name_stem (variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.NIGHT_NOBS_TEMP_SUFFIX),
+ space_grid_shape, output_path, night_nobs, TEMP_DATA_TYPE)
# make sure each file is closed when we're done with it
io_manager.close_file(full_file_path, file_object)
@@ -236,34 +256,44 @@ def space_day(*args):
# load the variable's density maps
var_workspace = Workspace.Workspace(dir=output_path)
- day_var_density = var_workspace[date_stamp + "_" + variable_name + io_manager.DAY_DENSITY_TEMP_SUFFIX][:]
- night_var_density = var_workspace[date_stamp + "_" + variable_name + io_manager.NIGHT_DENSITY_TEMP_SUFFIX][:]
-
- # set up nobs arrays to accumulate into
- day_nobs_total = numpy.zeros(space_grid_shape, dtype=TEMP_DATA_TYPE)
- night_nobs_total = numpy.zeros(space_grid_shape, dtype=TEMP_DATA_TYPE)
+ day_var_density = var_workspace[io_manager.build_name_stem(variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.DAY_DENSITY_TEMP_SUFFIX)][:]
+ night_var_density = var_workspace[io_manager.build_name_stem(variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.NIGHT_DENSITY_TEMP_SUFFIX)][:]
# only do the day data if we have some
if numpy.sum(day_var_density) > 0 :
# load the sparse space grid
- day_var_data = var_workspace[date_stamp + "_" + variable_name + io_manager.DAY_TEMP_SUFFIX][:]
+ day_var_data = var_workspace[io_manager.build_name_stem(variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.DAY_TEMP_SUFFIX)][:]
# collapse the space grid
final_day_data = space_gridding.pack_space_grid(day_var_data, day_var_density)
# save the final array to an appropriately named file
- io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.DAY_SUFFIX, space_grid_shape, output_path,
- final_day_data, TEMP_DATA_TYPE, file_permissions="w")
+ io_manager.save_data_to_file(io_manager.build_name_stem(variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.DAY_SUFFIX),
+ space_grid_shape, output_path, final_day_data,
+ TEMP_DATA_TYPE, file_permissions="w")
# load the nobs file
- nobs_counts = var_workspace[date_stamp + "_" + variable_name + io_manager.DAY_NOBS_TEMP_SUFFIX][:]
+ nobs_counts = var_workspace[io_manager.build_name_stem(variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.DAY_NOBS_TEMP_SUFFIX)][:]
# collapse the nobs
nobs_final = numpy.sum(nobs_counts, axis=0)
# save the final nobs array to an appropriately named file
- io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.DAY_NOBS_SUFFIX, space_grid_shape, output_path,
+ io_manager.save_data_to_file(io_manager.build_name_stem(variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.DAY_NOBS_SUFFIX),
+ space_grid_shape, output_path,
nobs_final, TEMP_DATA_TYPE, file_permissions="w")
else :
@@ -273,23 +303,33 @@ def space_day(*args):
if numpy.sum(night_var_density) > 0 :
# load the sparse space grid
- night_var_data = var_workspace[date_stamp + "_" + variable_name + io_manager.NIGHT_TEMP_SUFFIX][:]
+ night_var_data = var_workspace[io_manager.build_name_stem(variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.NIGHT_TEMP_SUFFIX)][:]
# collapse the space grid
- final_night_data = space_gridding.pack_space_grid(night_var_data, night_var_density)
+ final_night_data = space_gridding.pack_space_grid(night_var_data, night_var_density)
# save the final array to an appropriately named file
- io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.NIGHT_SUFFIX, space_grid_shape, output_path,
- final_night_data, TEMP_DATA_TYPE, file_permissions="w")
+ io_manager.save_data_to_file(io_manager.build_name_stem(variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.NIGHT_SUFFIX),
+ space_grid_shape, output_path, final_night_data,
+ TEMP_DATA_TYPE, file_permissions="w")
# load the nobs file
- nobs_counts = var_workspace[date_stamp + "_" + variable_name + io_manager.NIGHT_NOBS_TEMP_SUFFIX][:]
+ nobs_counts = var_workspace[io_manager.build_name_stem(variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.NIGHT_NOBS_TEMP_SUFFIX)][:]
# collapse the nobs
nobs_final = numpy.sum(nobs_counts, axis=0)
# save the final nobs array to an appropriately named file
- io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.NIGHT_NOBS_SUFFIX, space_grid_shape, output_path,
+ io_manager.save_data_to_file(io_manager.build_name_stem(variable_name, date_time=date_time_temp,
+ satellite=None, algorithm=None,
+ suffix=io_manager.NIGHT_NOBS_SUFFIX),
+ space_grid_shape, output_path,
nobs_final, TEMP_DATA_TYPE, file_permissions="w")
else :
@@ -299,7 +339,7 @@ def space_day(*args):
remove_suffixes = ["*" + p + "*" for p in io_manager.EXPECTED_TEMP_SUFFIXES]
remove_file_patterns(output_path, remove_suffixes)
- def stats_day(*args):
+ def stats_day(*args) :
"""given files of daily space gridded data, calculate daily stats
given an input directory that contains appropriate files,
calculate daily stats and put the resulting gridded files
@@ -316,7 +356,7 @@ def stats_day(*args):
min_scan_angle = options.minScanAngle
grid_degrees = float(options.gridDegrees)
- def stats_month(*args):
+ def stats_month(*args) :
"""given a month of daily space gridded data, calculate montly stats
given an input directory that contains appropriate files,
calculate monthly stats and put the resulting gridded files
Please sign in to comment.
Something went wrong with that request. Please try again.