Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

added time stamp to file names; built minimal visualization tools; ad…

…ded calculation and output of num-obs files
  • Loading branch information...
commit f03565e410a69c1742513128ce7998418a4b67af 1 parent 6bb1bca
Eva Schiffer authored
View
2  setup.py
@@ -25,7 +25,7 @@
setup( name="spacetimegrid",
version="0.1",
zip_safe = True,
- entry_points = { 'console_scripts': [ 'stg = stg.space_time_gridding:main' ] },
+ entry_points = { 'console_scripts': [ 'stg = stg.space_time_gridding:main', 'stg_plot = stg.plot_tools:main' ] },
packages = ['stg'], #find_packages('.'),
install_requires=[ 'numpy', 'scipy' ],
#package_data = {'': ['*.txt', '*.gif']}
View
18 stg/io_manager.py
@@ -34,16 +34,26 @@
NIGHT_TEMP_SUFFIX = "_nighttemp"
DAY_DENSITY_TEMP_SUFFIX = "_daydensitytemp"
NIGHT_DENSITY_TEMP_SUFFIX = "_nightdensitytemp"
-EXPECTED_TEMP_SUFFIXES = [DAY_TEMP_SUFFIX, NIGHT_TEMP_SUFFIX, DAY_DENSITY_TEMP_SUFFIX, NIGHT_DENSITY_TEMP_SUFFIX]
+DAY_NOBS_TEMP_SUFFIX = "_daynobstemp"
+NIGHT_NOBS_TEMP_SUFFIX = "_nightnobstemp"
+EXPECTED_TEMP_SUFFIXES = [DAY_TEMP_SUFFIX, NIGHT_TEMP_SUFFIX,
+ DAY_DENSITY_TEMP_SUFFIX, NIGHT_DENSITY_TEMP_SUFFIX,
+ DAY_NOBS_TEMP_SUFFIX, NIGHT_NOBS_TEMP_SUFFIX]
# these are suffixes used for the final, packed files
DAY_SUFFIX = "_dayfinal"
NIGHT_SUFFIX = "_nightfinal"
-EXPECTED_FINAL_SUFFIXES = [DAY_SUFFIX, NIGHT_SUFFIX]
+DAY_NOBS_SUFFIX = "_daynobsfinal"
+NIGHT_NOBS_SUFFIX = "_nightnobsfinal"
+EXPECTED_FINAL_SUFFIXES = [DAY_SUFFIX, NIGHT_SUFFIX,
+ DAY_NOBS_SUFFIX, NIGHT_NOBS_SUFFIX]
# all the suffixes we can produce
-ALL_EXPECTED_SUFFIXES = [DAY_TEMP_SUFFIX, NIGHT_TEMP_SUFFIX, DAY_DENSITY_TEMP_SUFFIX, NIGHT_DENSITY_TEMP_SUFFIX,
- DAY_SUFFIX, NIGHT_SUFFIX]
+ALL_EXPECTED_SUFFIXES = [DAY_TEMP_SUFFIX, NIGHT_TEMP_SUFFIX,
+ DAY_DENSITY_TEMP_SUFFIX, NIGHT_DENSITY_TEMP_SUFFIX,
+ DAY_NOBS_TEMP_SUFFIX, NIGHT_NOBS_TEMP_SUFFIX,
+ DAY_SUFFIX, NIGHT_SUFFIX,
+ DAY_NOBS_SUFFIX, NIGHT_NOBS_SUFFIX]
def open_file (file_path) :
"""
View
107 stg/plot_tools.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+This module draws simple plots for space and time gridded data in flat binary
+files.
+
+:author: Eva Schiffer (evas)
+:contact: eva.schiffer@ssec.wisc.edu
+:organization: Space Science and Engineering Center (SSEC)
+:copyright: Copyright (c) 2014 University of Wisconsin SSEC. All rights reserved.
+:date: Jan 2014
+:license: GNU GPLv3
+
+Copyright (C) 2014 Space Science and Engineering Center (SSEC),
+ University of Wisconsin-Madison.
+"""
+__docformat__ = "restructuredtext en"
+
+import os
+from glob import glob
+import numpy
+
+import matplotlib
+matplotlib.use('agg')
+from matplotlib import pyplot as plt
+import keoni.fbf.workspace as Workspace
+
+DEFAULT_FILE_PATTERN = "*.real4.*.*"
+DEFAULT_FILL_VALUE = numpy.nan
+DEFAULT_DPI = 150
+
+def plot_binary(bf, in_dir='.',
+ fill_value=DEFAULT_FILL_VALUE,
+ dpi_to_use=DEFAULT_DPI,
+ vmin=None, vmax=None):
+
+ # get the data from the file
+ var_workspace = Workspace.Workspace(dir=in_dir)
+ fbf_attr_name = bf.split(".")[0]
+ raw_data = var_workspace[fbf_attr_name][:]
+ # TODO, there's probably a better way to show 3D data
+ raw_data = numpy.nansum(raw_data, axis=0) if len(raw_data.shape) > 2 else raw_data
+
+
+ # mask the data based on the fill value
+ masked_data = numpy.ma.masked_where(raw_data == fill_value, raw_data)
+ print masked_data.min(), masked_data.max()
+ print masked_data.shape
+
+ # plot the figure
+ plt.figure()
+ plt.imshow(masked_data, vmin=vmin, vmax=vmax)
+ plt.bone()
+ plt.colorbar()
+ plt.savefig("plot_binary.%s.png" % fbf_attr_name, dpi=dpi_to_use)
+ plt.close()
+
+def sci_float(x):
+ x = x.replace("\"", "")
+ x = x.replace("\'", "")
+ return float(str(x))
+
+def main():
+ from argparse import ArgumentParser
+ description = """
+Plot binary files using matplotlib.
+ """
+ parser = ArgumentParser(description=description)
+ parser.add_argument("-f", dest="fill_value", default=DEFAULT_FILL_VALUE, type=sci_float,
+ help="Specify the fill_value of the input file(s)")
+ parser.add_argument('--vmin', dest="vmin", default=None, type=int,
+ help="Specify minimum brightness value. Defaults to minimum value of data.")
+ parser.add_argument('--vmax', dest="vmax", default=None, type=int,
+ help="Specify maximum brightness value. Defaults to maximum value of data.")
+ parser.add_argument("-p", dest="pattern",
+ help="filename pattern to search the current directory for")
+ parser.add_argument("binary_files", nargs="*",
+ help="list of flat binary files to be plotted in the current directory")
+ parser.add_argument('-d', '--dpi', dest="dpi", default=DEFAULT_DPI, type=float,
+ help="Specify the dpi for the resulting figure, higher dpi will result in larger figures and longer run times")
+ args = parser.parse_args()
+
+ workspace = '.'
+ binary_files = args.binary_files
+ if not args.binary_files and not args.pattern:
+ args.pattern = DEFAULT_FILE_PATTERN
+ if args.pattern:
+ workspace = os.path.split(args.pattern)[0]
+ binary_files = [ os.path.split(x)[1] for x in glob(args.pattern) ]
+
+ for bf in binary_files:
+ print "Plotting '%s'" % (bf,)
+ try:
+ plot_binary(bf, in_dir='.',
+ fill_value=args.fill_value,
+ dpi_to_use=args.dpi,
+ vmin=args.vmin, vmax=args.vmax)
+ except StandardError as e:
+ print "Could not plot '%s'" % (bf,)
+ if hasattr(e, "msg"): print e,e.msg
+ else: print e
+
+if __name__ == "__main__":
+ import sys
+
+ sys.exit(main())
+
View
19 stg/space_gridding.py
@@ -43,28 +43,39 @@ def space_grid_data (grid_lon_size, grid_lat_size, data, lon_indexes, lat_indexe
returns the filled space grid (empty space is NaN values), a density map of where the data is, and the size of the deepest bucket
"""
+ if data.size > 0 :
+ print ("data range: " + str(data.min()) + " " + str(data.max()))
+
space_grid_shape = (grid_lon_size, grid_lat_size) # TODO, is this the correct order?
# create the density map and figure out how dense the data will be
# FUTURE, I do not like this looping solution, figure out how to do this in native numpy ops
density_map = numpy.zeros(space_grid_shape)
+ nobs_map = numpy.zeros(space_grid_shape)
for index in range(len(data)) :
+ nobs_map[lon_indexes[index], lat_indexes[index]] += 1
if not numpy.isnan(data[index]) :
density_map[lon_indexes[index], lat_indexes[index]] += 1
max_depth = numpy.max(density_map)
+ print ("max depth: " + str(max_depth))
+
# create the space grids for this variable
- space_grid = numpy.ones((grid_lon_size, grid_lat_size, max_depth), dtype=numpy.float32) * numpy.nan #TODO, dtype
+ space_grid = numpy.ones((max_depth, grid_lon_size, grid_lat_size), dtype=numpy.float32) * numpy.nan #TODO, dtype
temp_depth = numpy.zeros(space_grid_shape)
# put the variable data into the space grid
# FUTURE, I do not like this looping solution, figure out how to do this in native numpy ops
for index in range(len(data)) :
if not numpy.isnan(data[index]) :
- space_grid[lon_indexes[index], lat_indexes[index], temp_depth[lon_indexes[index], lat_indexes[index]]] = data[index]
- temp_depth[lon_indexes[index], lat_indexes[index]] += 1
+ depth = temp_depth[lon_indexes[index], lat_indexes[index]]
+ space_grid[depth, lon_indexes[index], lat_indexes[index]] = data[index]
+ temp_depth[ lon_indexes[index], lat_indexes[index]] += 1
+
+ if space_grid.size > 0 :
+ print ("grid range: "), numpy.nanmin(space_grid), numpy.nanmax(space_grid)
- return space_grid, density_map, max_depth
+ return space_grid, density_map, nobs_map, max_depth
def pack_space_grid (data_array, density_array) :
"""
View
123 stg/space_time_gridding.py
@@ -128,11 +128,11 @@ def main():
The following functions represent available menu selections
"""
- def space(*args):
- """grid the input files in space
- given an input directory that contains appropriate files MODIS,
- grid them in space and put the resulting gridded date in the
- output directory.
+ def space_day(*args):
+ """grid one day of input files in space
+ given an input directory that contains appropriate files,
+ grid them in space and put the resulting gridded files
+ for that day in the output directory.
Note: the output directory will also be used for intermediary working
files.
@@ -154,15 +154,20 @@ def space(*args):
possible_files = os.listdir(input_path)
expected_vars = { }
all_vars = set()
- for file_name in possible_files :
+ date_time_temp = None
+ for file_name in sorted(possible_files) :
expected_vars[file_name] = general_guidebook.get_variable_names (file_name, user_requested_names=desired_variables)
all_vars.update(expected_vars[file_name])
+ date_time_temp = general_guidebook.parse_datetime_from_filename(file_name) if date_time_temp is None else date_time_temp
+
+ # figure out what our date stamp will look like
+ date_stamp = date_time_temp.strftime("%Y%m%d")
# check to make sure our intermediate file names don't exist already
for var_name in all_vars :
for suffix in io_manager.ALL_EXPECTED_SUFFIXES :
- temp_name = fbf.filename(var_name + suffix, TEMP_DATA_TYPE, shape=(space_grid_shape))
+ temp_name = fbf.filename(date_stamp + "_" + var_name + suffix, TEMP_DATA_TYPE, shape=(space_grid_shape))
if os.path.exists(os.path.join(output_path, temp_name)) :
LOG.warn ("Cannot process files because intermediate files exist in the output directory.")
return
@@ -198,24 +203,28 @@ def space(*args):
night_var_data = var_data[temp_aux_data[NIGHT_MASK_KEY]]
# space grid the data using the indexes we calculated earlier
- day_space_grid, day_density_map, day_max_depth = space_gridding.space_grid_data (grid_lon_size, grid_lat_size,
- day_var_data,
- day_lon_index, day_lat_index)
- night_space_grid, night_density_map, night_max_depth = space_gridding.space_grid_data (grid_lon_size, grid_lat_size,
- night_var_data,
- night_lon_index, night_lat_index)
+ day_space_grid, day_density_map, day_nobs, day_max_depth = space_gridding.space_grid_data (grid_lon_size, grid_lat_size,
+ day_var_data,
+ day_lon_index, day_lat_index)
+ night_space_grid, night_density_map, night_nobs, night_max_depth = space_gridding.space_grid_data (grid_lon_size, grid_lat_size,
+ night_var_data,
+ night_lon_index, night_lat_index)
# save the space grids and density info for this variable and it's density map to files
- io_manager.save_data_to_file(variable_name + io_manager.DAY_TEMP_SUFFIX, space_grid_shape, output_path,
- day_space_grid, TEMP_DATA_TYPE )
- io_manager.save_data_to_file(variable_name + io_manager.DAY_DENSITY_TEMP_SUFFIX, space_grid_shape, output_path,
+ io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.DAY_TEMP_SUFFIX, space_grid_shape, output_path,
+ day_space_grid, TEMP_DATA_TYPE )
+ io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.DAY_DENSITY_TEMP_SUFFIX, space_grid_shape, output_path,
day_density_map, TEMP_DATA_TYPE)
+ io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.DAY_NOBS_TEMP_SUFFIX, space_grid_shape, output_path,
+ day_nobs, TEMP_DATA_TYPE)
- io_manager.save_data_to_file(variable_name + io_manager.NIGHT_TEMP_SUFFIX, space_grid_shape, output_path,
- night_space_grid, TEMP_DATA_TYPE )
- io_manager.save_data_to_file(variable_name + io_manager.NIGHT_DENSITY_TEMP_SUFFIX, space_grid_shape, output_path,
+ io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.NIGHT_TEMP_SUFFIX, space_grid_shape, output_path,
+ night_space_grid, TEMP_DATA_TYPE )
+ io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.NIGHT_DENSITY_TEMP_SUFFIX, space_grid_shape, output_path,
night_density_map, TEMP_DATA_TYPE)
+ io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.NIGHT_NOBS_TEMP_SUFFIX, space_grid_shape, output_path,
+ night_nobs, TEMP_DATA_TYPE)
# make sure each file is closed when we're done with it
io_manager.close_file(full_file_path, file_object)
@@ -227,45 +236,105 @@ def space(*args):
# load the variable's density maps
var_workspace = Workspace.Workspace(dir=output_path)
- day_var_density = var_workspace[variable_name + io_manager.DAY_DENSITY_TEMP_SUFFIX][:]
- night_var_density = var_workspace[variable_name + io_manager.NIGHT_DENSITY_TEMP_SUFFIX][:]
+ day_var_density = var_workspace[date_stamp + "_" + variable_name + io_manager.DAY_DENSITY_TEMP_SUFFIX][:]
+ night_var_density = var_workspace[date_stamp + "_" + variable_name + io_manager.NIGHT_DENSITY_TEMP_SUFFIX][:]
+
+ # set up nobs arrays to accumulate into
+ day_nobs_total = numpy.zeros(space_grid_shape, dtype=TEMP_DATA_TYPE)
+ night_nobs_total = numpy.zeros(space_grid_shape, dtype=TEMP_DATA_TYPE)
# only do the day data if we have some
if numpy.sum(day_var_density) > 0 :
# load the sparse space grid
- day_var_data = var_workspace[variable_name + io_manager.DAY_TEMP_SUFFIX][:]
+ day_var_data = var_workspace[date_stamp + "_" + variable_name + io_manager.DAY_TEMP_SUFFIX][:]
# collapse the space grid
final_day_data = space_gridding.pack_space_grid(day_var_data, day_var_density)
# save the final array to an appropriately named file
- io_manager.save_data_to_file(variable_name + io_manager.DAY_SUFFIX, space_grid_shape, output_path,
+ io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.DAY_SUFFIX, space_grid_shape, output_path,
final_day_data, TEMP_DATA_TYPE, file_permissions="w")
+ # load the nobs file
+ nobs_counts = var_workspace[date_stamp + "_" + variable_name + io_manager.DAY_NOBS_TEMP_SUFFIX][:]
+
+ # collapse the nobs
+ nobs_final = numpy.sum(nobs_counts, axis=0)
+
+ # save the final nobs array to an appropriately named file
+ io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.DAY_NOBS_SUFFIX, space_grid_shape, output_path,
+ nobs_final, TEMP_DATA_TYPE, file_permissions="w")
+
else :
- LOG.warn("No day data was found for variable " + variable_name + ". Day file will not be written.")
+ LOG.warn("No day data was found for variable " + variable_name + ". Day files will not be written.")
# only do night data if we have some
if numpy.sum(night_var_density) > 0 :
# load the sparse space grid
- night_var_data = var_workspace[variable_name + io_manager.NIGHT_TEMP_SUFFIX][:]
+ night_var_data = var_workspace[date_stamp + "_" + variable_name + io_manager.NIGHT_TEMP_SUFFIX][:]
# collapse the space grid
final_night_data = space_gridding.pack_space_grid(night_var_data, night_var_density)
# save the final array to an appropriately named file
- io_manager.save_data_to_file(variable_name + io_manager.NIGHT_SUFFIX, space_grid_shape, output_path,
+ io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.NIGHT_SUFFIX, space_grid_shape, output_path,
final_night_data, TEMP_DATA_TYPE, file_permissions="w")
+ # load the nobs file
+ nobs_counts = var_workspace[date_stamp + "_" + variable_name + io_manager.NIGHT_NOBS_TEMP_SUFFIX][:]
+
+ # collapse the nobs
+ nobs_final = numpy.sum(nobs_counts, axis=0)
+
+ # save the final nobs array to an appropriately named file
+ io_manager.save_data_to_file(date_stamp + "_" + variable_name + io_manager.NIGHT_NOBS_SUFFIX, space_grid_shape, output_path,
+ nobs_final, TEMP_DATA_TYPE, file_permissions="w")
+
else :
- LOG.warn("No night data was found for variable " + variable_name + ". Night file will not be written.")
+ LOG.warn("No night data was found for variable " + variable_name + ". Night files will not be written.")
# remove the extra temporary files in the output directory
remove_suffixes = ["*" + p + "*" for p in io_manager.EXPECTED_TEMP_SUFFIXES]
remove_file_patterns(output_path, remove_suffixes)
+ def stats_day(*args):
+ """given files of daily space gridded data, calculate daily stats
+ given an input directory that contains appropriate files,
+ calculate daily stats and put the resulting gridded files
+ for that day in the output directory.
+
+ Note: the output directory will also be used for intermediary working
+ files.
+ """
+
+ # set up some of our input from the caller for easy access
+ desired_variables = list(args) if len(args) > 0 else [ ]
+ input_path = options.inputPath
+ output_path = options.outputPath
+ min_scan_angle = options.minScanAngle
+ grid_degrees = float(options.gridDegrees)
+
+ def stats_month(*args):
+ """given a month of daily space gridded data, calculate montly stats
+ given an input directory that contains appropriate files,
+ calculate monthly stats and put the resulting gridded files
+ for that month in the output directory.
+
+ Note: the output directory will also be used for intermediary working
+ files.
+ """
+
+ # set up some of our input from the caller for easy access
+ desired_variables = list(args) if len(args) > 0 else [ ]
+ input_path = options.inputPath
+ output_path = options.outputPath
+ min_scan_angle = options.minScanAngle
+ grid_degrees = float(options.gridDegrees)
+
+
+
# all the local public functions are considered part of the application, collect them up
commands.update(dict(x for x in locals().items() if x[0] not in prior))
Please sign in to comment.
Something went wrong with that request. Please try again.