Permalink
Browse files

Moved fog band creation and DNB scaling to the VIIRS frontend. Also

added fog band to the 8bit and 16bit default rescaling configuration
files. Updated glue scripts to use the new functionality.
  • Loading branch information...
1 parent c8e81d7 commit f887769d94162d99257c2abeddbcff3937c71f86 @davidh-ssec committed Nov 14, 2012
View
185 py/polar2grid/polar2grid/viirs2awips.py
@@ -16,7 +16,6 @@
from polar2grid.core import Workspace
from polar2grid.core.constants import *
from polar2grid.viirs import Frontend
-from polar2grid.core.rescale import dnb_scale
from .grids.grids import determine_grid_coverage_fbf,get_grid_info
from .remap import remap_bands
from .awips import Backend
@@ -25,7 +24,6 @@
import sys
import logging
from multiprocessing import Process
-import numpy
from glob import glob
log = logging.getLogger(__name__)
@@ -143,92 +141,6 @@ def remove_products():
for f in glob("SSEC_AWIPS_VIIRS*"):
_safe_remove(f)
-# XXX: Remove new_dnb when a method has been decided on
-# XXX: It is just temporary
-def run_prescaling(img_filepath, mode_filepath,
- new_dnb=False, fill_value=DEFAULT_FILL_VALUE):
- """A wrapper function for calling the prescaling function for dnb.
- This function will read the binary image data from ``img_filepath``
- as well as any other data that may be required to prescale the data
- correctly, such as day/night/twilight masks.
-
- :Parameters:
- img_filepath : str
- Filepath to the binary image swath data in FBF format
- (ex. ``image_I01.real4.6400.10176``).
- mode_filepath : str
- Filepath to the binary mode swath data in FBF format
- (ex. ``mode_I01.real4.6400.10176``).
- """
-
- img_attr = os.path.split(img_filepath)[1].split('.')[0]
- mode_attr = os.path.split(mode_filepath)[1].split('.')[0]
-
- # Rescale the image
- try:
- W = Workspace('.')
- img = getattr(W, img_attr)
- data = img.copy()
- log.debug("Data min: %f, Data max: %f" % (data.min(),data.max()))
- except StandardError:
- log.error("Could not open img file %s" % img_filepath)
- log.debug("Files matching %r" % glob(img_attr + "*"))
- raise
-
- scale_kwargs = {
- 'new_dnb':new_dnb # XXX
- }
- try:
- mode_mask = getattr(W, mode_attr)
- # Only add parameters if they're useful
- if mode_mask.shape == data.shape:
- log.debug("Adding mode mask to rescaling arguments")
- HIGH = 100
- LOW = 88
- MIXED_STEP = HIGH - LOW
- good_mask = ~((img == fill_value) | (mode_mask == fill_value))
- scale_kwargs["night_mask"] = (mode_mask >= HIGH) & good_mask
- scale_kwargs["day_mask"] = (mode_mask <= LOW ) & good_mask
- scale_kwargs["mixed_mask"] = []
- steps = range(LOW, HIGH+1, MIXED_STEP)
- if steps[-1] >= HIGH: steps[-1] = HIGH
- steps = zip(steps, steps[1:])
- for i,j in steps:
- log.debug("Processing step %d to %d" % (i,j))
- tmp = (mode_mask > i) & (mode_mask < j) & good_mask
- if numpy.sum(tmp) > 0:
- log.debug("Adding step %d to %d" % (i,j))
- scale_kwargs["mixed_mask"].append(tmp)
- del tmp
- del good_mask
-
- else:
- log.error("Mode shape is different than the data's shape (%s) vs (%s)" % (mode_mask.shape, data.shape))
- raise ValueError("Mode shape is different than the data's shape (%s) vs (%s)" % (mode_mask.shape, data.shape))
- except StandardError:
- log.error("Could not open mode mask file %s" % mode_filepath)
- log.debug("Files matching %r" % glob(mode_attr + "*"))
- raise
-
- try:
- rescaled_data = dnb_scale(data,
- **scale_kwargs)
- if (logging.getLogger('').handlers[0].level or 0) <= logging.DEBUG:
- log.debug("Data min: %f, Data max: %f" % (
- rescaled_data[ rescaled_data != fill_value ].min(),
- rescaled_data[ rescaled_data != fill_value ].max()
- ))
- rows,cols = rescaled_data.shape
- fbf_swath_var = "prescale_dnb"
- fbf_swath = "./%s.real4.%d.%d" % (fbf_swath_var, cols, rows)
- rescaled_data.tofile(fbf_swath)
- except StandardError:
- log.error("Unexpected error while rescaling data")
- log.debug("Rescaling error:", exc_info=1)
- raise
-
- return fbf_swath
-
def create_grid_jobs(sat, instrument, bands, fbf_lat, fbf_lon, backend,
forced_grids=None):
"""
@@ -292,60 +204,6 @@ def create_grid_jobs(sat, instrument, bands, fbf_lat, fbf_lon, backend,
return grid_jobs
-def create_pseudobands(bands, fill_value=DEFAULT_FILL_VALUE):
- # Fog pseudo-band
- if (BKIND_I, BID_04) in bands and (BKIND_I, BID_05) in bands:
- log.info("Creating IFOG pseudo band...")
- try:
- W = Workspace('.')
- mode_attr = bands[(BKIND_I,BID_05)]["fbf_mode"].split(".")[0]
- mode_data = getattr(W, mode_attr)
- night_mask = mode_data >= 100
- del mode_data
- except StandardError:
- log.error("Error getting mode data while creating FOG band")
- log.debug("Mode error:", exc_info=1)
- return
-
- num_night_points = numpy.sum(night_mask)
- if num_night_points == 0:
- # We only create fog mask if theres nighttime data
- log.info("No night data found to create FOG band for")
- return
- log.debug("Creating FOG band for %s nighttime data points" % num_night_points)
-
- fog_dict = {
- "kind" : BKIND_I,
- "band" : BID_FOG,
- "data_kind" : DKIND_FOG,
- "remap_data_as" : DKIND_BTEMP,
- "rows_per_scan" : bands[(BKIND_I, BID_05)]["rows_per_scan"],
- "fbf_img" : "image_IFOG.%s" % ".".join(bands[(BKIND_I, BID_05)]["fbf_img"].split(".")[1:]),
- "fbf_mode" : bands[(BKIND_I, BID_05)]["fbf_mode"],
- "swath_scans" : bands[(BKIND_I, BID_05)]["swath_scans"],
- "swath_rows" : bands[(BKIND_I, BID_05)]["swath_rows"],
- "swath_cols" : bands[(BKIND_I, BID_05)]["swath_cols"]
- }
- try:
- W = Workspace(".")
- i5_attr = bands[(BKIND_I, BID_05)]["fbf_img"].split(".")[0]
- i4_attr = bands[(BKIND_I, BID_04)]["fbf_img"].split(".")[0]
- i5 = getattr(W, i5_attr)
- i4 = getattr(W, i4_attr)
- fog_map = numpy.memmap(fog_dict["fbf_img"],
- dtype=numpy.float32,
- mode="w+",
- shape=i5.shape
- )
- numpy.subtract(i5, i4, fog_map)
- fog_map[ (~night_mask) | (i5 == fill_value) | (i4 == fill_value) ] = fill_value
- del fog_map
- del i5,i4
- bands[(BKIND_I, BID_FOG)] = fog_dict
- except StandardError:
- log.error("Error creating Fog pseudo band")
- log.debug("Fog creation error:", exc_info=1)
-
def process_data_sets(filepaths,
fornav_D=None, fornav_d=None,
forced_grid=None,
@@ -368,7 +226,13 @@ def process_data_sets(filepaths,
# Extract Swaths
log.info("Extracting swaths...")
try:
- meta_data = frontend.make_swaths(filepaths, cut_bad=True)
+ meta_data = frontend.make_swaths(
+ filepaths,
+ scale_dnb=True,
+ new_dnb=new_dnb,
+ create_fog=True,
+ cut_bad=True
+ )
# Let's be lazy and give names to the 'global' viirs info
sat = meta_data["sat"]
@@ -384,42 +248,9 @@ def process_data_sets(filepaths,
status_to_return |= STATUS_FRONTEND_FAIL
return status_to_return
- # Create pseudo-bands
- # FIXME: Move pseudoband creation to the frontend
- try:
- if create_pseudo:
- create_pseudobands(bands)
- except StandardError:
- log.error("Pseudo band creation failed")
- log.debug("Pseudo band error:", exc_info=1)
- status_to_return |= STATUS_FRONTEND_FAIL
- return status_to_return
-
- # Do any pre-remapping rescaling
- # FIXME: Move DNB scaling to the frontend
- for (band_kind, band_id),band_job in bands.items():
- if band_kind != BKIND_DNB:
- # It takes too long to read in the data, so just skip it
- band_job["fbf_swath"] = band_job["fbf_img"]
- continue
-
- log.info("Prescaling data before remapping...")
- try:
- fbf_swath = run_prescaling(
- band_job["fbf_img"],
- band_job["fbf_mode"],
- new_dnb=new_dnb # XXX
- )
- band_job["fbf_swath"] = fbf_swath
- except StandardError:
- log.error("Unexpected error prescaling %s, removing..." % band_job["band_name"])
- log.debug("Prescaling error:", exc_info=1)
- del bands[(band_kind, band_id)]
- status_to_return |= STATUS_FRONTEND_FAIL
-
if len(bands) == 0:
log.error("No more bands to process, quitting...")
- return status_to_return or UNKNOWN_FAIL
+ return status_to_return or STATUS_UNKNOWN_FAIL
# Determine grid
try:
View
45 py/polar2grid/polar2grid/viirs2binary.py
@@ -16,7 +16,7 @@
from polar2grid.core import Workspace
from polar2grid.core.constants import *
from polar2grid.viirs import Frontend
-from .viirs2awips import run_prescaling,_safe_remove,create_grid_jobs,create_pseudobands
+from .viirs2awips import _safe_remove,create_grid_jobs
from .remap import remap_bands
from .binary import Backend
@@ -127,7 +127,13 @@ def process_data_sets(filepaths,
# Extract Swaths
log.info("Extracting swaths...")
try:
- meta_data = frontend.make_swaths(filepaths, cut_bad=True)
+ meta_data = frontend.make_swaths(
+ filepaths,
+ scale_dnb=True,
+ new_dnb=new_dnb,
+ create_fog=True,
+ cut_bad=True
+ )
# Let's be lazy and give names to the 'global' viirs info
sat = meta_data["sat"]
@@ -143,42 +149,9 @@ def process_data_sets(filepaths,
status_to_return |= STATUS_FRONTEND_FAIL
return status_to_return
- # Create pseudo-bands
- # FIXME: Move pseudoband creation to the frontend
- try:
- if create_pseudo:
- create_pseudobands(bands)
- except StandardError:
- log.error("Pseudo band creation failed")
- log.debug("Pseudo band error:", exc_info=1)
- status_to_return |= STATUS_FRONTEND_FAIL
- return status_to_return
-
- # Do any pre-remapping rescaling
- # FIXME: Move DNB scaling to the frontend
- for (band_kind, band_id),band_job in bands.items():
- if band_kind != BKIND_DNB:
- # It takes too long to read in the data, so just skip it
- band_job["fbf_swath"] = band_job["fbf_img"]
- continue
-
- log.info("Prescaling data before remapping...")
- try:
- fbf_swath = run_prescaling(
- band_job["fbf_img"],
- band_job["fbf_mode"],
- new_dnb=new_dnb # XXX
- )
- band_job["fbf_swath"] = fbf_swath
- except StandardError:
- log.error("Unexpected error prescaling %s, removing..." % band_job["band_name"])
- log.debug("Prescaling error:", exc_info=1)
- del bands[(band_kind, band_id)]
- status_to_return |= STATUS_FRONTEND_FAIL
-
if len(bands) == 0:
log.error("No more bands to process, quitting...")
- return status_to_return or UNKNOWN_FAIL
+ return status_to_return or STATUS_UNKNOWN_FAIL
# Determine grid
try:
View
45 py/polar2grid/polar2grid/viirs2gtiff.py
@@ -16,7 +16,7 @@
from polar2grid.core import Workspace
from polar2grid.core.constants import *
from polar2grid.viirs import Frontend
-from .viirs2awips import run_prescaling,_safe_remove,create_grid_jobs,create_pseudobands
+from .viirs2awips import _safe_remove,create_grid_jobs
from .remap import remap_bands
from .gtiff_backend import Backend,_bits_to_etype
@@ -131,7 +131,13 @@ def process_data_sets(filepaths,
# Extract Swaths
log.info("Extracting swaths...")
try:
- meta_data = frontend.make_swaths(filepaths, cut_bad=True)
+ meta_data = frontend.make_swaths(
+ filepaths,
+ scale_dnb=True,
+ new_dnb=new_dnb,
+ create_fog=True,
+ cut_bad=True
+ )
# Let's be lazy and give names to the 'global' viirs info
sat = meta_data["sat"]
@@ -147,42 +153,9 @@ def process_data_sets(filepaths,
status_to_return |= STATUS_FRONTEND_FAIL
return status_to_return
- # Create pseudo-bands
- # FIXME: Move pseudoband creation to the frontend
- try:
- if create_pseudo:
- create_pseudobands(bands)
- except StandardError:
- log.error("Pseudo band creation failed")
- log.debug("Pseudo band error:", exc_info=1)
- status_to_return |= STATUS_FRONTEND_FAIL
- return status_to_return
-
- # Do any pre-remapping rescaling
- # FIXME: Move DNB scaling to the frontend
- for (band_kind, band_id),band_job in bands.items():
- if band_kind != BKIND_DNB:
- # It takes too long to read in the data, so just skip it
- band_job["fbf_swath"] = band_job["fbf_img"]
- continue
-
- log.info("Prescaling data before remapping...")
- try:
- fbf_swath = run_prescaling(
- band_job["fbf_img"],
- band_job["fbf_mode"],
- new_dnb=new_dnb # XXX
- )
- band_job["fbf_swath"] = fbf_swath
- except StandardError:
- log.error("Unexpected error prescaling %s, removing..." % band_job["band_name"])
- log.debug("Prescaling error:", exc_info=1)
- del bands[(band_kind, band_id)]
- status_to_return |= STATUS_FRONTEND_FAIL
-
if len(bands) == 0:
log.error("No more bands to process, quitting...")
- return status_to_return or UNKNOWN_FAIL
+ return status_to_return or STATUS_UNKNOWN_FAIL
# Determine grid
try:
View
77 py/polar2grid_core/polar2grid/core/rescale.py
@@ -130,76 +130,6 @@ def fog_scale(img, m, b, floor, floor_val, ceil, ceil_val, fill_in=DEFAULT_FILL_
img[mask] = fill_out
return img
-def dnb_scale(img, day_mask=None, mixed_mask=None, night_mask=None,
- new_dnb=False, fill_in=DEFAULT_FILL_IN, fill_out=DEFAULT_FILL_OUT):
- """
- This scaling method uses histogram equalization to flatten the image
- levels across the day and night masks. The masks are expected to be
- passed as `day_mask`, `mixed_mask`, and `night_mask` keyword arguments.
- They should be `None` if not used.
-
- `mixed_mask` should be a python list of masks for any number of desired
- 'levels' of distinction in the mixed data region. Each mask in
- `mixed_mask` should have the same shape as the data and day and night
- masks.
-
- A histogram equalization will be performed separately for each of the
- three masks that's present (not `None`).
- """
-
- #log.debug("Running 'dnb_scale'...")
- if new_dnb:
- log.debug("Running NEW DNB scaling...")
- else:
- log.debug("Running OLD DNB scaling...")
-
- # a way to hang onto our result
- # because the equalization is done in place, this is needed so the input data isn't corrupted
- img_result = img.copy()
-
- # build a mask of all the valid data in the image
- allValidData = numpy.zeros(img.shape, dtype=bool) # by default we don't believe any data is good
- allValidData = allValidData | day_mask if day_mask is not None else allValidData
- allValidData = allValidData | night_mask if night_mask is not None else allValidData
- if mixed_mask is not None :
- for mixed in mixed_mask :
- allValidData = allValidData | mixed
-
- """
- # TEMP, this is for testing tiled histogram equalization across the whole image
- _local_histogram_equalization(img, allValidData, local_radius_px=200)
- """
-
- if day_mask is not None and (numpy.sum(day_mask) > 0) :
- log.debug(" scaling DNB in day mask")
- temp_image = img.copy()
- if new_dnb:
- _local_histogram_equalization(temp_image, day_mask, valid_data_mask=allValidData, local_radius_px=200)
- else:
- _histogram_equalization(temp_image, day_mask)
- img_result[day_mask] = temp_image[day_mask]
-
- if mixed_mask is not None and (len(mixed_mask) > 0) :
- log.debug(" scaling DNB in twilight mask")
- for mask in mixed_mask:
- temp_image = img.copy()
- if new_dnb:
- _local_histogram_equalization(temp_image, mask, valid_data_mask=allValidData, local_radius_px=100)
- else:
- _histogram_equalization(temp_image, mask)
- img_result[mask] = temp_image[mask]
-
- if night_mask is not None and (numpy.sum(night_mask) > 0) :
- log.debug(" scaling DNB in night mask")
- temp_image = img.copy()
- if new_dnb:
- _local_histogram_equalization(temp_image, night_mask, valid_data_mask=allValidData, local_radius_px=200)
- else:
- _histogram_equalization(temp_image, night_mask)
- img_result[night_mask] = temp_image[night_mask]
-
- return img_result
-
# FUTURE, this version of histogram equalization is no longer used and could be removed
def _histogram_equalization (data, mask_to_equalize, number_of_bins=1000, std_mult_cutoff=4.0, do_zerotoone_normalization=True, local_radius_px=None, clip_limit=None) :
"""
@@ -586,10 +516,11 @@ def default_config_dir(self):
return os.path.split(os.path.realpath(__file__))[0]
_known_rescale_kinds = {
- 'sqrt' : sqrt_scale,
+ 'sqrt' : sqrt_scale,
'linear' : linear_scale,
- 'raw' : passive_scale,
- 'btemp' : bt_scale
+ 'raw' : passive_scale,
+ 'btemp' : bt_scale,
+ 'fog' : fog_scale
}
@property
def known_rescale_kinds(self):
View
11 py/polar2grid_core/polar2grid/core/rescale_configs/rescale.16bit.conf
@@ -1,9 +1,10 @@
# Example rescaling configuration file for polar2grid
# See documentation site for possible rescaling kinds and their arguments
# sat, instrument, kind, band, data_kind, rescale_kind, *rescaling args
-npp, viirs, i, 01, reflectance, sqrt, 100.0, 6553.5
-npp, viirs, i, 02, reflectance, sqrt, 100.0, 6553.5
-npp, viirs, i, 03, reflectance, sqrt, 100.0, 6553.5
-npp, viirs, i, 04, btemp, btemp, 242.0, 168960, 512, 107789.66, 259.23
-npp, viirs, i, 05, btemp, btemp, 242.0, 168960, 512, 107789.66, 259.23
+npp, viirs, i, 01, reflectance, sqrt, 100.0, 6553.5
+npp, viirs, i, 02, reflectance, sqrt, 100.0, 6553.5
+npp, viirs, i, 03, reflectance, sqrt, 100.0, 6553.5
+npp, viirs, i, 04, btemp, btemp, 242.0, 168960, 512, 107789.66, 259.23
+npp, viirs, i, 05, btemp, btemp, 242.0, 168960, 512, 107789.66, 259.23
npp, viirs, dnb, none, radiance, linear, 65535.0, 0.0
+npp, viirs, i, fog, fog, fog, 2570.0, 26985.0, 5, 4, 52685, 52942
View
5 py/polar2grid_core/polar2grid/core/rescale_configs/rescale.8bit.conf
@@ -4,6 +4,7 @@
npp, viirs, i, 01, reflectance, sqrt, 100.0, 25.5
npp, viirs, i, 02, reflectance, sqrt, 100.0, 25.5
npp, viirs, i, 03, reflectance, sqrt, 100.0, 25.5
-npp, viirs, i, 04, btemp, btemp, 242.0, 660, 2, 418, 1
-npp, viirs, i, 05, btemp, btemp, 242.0, 660, 2, 418, 1
+npp, viirs, i, 04, btemp, btemp, 242.0, 660, 2, 418, 1
+npp, viirs, i, 05, btemp, btemp, 242.0, 660, 2, 418, 1
npp, viirs, dnb, none, radiance, linear, 255.0, 0.0
+npp, viirs, i, fog, fog, fog, 10.0, 105.0, 5, 4, 205, 206
View
186 py/polar2grid_viirs/polar2grid/viirs/prescale.py
@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""Functions for prescaling data. Scaling that occurs after the raw
+data is loaded, but before the data is provided to the user. These
+functions should not create any new bands (see `polar2grid.viirs.pseudo`),
+but only modify data.
+
+There is no interface currently defined for these functions; arguments and
+returned values are up to the rest of the VIIRs frontend calling them.
+
+:author: David Hoese (davidh)
+:contact: david.hoese@ssec.wisc.edu
+:organization: Space Science and Engineering Center (SSEC)
+:copyright: Copyright (c) 2012 University of Wisconsin SSEC. All rights reserved.
+:date: Nov 2012
+:license: GNU GPLv3
+"""
+
+import numpy
+from polar2grid.core.rescale import _local_histogram_equalization,_histogram_equalization
+from polar2grid.core.constants import DEFAULT_FILL_VALUE
+from polar2grid.core import Workspace
+
+import os
+import sys
+import glob
+import logging
+
+log = logging.getLogger(__name__)
+
+def dnb_scale(img, day_mask=None, mixed_mask=None, night_mask=None,
+ new_dnb=False):
+ """
+ This scaling method uses histogram equalization to flatten the image
+ levels across the day and night masks. The masks are expected to be
+ passed as `day_mask`, `mixed_mask`, and `night_mask` keyword arguments.
+ They should be `None` if not used.
+
+ `mixed_mask` should be a python list of masks for any number of desired
+ 'levels' of distinction in the mixed data region. Each mask in
+ `mixed_mask` should have the same shape as the data and day and night
+ masks.
+
+ A histogram equalization will be performed separately for each of the
+ three masks that's present (not `None`).
+ """
+
+ #log.debug("Running 'dnb_scale'...")
+ if new_dnb:
+ log.debug("Running NEW DNB scaling...")
+ else:
+ log.debug("Running OLD DNB scaling...")
+
+ # a way to hang onto our result
+ # because the equalization is done in place, this is needed so the input data isn't corrupted
+ img_result = img.copy()
+
+ # build a mask of all the valid data in the image
+ allValidData = numpy.zeros(img.shape, dtype=bool) # by default we don't believe any data is good
+ allValidData = allValidData | day_mask if day_mask is not None else allValidData
+ allValidData = allValidData | night_mask if night_mask is not None else allValidData
+ if mixed_mask is not None :
+ for mixed in mixed_mask :
+ allValidData = allValidData | mixed
+
+ """
+ # TEMP, this is for testing tiled histogram equalization across the whole image
+ _local_histogram_equalization(img, allValidData, local_radius_px=200)
+ """
+
+ if day_mask is not None and (numpy.sum(day_mask) > 0) :
+ log.debug(" scaling DNB in day mask")
+ temp_image = img.copy()
+ if new_dnb:
+ _local_histogram_equalization(temp_image, day_mask, valid_data_mask=allValidData, local_radius_px=200)
+ else:
+ _histogram_equalization(temp_image, day_mask)
+ img_result[day_mask] = temp_image[day_mask]
+
+ if mixed_mask is not None and (len(mixed_mask) > 0) :
+ log.debug(" scaling DNB in twilight mask")
+ for mask in mixed_mask:
+ temp_image = img.copy()
+ if new_dnb:
+ _local_histogram_equalization(temp_image, mask, valid_data_mask=allValidData, local_radius_px=100)
+ else:
+ _histogram_equalization(temp_image, mask)
+ img_result[mask] = temp_image[mask]
+
+ if night_mask is not None and (numpy.sum(night_mask) > 0) :
+ log.debug(" scaling DNB in night mask")
+ temp_image = img.copy()
+ if new_dnb:
+ _local_histogram_equalization(temp_image, night_mask, valid_data_mask=allValidData, local_radius_px=200)
+ else:
+ _histogram_equalization(temp_image, night_mask)
+ img_result[night_mask] = temp_image[night_mask]
+
+ return img_result
+
+# XXX: Remove new_dnb when a method has been decided on
+# XXX: It is just temporary
+def run_dnb_scale(img_filepath, mode_filepath,
+ new_dnb=False, fill_value=DEFAULT_FILL_VALUE):
+ """A wrapper function for calling the prescaling function for dnb.
+ This function will read the binary image data from ``img_filepath``
+ as well as any other data that may be required to prescale the data
+ correctly, such as day/night/twilight masks.
+
+ :Parameters:
+ img_filepath : str
+ Filepath to the binary image swath data in FBF format
+ (ex. ``image_I01.real4.6400.10176``).
+ mode_filepath : str
+ Filepath to the binary mode swath data in FBF format
+ (ex. ``mode_I01.real4.6400.10176``).
+ """
+
+ img_attr = os.path.split(img_filepath)[1].split('.')[0]
+ mode_attr = os.path.split(mode_filepath)[1].split('.')[0]
+
+ # Rescale the image
+ try:
+ W = Workspace('.')
+ img = getattr(W, img_attr)
+ data = img.copy()
+ log.debug("Data min: %f, Data max: %f" % (data.min(),data.max()))
+ except StandardError:
+ log.error("Could not open img file %s" % img_filepath)
+ log.debug("Files matching %r" % glob(img_attr + "*"))
+ raise
+
+ scale_kwargs = {
+ 'new_dnb':new_dnb # XXX
+ }
+ try:
+ mode_mask = getattr(W, mode_attr)
+ # Only add parameters if they're useful
+ if mode_mask.shape == data.shape:
+ log.debug("Adding mode mask to rescaling arguments")
+ HIGH = 100
+ LOW = 88
+ MIXED_STEP = HIGH - LOW
+ good_mask = ~((img == fill_value) | (mode_mask == fill_value))
+ scale_kwargs["night_mask"] = (mode_mask >= HIGH) & good_mask
+ scale_kwargs["day_mask"] = (mode_mask <= LOW ) & good_mask
+ scale_kwargs["mixed_mask"] = []
+ steps = range(LOW, HIGH+1, MIXED_STEP)
+ if steps[-1] >= HIGH: steps[-1] = HIGH
+ steps = zip(steps, steps[1:])
+ for i,j in steps:
+ log.debug("Processing step %d to %d" % (i,j))
+ tmp = (mode_mask > i) & (mode_mask < j) & good_mask
+ if numpy.sum(tmp) > 0:
+ log.debug("Adding step %d to %d" % (i,j))
+ scale_kwargs["mixed_mask"].append(tmp)
+ del tmp
+ del good_mask
+
+ else:
+ log.error("Mode shape is different than the data's shape (%s) vs (%s)" % (mode_mask.shape, data.shape))
+ raise ValueError("Mode shape is different than the data's shape (%s) vs (%s)" % (mode_mask.shape, data.shape))
+ except StandardError:
+ log.error("Could not open mode mask file %s" % mode_filepath)
+ log.debug("Files matching %r" % glob(mode_attr + "*"))
+ raise
+
+ try:
+ rescaled_data = dnb_scale(data,
+ **scale_kwargs)
+ if (logging.getLogger('').handlers[0].level or 0) <= logging.DEBUG:
+ log.debug("Data min: %f, Data max: %f" % (
+ rescaled_data[ rescaled_data != fill_value ].min(),
+ rescaled_data[ rescaled_data != fill_value ].max()
+ ))
+ rows,cols = rescaled_data.shape
+ fbf_swath_var = "prescale_dnb"
+ fbf_swath = "./%s.real4.%d.%d" % (fbf_swath_var, cols, rows)
+ rescaled_data.tofile(fbf_swath)
+ except StandardError:
+ log.error("Unexpected error while rescaling data")
+ log.debug("Rescaling error:", exc_info=1)
+ raise
+
+ return fbf_swath
+
View
78 py/polar2grid_viirs/polar2grid/viirs/pseudo.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""Functions to create 'pseudo' or 'synthetic' bands from raw VIIRS data.
+
+There is no interface currently defined for these functions; arguments and
+returned values are up to the rest of the VIIRS frontend calling them.
+
+:author: David Hoese (davidh)
+:contact: david.hoese@ssec.wisc.edu
+:organization: Space Science and Engineering Center (SSEC)
+:copyright: Copyright (c) 2012 University of Wisconsin SSEC. All rights reserved.
+:date: Nov 2012
+:license: GNU GPLv3
+"""
+
+import numpy
+from polar2grid.core.constants import *
+from polar2grid.core import Workspace
+
+import os
+import sys
+import logging
+
+log = logging.getLogger(__name__)
+
+def create_fog_band(bands, fill_value=DEFAULT_FILL_VALUE):
+ # Fog pseudo-band
+ if (BKIND_I, BID_04) in bands and (BKIND_I, BID_05) in bands:
+ log.info("Creating IFOG pseudo band...")
+ try:
+ W = Workspace('.')
+ mode_attr = bands[(BKIND_I,BID_05)]["fbf_mode"].split(".")[0]
+ mode_data = getattr(W, mode_attr)
+ night_mask = mode_data >= 100
+ del mode_data
+ except StandardError:
+ log.error("Error getting mode data while creating FOG band")
+ log.debug("Mode error:", exc_info=1)
+ return
+
+ num_night_points = numpy.sum(night_mask)
+ if num_night_points == 0:
+ # We only create fog mask if theres nighttime data
+ log.info("No night data found to create FOG band for")
+ return
+ log.debug("Creating FOG band for %s nighttime data points" % num_night_points)
+
+ fog_dict = {
+ "kind" : BKIND_I,
+ "band" : BID_FOG,
+ "data_kind" : DKIND_FOG,
+ "remap_data_as" : DKIND_BTEMP,
+ "rows_per_scan" : bands[(BKIND_I, BID_05)]["rows_per_scan"],
+ "fbf_img" : "image_IFOG.%s" % ".".join(bands[(BKIND_I, BID_05)]["fbf_img"].split(".")[1:]),
+ "fbf_mode" : bands[(BKIND_I, BID_05)]["fbf_mode"],
+ "swath_scans" : bands[(BKIND_I, BID_05)]["swath_scans"],
+ "swath_rows" : bands[(BKIND_I, BID_05)]["swath_rows"],
+ "swath_cols" : bands[(BKIND_I, BID_05)]["swath_cols"]
+ }
+ try:
+ W = Workspace(".")
+ i5_attr = bands[(BKIND_I, BID_05)]["fbf_img"].split(".")[0]
+ i4_attr = bands[(BKIND_I, BID_04)]["fbf_img"].split(".")[0]
+ i5 = getattr(W, i5_attr)
+ i4 = getattr(W, i4_attr)
+ fog_map = numpy.memmap(fog_dict["fbf_img"],
+ dtype=numpy.float32,
+ mode="w+",
+ shape=i5.shape
+ )
+ numpy.subtract(i5, i4, fog_map)
+ fog_map[ (~night_mask) | (i5 == fill_value) | (i4 == fill_value) ] = fill_value
+ del fog_map
+ del i5,i4
+ bands[(BKIND_I, BID_FOG)] = fog_dict
+ except StandardError:
+ log.error("Error creating Fog pseudo band")
+ log.debug("Fog creation error:", exc_info=1)
View
43 py/polar2grid_viirs/polar2grid/viirs/viirs_imager_to_swath.py
@@ -16,7 +16,9 @@
__docformat__ = "restructuredtext en"
from .viirs_guidebook import file_info,geo_info,read_file_info,read_geo_info
-from polar2grid.core.constants import SAT_NPP,INST_VIIRS
+from .prescale import run_dnb_scale
+from .pseudo import create_fog_band
+from polar2grid.core.constants import SAT_NPP,INST_VIIRS,BKIND_DNB,NOT_APPLICABLE
from polar2grid.core import roles
import numpy
@@ -551,7 +553,44 @@ def __init__(self):
pass
def make_swaths(self, *args, **kwargs):
- return make_swaths(*args, **kwargs)
+ scale_dnb = kwargs.pop("scale_dnb", False)
+ new_dnb = kwargs.pop("new_dnb", False)
+ create_fog = kwargs.pop("create_fog", False)
+
+ meta_data = make_swaths(*args, **kwargs)
+ bands = meta_data["bands"]
+
+ # These steps used to be part of the glue scripts
+ # Due to laziness they are just called as separate functions here
+ if create_fog:
+ try:
+ create_fog_band(bands)
+ except StandardError:
+ log.error("Fog band creation failed")
+ raise
+
+ # These steps used to be part of the glue scripts
+ # Due to laziness they are just called as separate functions here
+ for (band_kind, band_id),band_job in bands.items():
+ if band_kind != BKIND_DNB or not scale_dnb:
+ # We don't need to scale non-DNB data
+ band_job["fbf_swath"] = band_job["fbf_img"]
+ continue
+
+ log.info("Prescaling DNB data...")
+ try:
+ fbf_swath = run_dnb_scale(
+ band_job["fbf_img"],
+ band_job["fbf_mode"],
+ new_dnb=new_dnb # XXX
+ )
+ band_job["fbf_swath"] = fbf_swath
+ except StandardError:
+ log.error("Unexpected error DNB, removing job...")
+ log.debug("DNB scaling error:", exc_info=1)
+ del bands[(band_kind, band_id)]
+
+ return meta_data
def main():
import optparse

0 comments on commit f887769

Please sign in to comment.