Skip to content

Commit

Permalink
#126 Mosaic.median() uses Layers and works well
Browse files Browse the repository at this point in the history
  • Loading branch information
@akorosov
akorosov committed Jun 18, 2015
1 parent 75cc64c commit 51e2d8e
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 65 deletions.
106 changes: 42 additions & 64 deletions nansat/mosaic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@ class Layer:
''' Small class to get mask and arrays from many bands '''
def __init__(self, fileName, bands=[1],
opener=Nansat, maskName='mask',
doReproject=True, period=(None,None)):
doReproject=True, eResampleAlg=0,
period=(None,None)):
''' Set parameters of processing '''
self.fileName = fileName
self.bands = bands
Expand All @@ -97,12 +98,13 @@ def __init__(self, fileName, bands=[1],
self.doReproject = doReproject
self.period = period
self.n = None
self.eResampleAlg = eResampleAlg

def make_nansat_object(self, domain=None):
''' Open self.fileName with self.opener '''
self.n = self.opener(self.fileName)
if self.doReproject:
self.n.reproject(domain)
self.n.reproject(domain, eResampleAlg=self.eResampleAlg)

def within_period(self):
''' Test if given file is within period of time '''
Expand Down Expand Up @@ -160,37 +162,6 @@ class Mosaic(Nansat):
Mosaic inherits everything from Nansat
'''

# default parameters
nClass = Nansat
eResampleAlg = 0
period = None, None
threads = 1
maskName = 'mask'
doReproject = True
bandIDs = [1]
mapper = 'mosaic'

def _set_defaults(self, idict):
'''Check input params and set defaut values
Look throught default parameters (in self) and given parameters (dict)
and paste value from input if the key matches
Parameters
----------
idict : dictionary
parameter names and values
Modifies
---------
self
'''
for key in idict:
if hasattr(self, key):
setattr(self, key, idict[key])


def average(self, files=[], bands=[1], doReproject=True, maskName='mask',
opener=Nansat, threads=1, eResampleAlg=0, period=(None,None)):
'''Memory-friendly, multithreaded mosaicing(averaging) of input files
Expand Down Expand Up @@ -246,8 +217,8 @@ def average(self, files=[], bands=[1], doReproject=True, maskName='mask',

# create list of layers
domain = Nansat(domain=self)
layers = [Layer(ifile, bands, opener, maskName, doReproject, period)
for ifile in files]
layers = [Layer(ifile, bands, opener, maskName, doReproject,
eResampleAlg, period) for ifile in files]

# get desired shape
dstShape = self.shape()
Expand Down Expand Up @@ -308,7 +279,9 @@ def average(self, files=[], bands=[1], doReproject=True, maskName='mask',
self.add_band(array=stdMat[bi], parameters=bandsMeta[bi])


def _get_cube(self, files, band):
def _get_cube(self, files, band, doReproject, maskName, opener,
eResampleAlg,
period):
'''Make cube with data from one band of input files
Open files, reproject, get band, insert into cube
Expand All @@ -323,11 +296,18 @@ def _get_cube(self, files, band):
Should we reproject input files?
maskName : string
Name of the mask in the input file
opener : class
Nansat or any Nansat child to open input image
eResampleAlg : int
parameter for Nansat.reproject()
period : tuple
valid (start_date, end_date) or (None, None)
Returns:
--------
dataCube : Numpy 3D array with bands
mask : Numpy array with L2-mask
dataCube : Numpy 3D array with bands
mask : Numpy array with L2-mask
metadata : dict with band metadata
'''
# preallocate 3D cube and mask
self.logger.debug('Allocating 3D cube')
Expand All @@ -337,32 +317,31 @@ def _get_cube(self, files, band):
# for all input files
for i, f in enumerate(files):
self.logger.info('Processing %s' % f)
layer = Layer(f, [band], opener, maskName, doReproject, eResampleAlg, period)

# get image and mask
n, mask = self._get_layer(f)
if n is None:
# get nansat from the input Layer
layer.make_nansat_object(domain)
# if not in the period, quit
if not layer.within_period():
continue
# get band from input image
a = None
try:
a = n[band].astype('float32')
except:
self.logger.error('%s is not in %s' % (band, n.fileName))
if a is not None:
# mask invalid data
a[mask <= 2] = np.nan
# get mask
mask = layer.get_mask_array()
# get arrays with data
bandArray = layer.n[band]
# get metadata
bandMetadata = layer.n.get_metadata(bandID=band)

# add band to the cube
dataCube[i, :, :] = a
dataCube[i, :, :] = bandArray

# add data to mask matrix (maximum of 0, 1, 2, 64)
maskMat[0, :, :] = mask
maskMat[1, :, :] = maskMat.max(0)

return dataCube, maskMat.max(0)
return dataCube, maskMat.max(0), bandMetadata

def median(self, files=[], bands=[1], doReproject=True, maskName='mask',
**kwargs):
opener=Nansat, threads=1, eResampleAlg=0, period=(None,None)):
'''Calculate median of input bands
Memory and CPU greedy method. Generates 3D cube from bands of
Expand Down Expand Up @@ -391,25 +370,23 @@ def median(self, files=[], bands=[1], doReproject=True, maskName='mask',
self.logger.error('No input files given!')
return

# modify default values
self.bandIDs = bands
self.doReproject = doReproject
self.maskName = maskName
self._set_defaults(kwargs)

lastN = self._get_layer_image(files[-1])
# add medians of all bands
for band in bands:
bandCube, mask = self._get_cube(files, band)
bandMedian = st.nanmedian(bandCube, axis=0)
cube, mask, metadata = self._get_cube(files, band,
doReproject,
maskName,
opener,
eResampleAlg,
period)
median = st.nanmedian(cube, axis=0)

# get metadata of this band from the last image
parameters = lastN.get_metadata(bandID=band)
# add band and std with metadata
self.add_band(array=bandMedian, parameters=parameters)
self.add_band(array=median, parameters=metadata)

self.add_band(array=mask, parameters={'name': 'mask'})

"""
def latest(self, files=[], bands=[1], doReproject=True, maskName='mask',
**kwargs):
'''Mosaic by adding the latest image on top without averaging
Expand Down Expand Up @@ -541,3 +518,4 @@ def latest(self, files=[], bands=[1], doReproject=True, maskName='mask',
self.add_band(array=maxIndex, parameters={'name': 'date_index',
'values': timeString})
"""
22 changes: 21 additions & 1 deletion nansat/tests/test_mosaic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,34 @@ def test_average(self):
L_469 = mo['L_469']

tmpfilename = os.path.join(ntd.tmp_data_path,
'mosaic_export.nc')
'mosaic_average_export.nc')
bands = {
'L_645' : {'type': '>i1'},
'L_555' : {'type': '>i1'},
'L_469' : {'type': '>i1'},
}
mo.export2thredds(tmpfilename, bands)

def test_median(self):
mo = Mosaic(domain=self.domain)
mo.median([self.test_file_gcps, self.test_file_stere],
bands=['L_645', 'L_555', 'L_469'])

mask = mo['mask']
L_645 = mo['L_645']
L_555 = mo['L_555']
L_469 = mo['L_469']

tmpfilename = os.path.join(ntd.tmp_data_path,
'mosaic_median_export.nc')
bands = {
'mask' : {'type': '>i1'},
'L_645' : {'type': '>i1'},
'L_555' : {'type': '>i1'},
'L_469' : {'type': '>i1'},
}
mo.export2thredds(tmpfilename, bands)

class LayerTest(unittest.TestCase):
def setUp(self):
self.domain = Domain(4326, '-lle 27 70 31 72 -ts 700 650')
Expand Down

0 comments on commit 51e2d8e

Please sign in to comment.