Skip to content

Commit

Permalink
Merge pull request #1431 from mraspaud/fix-gac-reader
Browse files Browse the repository at this point in the history 
Fix crash when TLE files are missing
  • Loading branch information
@mraspaud
mraspaud committed Nov 11, 2020
2 parents b411722 + 945d31d commit cafa071
Show file tree
Hide file tree
Showing 2 changed files with 154 additions and 90 deletions.
2 changes: 1 addition & 1 deletion satpy/readers/avhrr_l1b_gaclac.py
View file
Expand Up @@ -301,7 +301,7 @@ def _update_attrs(self, res):
res.attrs['sensor'] = self.sensor
try:
res.attrs['orbital_parameters'] = {'tle': self.reader.get_tle_lines()}
except IndexError:
except (IndexError, RuntimeError):
pass

@property
Expand Down
242 changes: 153 additions & 89 deletions satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py
View file
Expand Up @@ -69,8 +69,29 @@
'NSS.LHRR.NP.D16306.S1803.E1814.B3985555.WI']


class TestGACLACFile(TestCase):
"""Test the GACLAC file handler."""
@mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile.__init__', return_value=None)
def _get_fh_mocked(init_mock, **attrs):
"""Create a mocked file handler with the given attributes."""
from satpy.readers.avhrr_l1b_gaclac import GACLACFile

fh = GACLACFile()
for name, value in attrs.items():
setattr(fh, name, value)
return fh


def _get_reader_mocked(along_track=3):
"""Create a mocked reader."""
reader = mock.MagicMock(spacecraft_name='spacecraft_name',
meta_data={'foo': 'bar'})
reader.mask = [0, 0]
reader.get_times.return_value = np.arange(along_track)
reader.get_tle_lines.return_value = 'tle'
return reader


class ModulePatcher(TestCase):
"""Patch pygac."""

def setUp(self):
"""Patch pygac imports."""
Expand All @@ -97,32 +118,17 @@ def tearDown(self):
"""Unpatch the pygac imports."""
self.module_patcher.stop()


class TestGACLACFile(ModulePatcher):
"""Test the GACLAC file handler."""

def _get_fh(self, filename='NSS.GHRR.NG.D88002.S0614.E0807.B0670506.WI',
**kwargs):
"""Create a file handler."""
from trollsift import parse
filename_info = parse(GAC_PATTERN, filename)
return self.GACLACFile(filename, filename_info, {}, **kwargs)

@mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile.__init__', return_value=None)
def _get_fh_mocked(self, init_mock, **attrs):
"""Create a mocked file handler with the given attributes."""
from satpy.readers.avhrr_l1b_gaclac import GACLACFile

fh = GACLACFile()
for name, value in attrs.items():
setattr(fh, name, value)
return fh

def _get_reader_mocked(self, along_track=3):
"""Create a mocked reader."""
reader = mock.MagicMock(spacecraft_name='spacecraft_name',
meta_data={'foo': 'bar'})
reader.mask = [0, 0]
reader.get_times.return_value = np.arange(along_track)
reader.get_tle_lines.return_value = 'tle'
return reader

def test_init(self):
"""Test GACLACFile initialization."""
from pygac.gac_klm import GACKLMReader
Expand All @@ -148,59 +154,13 @@ def test_init(self):
self.assertIs(fh.reader_class, reader_cls,
'Wrong reader class assigned to {}'.format(filename))

@mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile.__init__', return_value=None)
@mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile.read_raw_data')
@mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile._get_channel')
def test_get_dataset_channels(self, get_channel, *mocks):
"""Test getting the channel datasets."""
from satpy.tests.utils import make_dataid

# Mock reader and file handler
fh = self._get_fh_mocked(
reader=self._get_reader_mocked(),
chn_dict={'1': 0, '5': 0},
start_line=None,
end_line=None,
strip_invalid_coords=False,
filename_info={'orbit_number': 123},
sensor='sensor',
)

# Test calibration to reflectance as well as attributes.
counts = np.ones((3, 3))
get_channel.return_value = counts
key = make_dataid(name='1', calibration='reflectance')
info = {'name': '1', 'standard_name': 'my_standard_name'}

res = fh.get_dataset(key=key, info=info)
exp = xr.DataArray(da.ones((3, 3)),
name=res.name,
dims=('y', 'x'),
coords={'acq_time': ('y', [0, 1, 2])},
attrs={'name': '1',
'platform_name': 'spacecraft_name',
'orbit_number': 123,
'sensor': 'sensor',
'orbital_parameters': {'tle': 'tle'},
'foo': 'bar',
'standard_name': 'my_standard_name'})
exp.coords['acq_time'].attrs['long_name'] = 'Mean scanline acquisition time'
xr.testing.assert_identical(res, exp)
get_channel.assert_called_with(key)

# Counts & brightness temperature: Similar, just check _get_channel() call
for key in [make_dataid(name='1', calibration='counts'),
make_dataid(name='5', calibration='brightness_temperature')]:
fh.get_dataset(key=key, info={'name': 1})
get_channel.assert_called_with(key)

def test_read_raw_data(self):
"""Test raw data reading."""
fh = self._get_fh_mocked(reader=None,
interpolate_coords='interpolate_coords',
creation_site='creation_site',
reader_kwargs={'foo': 'bar'},
filename='myfile')
fh = _get_fh_mocked(reader=None,
interpolate_coords='interpolate_coords',
creation_site='creation_site',
reader_kwargs={'foo': 'bar'},
filename='myfile')
reader = mock.MagicMock(mask=[0])
reader_cls = mock.MagicMock(return_value=reader)
fh.reader_class = reader_cls
Expand Down Expand Up @@ -244,8 +204,8 @@ def slice_patched(data, times):
{'strip_invalid_coords': True,
'start_line': 123, 'end_line': 456}]
for kwargs in kwargs_list:
fh = self._get_fh_mocked(
reader=self._get_reader_mocked(along_track=len(acq)),
fh = _get_fh_mocked(
reader=_get_reader_mocked(along_track=len(acq)),
chn_dict={'1': 0},
**kwargs
)
Expand All @@ -265,9 +225,9 @@ def test_get_dataset_latlon(self, *mocks):

lons = np.ones((3, 3))
lats = 2 * lons
reader = self._get_reader_mocked()
reader = _get_reader_mocked()
reader.get_lonlat.return_value = lons, lats
fh = self._get_fh_mocked(
fh = _get_fh_mocked(
reader=reader,
start_line=None,
end_line=None,
Expand Down Expand Up @@ -303,8 +263,8 @@ def test_get_dataset_angles(self, get_angle, *mocks):

ones = np.ones((3, 3))
get_angle.return_value = ones
reader = self._get_reader_mocked()
fh = self._get_fh_mocked(
reader = _get_reader_mocked()
fh = _get_fh_mocked(
reader=reader,
start_line=None,
end_line=None,
Expand Down Expand Up @@ -337,9 +297,9 @@ def test_get_dataset_qual_flags(self, *mocks):
from satpy.tests.utils import make_dataid

qual_flags = np.ones((3, 7))
reader = self._get_reader_mocked()
reader = _get_reader_mocked()
reader.get_qual_flags.return_value = qual_flags
fh = self._get_fh_mocked(
fh = _get_fh_mocked(
reader=reader,
start_line=None,
end_line=None,
Expand Down Expand Up @@ -370,11 +330,11 @@ def test_get_channel(self):
counts = np.moveaxis(np.array([[[1, 2, 3],
[4, 5, 6]]]), 0, 2)
calib_channels = 2 * counts
reader = self._get_reader_mocked()
reader = _get_reader_mocked()
reader.get_counts.return_value = counts
reader.get_calibrated_channels.return_value = calib_channels
fh = self._get_fh_mocked(reader=reader, counts=None, calib_channels=None,
chn_dict={'1': 0})
fh = _get_fh_mocked(reader=reader, counts=None, calib_channels=None,
chn_dict={'1': 0})

key = make_dataid(name='1', calibration='counts')
# Counts
Expand Down Expand Up @@ -413,7 +373,7 @@ def test_get_angle(self):

reader = mock.MagicMock()
reader.get_angles.return_value = 1, 2, 3, 4, 5
fh = self._get_fh_mocked(reader=reader, angles=None)
fh = _get_fh_mocked(reader=reader, angles=None)

# Test angle readout
key = make_dataid(name='sensor_zenith_angle')
Expand All @@ -436,7 +396,7 @@ def test_strip_invalid_lat(self):

reader = mock.MagicMock()
reader.get_lonlat.return_value = None, None
fh = self._get_fh_mocked(reader=reader, first_valid_lat=None)
fh = _get_fh_mocked(reader=reader, first_valid_lat=None)

# Test stripping
pygac.utils.strip_invalid_lat.return_value = 1, 2
Expand All @@ -458,7 +418,7 @@ def _slice_patched(data):
data = np.zeros((4, 2))
times = np.array([1, 2, 3, 4], dtype='datetime64[us]')

fh = self._get_fh_mocked(start_line=1, end_line=3, strip_invalid_coords=False)
fh = _get_fh_mocked(start_line=1, end_line=3, strip_invalid_coords=False)
data_slc, times_slc = fh.slice(data, times)
np.testing.assert_array_equal(data_slc, data[1:3])
np.testing.assert_array_equal(times_slc, times[1:3])
Expand All @@ -478,7 +438,7 @@ def test__slice(self, strip_invalid_lat, get_qual_flags):
data = np.zeros((2, 2))

# a) Only start/end line given
fh = self._get_fh_mocked(start_line=5, end_line=6, strip_invalid_coords=False)
fh = _get_fh_mocked(start_line=5, end_line=6, strip_invalid_coords=False)
data_slc = fh._slice(data)
self.assertEqual(data_slc, 'sliced')
pygac.utils.check_user_scanlines.assert_called_with(
Expand All @@ -489,14 +449,14 @@ def test__slice(self, strip_invalid_lat, get_qual_flags):
first_valid_lat=None, last_valid_lat=None)

# b) Only strip_invalid_coords=True
fh = self._get_fh_mocked(start_line=None, end_line=None, strip_invalid_coords=True)
fh = _get_fh_mocked(start_line=None, end_line=None, strip_invalid_coords=True)
fh._slice(data)
pygac.utils.check_user_scanlines.assert_called_with(
start_line=0, end_line=0,
first_valid_lat=3, last_valid_lat=4, along_track=2)

# c) Both
fh = self._get_fh_mocked(start_line=5, end_line=6, strip_invalid_coords=True)
fh = _get_fh_mocked(start_line=5, end_line=6, strip_invalid_coords=True)
fh._slice(data)
pygac.utils.check_user_scanlines.assert_called_with(
start_line=5, end_line=6,
Expand All @@ -506,3 +466,107 @@ def test__slice(self, strip_invalid_lat, get_qual_flags):
pygac.utils.slice_channel.return_value = ('sliced', 'foo', 'bar')
data_slc = fh._slice(data)
self.assertEqual(data_slc, 'sliced')


class TestGetDataset(ModulePatcher):
"""Test the get_dataset method."""

def setUp(self):
"""Set up the instance."""
self.exp = xr.DataArray(da.ones((3, 3)),
name='1',
dims=('y', 'x'),
coords={'acq_time': ('y', [0, 1, 2])},
attrs={'name': '1',
'platform_name': 'spacecraft_name',
'orbit_number': 123,
'sensor': 'sensor',
'foo': 'bar',
'standard_name': 'my_standard_name'})
self.exp.coords['acq_time'].attrs['long_name'] = 'Mean scanline acquisition time'
super().setUp()

@mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile.__init__', return_value=None)
@mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile.read_raw_data')
@mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile._get_channel', return_value=np.ones((3, 3)))
def test_get_dataset_channels(self, get_channel, *mocks):
"""Test getting the channel datasets."""
pygac_reader = _get_reader_mocked()
fh = self._create_file_handler(pygac_reader)

# Test calibration to reflectance as well as attributes.
key, res = self._get_dataset(fh)
exp = self._create_expected(res.name)
exp.attrs['orbital_parameters'] = {'tle': 'tle'}

xr.testing.assert_identical(res, exp)
get_channel.assert_called_with(key)

self._check_get_channel_calls(fh, get_channel)

@staticmethod
def _get_dataset(fh):
from satpy.tests.utils import make_dataid

key = make_dataid(name='1', calibration='reflectance')
info = {'name': '1', 'standard_name': 'my_standard_name'}
res = fh.get_dataset(key=key, info=info)
return key, res

@staticmethod
def _create_file_handler(reader):
"""Mock reader and file handler."""
fh = _get_fh_mocked(
reader=reader,
chn_dict={'1': 0, '5': 0},
start_line=None,
end_line=None,
strip_invalid_coords=False,
filename_info={'orbit_number': 123},
sensor='sensor',
)
return fh

@staticmethod
def _create_expected(name):
exp = xr.DataArray(da.ones((3, 3)),
name=name,
dims=('y', 'x'),
coords={'acq_time': ('y', [0, 1, 2])},
attrs={'name': '1',
'platform_name': 'spacecraft_name',
'orbit_number': 123,
'sensor': 'sensor',
'foo': 'bar',
'standard_name': 'my_standard_name'})
exp.coords['acq_time'].attrs['long_name'] = 'Mean scanline acquisition time'
return exp

@mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile.__init__', return_value=None)
@mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile.read_raw_data')
@mock.patch('satpy.readers.avhrr_l1b_gaclac.GACLACFile._get_channel', return_value=np.ones((3, 3)))
def test_get_dataset_no_tle(self, get_channel, *mocks):
"""Test getting the channel datasets when no TLEs are present."""
pygac_reader = _get_reader_mocked()
pygac_reader.get_tle_lines = mock.MagicMock()
pygac_reader.get_tle_lines.side_effect = RuntimeError()

fh = self._create_file_handler(pygac_reader)

# Test calibration to reflectance as well as attributes.
key, res = self._get_dataset(fh)
exp = self._create_expected(res.name)
xr.testing.assert_identical(res, exp)
get_channel.assert_called_with(key)

self._check_get_channel_calls(fh, get_channel)

@staticmethod
def _check_get_channel_calls(fh, get_channel):
"""Check _get_channel() calls."""
from satpy.tests.utils import make_dataid

for key in [make_dataid(name='1', calibration='counts'),
make_dataid(name='5', calibration='brightness_temperature')]:
fh.get_dataset(key=key, info={'name': 1})
get_channel.assert_called_with(key)

0 comments on commit cafa071

Please sign in to comment.