From 93cfdaaae202bb8336c8c998c7a87cc73e298110 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 20 Oct 2020 21:18:14 +0200 Subject: [PATCH 01/17] Add support for reading OSISAF SST GHRSST data Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/ghrsst_l2.yaml | 36 +++++++++++++++++++++ satpy/etc/readers/ghrsst_l3c_sst.yaml | 17 ---------- satpy/etc/readers/slstr_l2.yaml | 24 +++++++------- satpy/readers/{slstr_l2.py => ghrsst_l2.py} | 16 ++++----- 4 files changed, 56 insertions(+), 37 deletions(-) create mode 100644 satpy/etc/readers/ghrsst_l2.yaml delete mode 100644 satpy/etc/readers/ghrsst_l3c_sst.yaml rename satpy/readers/{slstr_l2.py => ghrsst_l2.py} (85%) diff --git a/satpy/etc/readers/ghrsst_l2.yaml b/satpy/etc/readers/ghrsst_l2.yaml new file mode 100644 index 0000000000..33c2d03db2 --- /dev/null +++ b/satpy/etc/readers/ghrsst_l2.yaml @@ -0,0 +1,36 @@ +reader: + description: NC Reader for GHRSST Level 2 data + name: ghrsst_l2 + sensors: [] + default_channels: [] + reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader + +file_types: + GHRSST_OSISAF: + file_reader: !!python/name:satpy.readers.ghrsst_l2.GHRSSTL2FileHandler + # S-OSI_-FRA_-NPP_-NARSST_FIELD-202010141300Z.nc + file_patterns: ['S-OSI_-{generating_centre:4s}-{satid:4s}-{field_type:s}_FIELD-{valid_time:%Y%m%d%H%M}Z.nc'] + + +datasets: + longitude: + name: longitude + resolution: 2000 + file_type: GHRSST_OSISAF + standard_name: lon + units: degree + + latitude: + name: latitude + resolution: 2000 + file_type: GHRSST_OSISAF + standard_name: lat + units: degree + + sea_surface_temperature: + name: sea_surface_temperature + coordinates: [longitude, latitude] + file_type: GHRSST_OSISAF + resolution: 2000 + units: kelvin + standard_name: sea_surface_temperature diff --git a/satpy/etc/readers/ghrsst_l3c_sst.yaml b/satpy/etc/readers/ghrsst_l3c_sst.yaml deleted file mode 100644 index fd3ada064f..0000000000 --- a/satpy/etc/readers/ghrsst_l3c_sst.yaml +++ /dev/null @@ -1,17 +0,0 @@ -reader: - description: OSISAF SST GHRSST netCDF reader - name: ghrsst_l3c_sst - reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader - sensors: [avhrr/3, viirs] - -datasets: - - sea_surface_temperature: - name: sea_surface_temperature - file_type: ghrsst_osisaf_l2 - resolution: 1000 - -file_types: - ghrsst_osisaf_l2: - file_reader: !!python/name:satpy.readers.ghrsst_l3c_sst.GHRSST_OSISAFL2 - file_patterns: ['S-OSI_-FRA_-{satid:3s}_-NARSST_FIELD-{start_time:%Y%m%d%H00}Z.nc'] diff --git a/satpy/etc/readers/slstr_l2.yaml b/satpy/etc/readers/slstr_l2.yaml index 5af39c1813..36ca66ff59 100644 --- a/satpy/etc/readers/slstr_l2.yaml +++ b/satpy/etc/readers/slstr_l2.yaml @@ -1,13 +1,13 @@ reader: - description: NC Reader for Sentinel-3 SLSTR Level 2 data + description: NC Reader for Sentinel-3 SLSTR GHRSST Level 2 data name: slstr_l2 - sensors: [slstr_l2] + sensors: [slstr] default_channels: [] reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader file_types: - SLSTRB: - file_reader: !!python/name:satpy.readers.slstr_l2.SLSTRL2FileHandler + SLSTR: + file_reader: !!python/name:satpy.readers.ghrsst_l2.GHRSSTL2FileHandler file_patterns: ['{dt1:%Y%m%d%H%M%S}-{generating_centre:3s}-{type_id:3s}_GHRSST-SSTskin-SLSTR{something:1s}-{dt2:%Y%m%d%H%M%S}-{version}.nc', '{mission_id:3s}_SL_{processing_level:1s}_WST____{start_time:%Y%m%dT%H%M%S}_{end_time:%Y%m%dT%H%M%S}_{creation_time:%Y%m%dT%H%M%S}_{duration:4d}_{cycle:3d}_{relative_orbit:3d}_{frame:4s}_{centre:3s}_{mode:1s}_{timeliness:2s}_{collection:3s}.SEN3.tar'] @@ -16,7 +16,7 @@ datasets: name: longitude resolution: 1000 view: nadir - file_type: SLSTRB + file_type: SLSTR standard_name: lon units: degree @@ -24,15 +24,15 @@ datasets: name: latitude resolution: 1000 view: nadir - file_type: SLSTRB + file_type: SLSTR standard_name: lat units: degree sea_surface_temperature: name: sea_surface_temperature - sensor: slstr_l2 + sensor: slstr coordinates: [longitude, latitude] - file_type: SLSTRB + file_type: SLSTR resolution: 1000 view: nadir units: kelvin @@ -40,9 +40,9 @@ datasets: sea_ice_fraction: name: sea_ice_fraction - sensor: slstr_l2 + sensor: slstr coordinates: [longitude, latitude] - file_type: SLSTRB + file_type: SLSTR resolution: 1000 view: nadir units: "%" @@ -51,9 +51,9 @@ datasets: # Quality estimation 0-5: no data, cloud, worst, low, acceptable, best quality_level: name: quality_level - sensor: slstr_l2 + sensor: slstr coordinates: [longitude, latitude] - file_type: SLSTRB + file_type: SLSTR resolution: 1000 view: nadir standard_name: quality_level diff --git a/satpy/readers/slstr_l2.py b/satpy/readers/ghrsst_l2.py similarity index 85% rename from satpy/readers/slstr_l2.py rename to satpy/readers/ghrsst_l2.py index 7531834be8..72d117807c 100644 --- a/satpy/readers/slstr_l2.py +++ b/satpy/readers/ghrsst_l2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2017 Satpy developers +# Copyright (c) 2017 - 2020 Satpy developers # -# This file is part of satpy. +# This file is part of Satpy. # # satpy is free software: you can redistribute it and/or modify it under the # terms of the GNU General Public License as published by the Free Software @@ -14,7 +14,7 @@ # # You should have received a copy of the GNU General Public License along with # satpy. If not, see . -"""Reader for Sentinel-3 SLSTR SST data.""" +"""Reader for the GHRSST level-2 formatted data.""" from datetime import datetime from satpy.readers.file_handlers import BaseFileHandler @@ -22,12 +22,12 @@ import xarray as xr -class SLSTRL2FileHandler(BaseFileHandler): - """File handler for Sentinel-3 SSL L2 netCDF files.""" +class GHRSSTL2FileHandler(BaseFileHandler): + """File handler for GHRSST L2 netCDF files.""" def __init__(self, filename, filename_info, filetype_info, engine=None): - """Initialize the file handler for Sentinel-3 SSL L2 netCDF data.""" - super(SLSTRL2FileHandler, self).__init__(filename, filename_info, filetype_info) + """Initialize the file handler for GHRSST L2 netCDF data.""" + super(GHRSSTL2FileHandler, self).__init__(filename, filename_info, filetype_info) if filename.endswith('tar'): import tarfile @@ -36,7 +36,7 @@ def __init__(self, filename, filename_info, filetype_info, engine=None): with tempfile.TemporaryDirectory() as tempdir: with tarfile.open(name=filename, mode='r') as tf: sst_filename = next((name for name in tf.getnames() - if name.endswith('nc') and 'GHRSST-SSTskin' in name)) + if name.endswith('nc') and 'GHRSST-SSTskin' in name)) tf.extract(sst_filename, tempdir) fullpath = os.path.join(tempdir, sst_filename) self.nc = xr.open_dataset(fullpath, From 1d2e85acfb0c36d06137688793d9ac11957be85a Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 21 Oct 2020 20:35:54 +0200 Subject: [PATCH 02/17] Add sensor property to the reader and merge into one yaml file for both SLSTR and OSISAF Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/ghrsst_l2.yaml | 75 +++++++++++++++++++++++++++++--- satpy/etc/readers/slstr_l2.yaml | 59 ------------------------- satpy/readers/ghrsst_l2.py | 11 +++-- 3 files changed, 76 insertions(+), 69 deletions(-) delete mode 100644 satpy/etc/readers/slstr_l2.yaml diff --git a/satpy/etc/readers/ghrsst_l2.yaml b/satpy/etc/readers/ghrsst_l2.yaml index 33c2d03db2..d3b2731090 100644 --- a/satpy/etc/readers/ghrsst_l2.yaml +++ b/satpy/etc/readers/ghrsst_l2.yaml @@ -1,7 +1,7 @@ reader: description: NC Reader for GHRSST Level 2 data name: ghrsst_l2 - sensors: [] + sensors: ['slstr', 'avhrr/3', 'viirs'] default_channels: [] reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader @@ -11,26 +11,87 @@ file_types: # S-OSI_-FRA_-NPP_-NARSST_FIELD-202010141300Z.nc file_patterns: ['S-OSI_-{generating_centre:4s}-{satid:4s}-{field_type:s}_FIELD-{valid_time:%Y%m%d%H%M}Z.nc'] + SLSTR: + file_reader: !!python/name:satpy.readers.ghrsst_l2.GHRSSTL2FileHandler + file_patterns: ['{dt1:%Y%m%d%H%M%S}-{generating_centre:3s}-{type_id:3s}_GHRSST-SSTskin-SLSTR{something:1s}-{dt2:%Y%m%d%H%M%S}-{version}.nc', + '{mission_id:3s}_SL_{processing_level:1s}_WST____{start_time:%Y%m%dT%H%M%S}_{end_time:%Y%m%dT%H%M%S}_{creation_time:%Y%m%dT%H%M%S}_{duration:4d}_{cycle:3d}_{relative_orbit:3d}_{frame:4s}_{centre:3s}_{mode:1s}_{timeliness:2s}_{collection:3s}.SEN3.tar'] datasets: - longitude: - name: longitude + # SLSTR SST and Sea Ice products + longitude_slstr: + name: longitude_slstr + resolution: 1000 + view: nadir + file_type: SLSTR + standard_name: lon + units: degree + + latitude_slstr: + name: latitude_slstr + resolution: 1000 + view: nadir + file_type: SLSTR + standard_name: lat + units: degree + + sea_surface_temperature_slstr: + name: sea_surface_temperature + sensor: slstr + coordinates: [longitude_slstr, latitude_slstr] + file_type: SLSTR + resolution: 1000 + view: nadir + units: kelvin + standard_name: sea_surface_temperature + + sea_ice_fraction_slstr: + name: sea_ice_fraction + sensor: slstr + coordinates: [longitude_slstr, latitude_slstr] + file_type: SLSTR + resolution: 1000 + view: nadir + units: "%" + standard_name: sea_ice_fraction + + # Quality estimation 0-5: no data, cloud, worst, low, acceptable, best + quality_level_slstr: + name: quality_level + sensor: slstr + coordinates: [longitude_slstr, latitude_slstr] + file_type: SLSTR + resolution: 1000 + view: nadir + standard_name: quality_level + + + # OSISAF SST: + longitude_osisaf: + name: longitude_osisaf resolution: 2000 file_type: GHRSST_OSISAF standard_name: lon units: degree - latitude: - name: latitude + latitude_osisaf: + name: latitude_osisaf resolution: 2000 file_type: GHRSST_OSISAF standard_name: lat units: degree - sea_surface_temperature: + sea_surface_temperature_osisaf: name: sea_surface_temperature - coordinates: [longitude, latitude] + coordinates: [longitude_osisaf, latitude_osisaf] file_type: GHRSST_OSISAF resolution: 2000 units: kelvin standard_name: sea_surface_temperature + + sea_ice_fraction_osisaf: + name: sea_ice_fraction + coordinates: [longitude_osisaf, latitude_osisaf] + file_type: GHRSST_OSISAF + resolution: 2000 + units: "%" + standard_name: sea_ice_fraction diff --git a/satpy/etc/readers/slstr_l2.yaml b/satpy/etc/readers/slstr_l2.yaml deleted file mode 100644 index 36ca66ff59..0000000000 --- a/satpy/etc/readers/slstr_l2.yaml +++ /dev/null @@ -1,59 +0,0 @@ -reader: - description: NC Reader for Sentinel-3 SLSTR GHRSST Level 2 data - name: slstr_l2 - sensors: [slstr] - default_channels: [] - reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader - -file_types: - SLSTR: - file_reader: !!python/name:satpy.readers.ghrsst_l2.GHRSSTL2FileHandler - file_patterns: ['{dt1:%Y%m%d%H%M%S}-{generating_centre:3s}-{type_id:3s}_GHRSST-SSTskin-SLSTR{something:1s}-{dt2:%Y%m%d%H%M%S}-{version}.nc', - '{mission_id:3s}_SL_{processing_level:1s}_WST____{start_time:%Y%m%dT%H%M%S}_{end_time:%Y%m%dT%H%M%S}_{creation_time:%Y%m%dT%H%M%S}_{duration:4d}_{cycle:3d}_{relative_orbit:3d}_{frame:4s}_{centre:3s}_{mode:1s}_{timeliness:2s}_{collection:3s}.SEN3.tar'] - -datasets: - longitude: - name: longitude - resolution: 1000 - view: nadir - file_type: SLSTR - standard_name: lon - units: degree - - latitude: - name: latitude - resolution: 1000 - view: nadir - file_type: SLSTR - standard_name: lat - units: degree - - sea_surface_temperature: - name: sea_surface_temperature - sensor: slstr - coordinates: [longitude, latitude] - file_type: SLSTR - resolution: 1000 - view: nadir - units: kelvin - standard_name: sea_surface_temperature - - sea_ice_fraction: - name: sea_ice_fraction - sensor: slstr - coordinates: [longitude, latitude] - file_type: SLSTR - resolution: 1000 - view: nadir - units: "%" - standard_name: sea_ice_fraction - - # Quality estimation 0-5: no data, cloud, worst, low, acceptable, best - quality_level: - name: quality_level - sensor: slstr - coordinates: [longitude, latitude] - file_type: SLSTR - resolution: 1000 - view: nadir - standard_name: quality_level diff --git a/satpy/readers/ghrsst_l2.py b/satpy/readers/ghrsst_l2.py index 72d117807c..0c0b214244 100644 --- a/satpy/readers/ghrsst_l2.py +++ b/satpy/readers/ghrsst_l2.py @@ -20,6 +20,9 @@ from satpy.readers.file_handlers import BaseFileHandler from satpy import CHUNK_SIZE import xarray as xr +import tarfile +import os +import tempfile class GHRSSTL2FileHandler(BaseFileHandler): @@ -30,9 +33,6 @@ def __init__(self, filename, filename_info, filetype_info, engine=None): super(GHRSSTL2FileHandler, self).__init__(filename, filename_info, filetype_info) if filename.endswith('tar'): - import tarfile - import os - import tempfile with tempfile.TemporaryDirectory() as tempdir: with tarfile.open(name=filename, mode='r') as tf: sst_filename = next((name for name in tf.getnames() @@ -73,3 +73,8 @@ def start_time(self): def end_time(self): """Get end time.""" return self.filename_info['end_time'] + + @property + def sensor(self): + """Get the sensor name.""" + return self.nc.attrs['sensor'].lower() From 50aece86b97dc5d520d0f10afade811c1e412cb0 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 22 Mar 2022 20:19:11 +0100 Subject: [PATCH 03/17] Adapt name pattern to handle variable length platform-names ('NOAA-20_') Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/ghrsst_l2.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/etc/readers/ghrsst_l2.yaml b/satpy/etc/readers/ghrsst_l2.yaml index d3b2731090..0fbcf3a4b6 100644 --- a/satpy/etc/readers/ghrsst_l2.yaml +++ b/satpy/etc/readers/ghrsst_l2.yaml @@ -9,7 +9,7 @@ file_types: GHRSST_OSISAF: file_reader: !!python/name:satpy.readers.ghrsst_l2.GHRSSTL2FileHandler # S-OSI_-FRA_-NPP_-NARSST_FIELD-202010141300Z.nc - file_patterns: ['S-OSI_-{generating_centre:4s}-{satid:4s}-{field_type:s}_FIELD-{valid_time:%Y%m%d%H%M}Z.nc'] + file_patterns: ['S-OSI_-{generating_centre:4s}-{satid:s}-{field_type:s}_FIELD-{valid_time:%Y%m%d%H%M}Z.nc'] SLSTR: file_reader: !!python/name:satpy.readers.ghrsst_l2.GHRSSTL2FileHandler @@ -76,7 +76,7 @@ datasets: latitude_osisaf: name: latitude_osisaf resolution: 2000 - file_type: GHRSST_OSISAF + file_type: GHRSST_OSISAF standard_name: lat units: degree From ce2c97f25828340144a7ae316fe6c7697a3a3c12 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 25 Mar 2022 14:31:50 +0100 Subject: [PATCH 04/17] Adding an enhancement to produce an SST image with a color legend Color legend originally provided by MET Norway and the OSISAF --- satpy/composites/__init__.py | 4 ++-- satpy/etc/enhancements/generic.yaml | 36 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 90f4e67aac..73c2f06d80 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2015-2020 Satpy developers +# Copyright (c) 2015-2020, 2022 Satpy developers # # This file is part of satpy. # @@ -495,7 +495,7 @@ def build_colormap(palette, dtype, info): Colormaps come in different forms, but they are all supposed to have color values between 0 and 255. The following cases are considered: - - Palettes comprised of only a list on colors. If *dtype* is uint8, + - Palettes comprised of only a list of colors. If *dtype* is uint8, the values of the colormap are the enumeration of the colors. Otherwise, the colormap values will be spread evenly from the min to the max of the valid_range provided in `info`. diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index 386b2ec002..e7cf1eff94 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -930,3 +930,39 @@ enhancements: factor: 21.0 min_stretch: 0.0 max_stretch: 20.0 + + osisaf_sst_smhi_colormap: + standard_name: sea_surface_subskin_temperature + operations: + - name: osisaf_sst + method: !!python/name:satpy.enhancements.colorize + kwargs: + palettes: + - colors: [ + [255, 0, 255], + [195, 0, 129], + [129, 0, 47], + [195, 0, 0], + [255, 0, 0], + [236, 43, 0], + [217, 86, 0], + [200, 128, 0], + [211, 154, 13], + [222, 180, 26], + [233, 206, 39], + [244, 232, 52], + [255.99609375, 255.99609375, 63.22265625], + [203.125, 255.99609375, 52.734375], + [136.71875, 255.99609375, 27.34375], + [0, 255.99609375, 0], + [0, 207.47265625, 0], + [0, 158.94921875, 0], + [0, 110.42578125, 0], + [0, 82.8203125, 63.99609375], + [0, 55.21484375, 127.9921875], + [0, 27.609375, 191.98828125], + [0, 0, 255.99609375], + [100.390625, 100.390625, 255.99609375], + [150.5859375, 150.5859375, 255.99609375]] + min_value: 296.55 + max_value: 273.55 From a7e6be088ddd6fdc1a4acfb2abf177ff7d341cd7 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 30 Mar 2022 15:44:01 +0200 Subject: [PATCH 05/17] Rename the slstr_l2 test module and extend test coverage a bit Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/test_ghrsst_l2.py | 121 +++++++++++++++++++++ satpy/tests/reader_tests/test_slstr_l2.py | 69 ------------ 2 files changed, 121 insertions(+), 69 deletions(-) create mode 100644 satpy/tests/reader_tests/test_ghrsst_l2.py delete mode 100644 satpy/tests/reader_tests/test_slstr_l2.py diff --git a/satpy/tests/reader_tests/test_ghrsst_l2.py b/satpy/tests/reader_tests/test_ghrsst_l2.py new file mode 100644 index 0000000000..129e197c1a --- /dev/null +++ b/satpy/tests/reader_tests/test_ghrsst_l2.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2018, 2022 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Module for testing the satpy.readers.ghrsst_l2 module.""" + +import unittest +from datetime import datetime +from unittest import mock +from unittest.mock import MagicMock, patch + +import numpy as np +import xarray as xr + +from satpy.readers.ghrsst_l2 import GHRSSTL2FileHandler + + +class TestGHRSSTL2Reader(unittest.TestCase): + """Test Sentinel-3 SST L2 reader.""" + + @mock.patch('satpy.readers.ghrsst_l2.xr') + def setUp(self, xr_): + """Create a fake osisaf ghrsst dataset.""" + self.base_data = np.array(([-32768, 1135, 1125], [1138, 1128, 1080])) + self.sst = xr.DataArray( + self.base_data, + dims=('nj', 'ni'), + attrs={'scale_factor': 0.01, 'add_offset': 273.15, + '_FillValue': -32768, 'units': 'kelvin', + } + ) + self.fake_dataset = xr.Dataset( + data_vars={ + 'sea_surface_temperature': self.sst, + }, + attrs={ + "start_time": "20220321T112640Z", + "stop_time": "20220321T145711Z", + "platform": 'NOAA20', + "sensor": "VIIRS", + }, + ) + + @mock.patch('xarray.open_dataset') + def test_instantiate(self, mocked_dataset): + """Test initialization of file handlers.""" + filename_info = {} + tmp = MagicMock(start_time='20191120T125002Z', stop_time='20191120T125002Z') + tmp.rename.return_value = tmp + xr.open_dataset.return_value = tmp + GHRSSTL2FileHandler('somedir/somefile.nc', filename_info, None) + mocked_dataset.assert_called() + mocked_dataset.reset_mock() + + with patch('tarfile.open') as tf: + tf.return_value.__enter__.return_value = MagicMock(getnames=lambda *a: ["GHRSST-SSTskin.nc"]) + GHRSSTL2FileHandler('somedir/somefile.tar', filename_info, None) + mocked_dataset.assert_called() + mocked_dataset.reset_mock() + + @mock.patch('xarray.open_dataset') + def test_get_dataset(self, mocked_dataset): + """Test retrieval of datasets.""" + filename_info = {} + tmp = MagicMock(start_time='20191120T125002Z', stop_time='20191120T125002Z') + tmp.rename.return_value = tmp + xr.open_dataset.return_value = tmp + test = GHRSSTL2FileHandler('somedir/somefile.nc', filename_info, None) + test.nc = {'longitude': xr.Dataset(), + 'latitude': xr.Dataset(), + 'sea_surface_temperature': xr.Dataset(), + 'sea_ice_fraction': xr.Dataset(), + } + test.get_dataset('longitude', {'standard_name': 'longitude'}) + test.get_dataset('latitude', {'standard_name': 'latitude'}) + test.get_dataset('sea_surface_temperature', {'standard_name': 'sea_surface_temperature'}) + test.get_dataset('sea_ice_fraction', {'standard_name': 'sea_ice_fraction'}) + with self.assertRaises(KeyError): + test.get_dataset('erroneous dataset', {'standard_name': 'erroneous dataset'}) + mocked_dataset.assert_called() + mocked_dataset.reset_mock() + + @mock.patch('xarray.open_dataset') + def test_get_sensor(self, mocked_dataset): + """Test retrieval of the sensor name from the netCDF file.""" + mocked_dataset.return_value = self.fake_dataset + dt_valid = datetime(2022, 3, 21, 11, 26, 40) # 202203211200Z + filename_info = {'field_type': 'NARSST', 'generating_centre': 'FRA_', + 'satid': 'NOAA20_', 'valid_time': dt_valid} + + test = GHRSSTL2FileHandler('somedir/somefile.nc', filename_info, None) + assert test.sensor == 'viirs' + + @mock.patch('xarray.open_dataset') + def test_get_start_and_end_times(self, mocked_dataset): + """Test retrieval of the sensor name from the netCDF file.""" + mocked_dataset.return_value = self.fake_dataset + dt_valid = datetime(2022, 3, 21, 11, 26, 40) # 202203211200Z + good_start_time = datetime(2022, 3, 21, 11, 26, 40) # 20220321T112640Z + good_stop_time = datetime(2022, 3, 21, 14, 57, 11) # 20220321T145711Z + + filename_info = {'field_type': 'NARSST', 'generating_centre': 'FRA_', + 'satid': 'NOAA20_', 'valid_time': dt_valid} + + test = GHRSSTL2FileHandler('somedir/somefile.nc', filename_info, None) + + assert test.start_time == good_start_time + assert test.end_time == good_stop_time diff --git a/satpy/tests/reader_tests/test_slstr_l2.py b/satpy/tests/reader_tests/test_slstr_l2.py deleted file mode 100644 index 5330a10e3e..0000000000 --- a/satpy/tests/reader_tests/test_slstr_l2.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2018 Satpy developers -# -# This file is part of satpy. -# -# satpy is free software: you can redistribute it and/or modify it under the -# terms of the GNU General Public License as published by the Free Software -# Foundation, either version 3 of the License, or (at your option) any later -# version. -# -# satpy is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# satpy. If not, see . -"""Module for testing the satpy.readers.slstr_l2 module.""" - -import unittest -from unittest import mock -from unittest.mock import MagicMock, patch - -import xarray as xr - -from satpy.readers.slstr_l2 import SLSTRL2FileHandler - - -class TestSLSTRL2Reader(unittest.TestCase): - """Test Sentinel-3 SST L2 reader.""" - - @mock.patch('xarray.open_dataset') - def test_instantiate(self, mocked_dataset): - """Test initialization of file handlers.""" - filename_info = {} - tmp = MagicMock(start_time='20191120T125002Z', stop_time='20191120T125002Z') - tmp.rename.return_value = tmp - xr.open_dataset.return_value = tmp - SLSTRL2FileHandler('somedir/somefile.nc', filename_info, None) - mocked_dataset.assert_called() - mocked_dataset.reset_mock() - - with patch('tarfile.open') as tf: - tf.return_value.__enter__.return_value = MagicMock(getnames=lambda *a: ["GHRSST-SSTskin.nc"]) - SLSTRL2FileHandler('somedir/somefile.tar', filename_info, None) - mocked_dataset.assert_called() - mocked_dataset.reset_mock() - - @mock.patch('xarray.open_dataset') - def test_get_dataset(self, mocked_dataset): - """Test retrieval of datasets.""" - filename_info = {} - tmp = MagicMock(start_time='20191120T125002Z', stop_time='20191120T125002Z') - tmp.rename.return_value = tmp - xr.open_dataset.return_value = tmp - test = SLSTRL2FileHandler('somedir/somefile.nc', filename_info, None) - test.nc = {'longitude': xr.Dataset(), - 'latitude': xr.Dataset(), - 'sea_surface_temperature': xr.Dataset(), - 'sea_ice_fraction': xr.Dataset(), - } - test.get_dataset('longitude', {'standard_name': 'longitude'}) - test.get_dataset('latitude', {'standard_name': 'latitude'}) - test.get_dataset('sea_surface_temperature', {'standard_name': 'sea_surface_temperature'}) - test.get_dataset('sea_ice_fraction', {'standard_name': 'sea_ice_fraction'}) - with self.assertRaises(KeyError): - test.get_dataset('erroneous dataset', {'standard_name': 'erroneous dataset'}) - mocked_dataset.assert_called() - mocked_dataset.reset_mock() From 9f1ab58b90786b07a614e9ef1bc73e45fa21a4fb Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Wed, 6 Apr 2022 09:21:22 +0200 Subject: [PATCH 06/17] Update satpy/readers/ghrsst_l2.py Follow Python 3 standard for class inherritance Co-authored-by: Martin Raspaud --- satpy/readers/ghrsst_l2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/ghrsst_l2.py b/satpy/readers/ghrsst_l2.py index 39bee6adf1..28df2ab7f3 100644 --- a/satpy/readers/ghrsst_l2.py +++ b/satpy/readers/ghrsst_l2.py @@ -32,7 +32,7 @@ class GHRSSTL2FileHandler(BaseFileHandler): def __init__(self, filename, filename_info, filetype_info, engine=None): """Initialize the file handler for GHRSST L2 netCDF data.""" - super(GHRSSTL2FileHandler, self).__init__(filename, filename_info, filetype_info) + super().__init__(filename, filename_info, filetype_info) if filename.endswith('tar'): with tempfile.TemporaryDirectory() as tempdir: From 04107ac4fabfbbee056594e9cd32dd9808672436 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 6 Apr 2022 09:28:59 +0200 Subject: [PATCH 07/17] Update list of readers reflecting the change applicable to the GHRSST reading Signed-off-by: Adam.Dybbroe --- doc/source/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index 5d7da435c2..2a7d37e950 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -181,7 +181,7 @@ the base Satpy installation. - `slstr_l1b` - In development * - OSISAF SST data in GHRSST (netcdf) format - - `ghrsst_l3c_sst` + - `ghrsst_l2` - In development * - NUCAPS EDR Retrieval in NetCDF4 format - `nucaps` @@ -245,7 +245,7 @@ the base Satpy installation. - `glm_l2` - Beta * - Sentinel-3 SLSTR SST data in NetCDF4 format - - `slstr_l2` + - `ghrsst_l2` - Beta * - IASI level 2 SO2 in BUFR format - `iasi_l2_so2_bufr` From cff9d998ee936016693fd5be337f1d740ce5b89d Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 6 Apr 2022 09:31:01 +0200 Subject: [PATCH 08/17] Remove SMHI specific SST colormap from the enhancement Signed-off-by: Adam.Dybbroe --- satpy/etc/enhancements/generic.yaml | 36 ----------------------------- 1 file changed, 36 deletions(-) diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index e7cf1eff94..386b2ec002 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -930,39 +930,3 @@ enhancements: factor: 21.0 min_stretch: 0.0 max_stretch: 20.0 - - osisaf_sst_smhi_colormap: - standard_name: sea_surface_subskin_temperature - operations: - - name: osisaf_sst - method: !!python/name:satpy.enhancements.colorize - kwargs: - palettes: - - colors: [ - [255, 0, 255], - [195, 0, 129], - [129, 0, 47], - [195, 0, 0], - [255, 0, 0], - [236, 43, 0], - [217, 86, 0], - [200, 128, 0], - [211, 154, 13], - [222, 180, 26], - [233, 206, 39], - [244, 232, 52], - [255.99609375, 255.99609375, 63.22265625], - [203.125, 255.99609375, 52.734375], - [136.71875, 255.99609375, 27.34375], - [0, 255.99609375, 0], - [0, 207.47265625, 0], - [0, 158.94921875, 0], - [0, 110.42578125, 0], - [0, 82.8203125, 63.99609375], - [0, 55.21484375, 127.9921875], - [0, 27.609375, 191.98828125], - [0, 0, 255.99609375], - [100.390625, 100.390625, 255.99609375], - [150.5859375, 150.5859375, 255.99609375]] - min_value: 296.55 - max_value: 273.55 From d5b98b6c8e8ef19708653554877cc991990e084f Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 6 Apr 2022 11:18:10 +0200 Subject: [PATCH 09/17] Skip unpacking the tar archive and let open_data set work on the file object instead Signed-off-by: Adam.Dybbroe --- satpy/readers/ghrsst_l2.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/satpy/readers/ghrsst_l2.py b/satpy/readers/ghrsst_l2.py index 28df2ab7f3..5c34f8ea40 100644 --- a/satpy/readers/ghrsst_l2.py +++ b/satpy/readers/ghrsst_l2.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2017 - 2020, 2022 Satpy developers +# Copyright (c) 2017 - 2022 Satpy developers # # This file is part of Satpy. # @@ -18,7 +18,7 @@ import os import tarfile -import tempfile +from contextlib import suppress from datetime import datetime import xarray as xr @@ -34,19 +34,17 @@ def __init__(self, filename, filename_info, filetype_info, engine=None): """Initialize the file handler for GHRSST L2 netCDF data.""" super().__init__(filename, filename_info, filetype_info) - if filename.endswith('tar'): - with tempfile.TemporaryDirectory() as tempdir: - with tarfile.open(name=filename, mode='r') as tf: - sst_filename = next((name for name in tf.getnames() - if name.endswith('nc') and 'GHRSST-SSTskin' in name)) - tf.extract(sst_filename, tempdir) - fullpath = os.path.join(tempdir, sst_filename) - self.nc = xr.open_dataset(fullpath, - decode_cf=True, - mask_and_scale=True, - engine=engine, - chunks={'ni': CHUNK_SIZE, - 'nj': CHUNK_SIZE}) + if os.fspath(filename).endswith('tar'): + self._tarfile = tarfile.open(name=filename, mode='r') + sst_filename = next((name for name in self._tarfile.getnames() + if name.endswith('nc') and 'GHRSST-SSTskin' in name)) + file_obj = self._tarfile.extractfile(sst_filename) + self.nc = xr.open_dataset(file_obj, + decode_cf=True, + mask_and_scale=True, + engine=engine, + chunks={'ni': CHUNK_SIZE, + 'nj': CHUNK_SIZE}) else: self.nc = xr.open_dataset(filename, decode_cf=True, @@ -80,3 +78,8 @@ def end_time(self): def sensor(self): """Get the sensor name.""" return self.nc.attrs['sensor'].lower() + + def __del__(self): + """Close the tarfile object.""" + with suppress(AttributeError): + self._tarfile.close() From 55607897748c5cf61f6412424fb46eb87d629b24 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Sat, 9 Apr 2022 20:12:04 +0200 Subject: [PATCH 10/17] Refactor and skip mocking Signed-off-by: Adam.Dybbroe --- satpy/readers/ghrsst_l2.py | 9 +- satpy/tests/reader_tests/test_ghrsst_l2.py | 101 +++++++++++++-------- 2 files changed, 71 insertions(+), 39 deletions(-) diff --git a/satpy/readers/ghrsst_l2.py b/satpy/readers/ghrsst_l2.py index 5c34f8ea40..6b312a3d55 100644 --- a/satpy/readers/ghrsst_l2.py +++ b/satpy/readers/ghrsst_l2.py @@ -37,7 +37,7 @@ def __init__(self, filename, filename_info, filetype_info, engine=None): if os.fspath(filename).endswith('tar'): self._tarfile = tarfile.open(name=filename, mode='r') sst_filename = next((name for name in self._tarfile.getnames() - if name.endswith('nc') and 'GHRSST-SSTskin' in name)) + if self._is_sst_file(name))) file_obj = self._tarfile.extractfile(sst_filename) self.nc = xr.open_dataset(file_obj, decode_cf=True, @@ -59,6 +59,13 @@ def __init__(self, filename, filename_info, filetype_info, engine=None): self.filename_info['end_time'] = datetime.strptime( self.nc.stop_time, '%Y%m%dT%H%M%SZ') + @staticmethod + def _is_sst_file(name): + """Check if file in the tar archive is a valid SST file.""" + if name.endswith('nc') and 'GHRSST-SSTskin' in name: + return True + return False + def get_dataset(self, key, info): """Get any available dataset.""" stdname = info.get('standard_name') diff --git a/satpy/tests/reader_tests/test_ghrsst_l2.py b/satpy/tests/reader_tests/test_ghrsst_l2.py index 129e197c1a..cd8f688977 100644 --- a/satpy/tests/reader_tests/test_ghrsst_l2.py +++ b/satpy/tests/reader_tests/test_ghrsst_l2.py @@ -17,10 +17,13 @@ # satpy. If not, see . """Module for testing the satpy.readers.ghrsst_l2 module.""" +import os +import tarfile +import tempfile import unittest from datetime import datetime +from pathlib import Path from unittest import mock -from unittest.mock import MagicMock, patch import numpy as np import xarray as xr @@ -35,6 +38,22 @@ class TestGHRSSTL2Reader(unittest.TestCase): def setUp(self, xr_): """Create a fake osisaf ghrsst dataset.""" self.base_data = np.array(([-32768, 1135, 1125], [1138, 1128, 1080])) + self.lon_data = np.array(([-13.43, 1.56, 11.25], [-11.38, 1.28, 10.80])) + self.lat_data = np.array(([43.43, 55.56, 61.25], [41.38, 50.28, 60.80])) + self.lon = xr.DataArray( + self.lon_data, + dims=('nj', 'ni'), + attrs={'standard_name': 'longitude', + 'units': 'degrees_east', + } + ) + self.lat = xr.DataArray( + self.lat_data, + dims=('nj', 'ni'), + attrs={'standard_name': 'latitude', + 'units': 'degrees_north', + } + ) self.sst = xr.DataArray( self.base_data, dims=('nj', 'ni'), @@ -45,6 +64,8 @@ def setUp(self, xr_): self.fake_dataset = xr.Dataset( data_vars={ 'sea_surface_temperature': self.sst, + 'longitude': self.lon, + 'latitude': self.lat, }, attrs={ "start_time": "20220321T112640Z", @@ -53,61 +74,59 @@ def setUp(self, xr_): "sensor": "VIIRS", }, ) + self._tmpfile = tempfile.NamedTemporaryFile(suffix='.nc') + self.fake_dataset.to_netcdf(self._tmpfile.name) + + self._create_tarfile_with_testdata() + + def _create_tarfile_with_testdata(self): + """Create a 'fake' testdata set in a tar file.""" + self._tmpdir = tempfile.TemporaryDirectory() - @mock.patch('xarray.open_dataset') - def test_instantiate(self, mocked_dataset): + slstr_fakename = "S3A_SL_2_WST_MAR_O_NR_003.SEN3" + tarfile_fakename = "S3A_SL_2_WST_MAR_O_NR_003.SEN3.tar" + self._slstrdir = Path(self._tmpdir.name) / slstr_fakename + self._slstrdir.mkdir(parents=True, exist_ok=True) + self._tarfile_path = Path(self._tmpdir.name) / tarfile_fakename + + with tempfile.NamedTemporaryFile(suffix='.nc', + prefix='L2P_GHRSST-SSTskin', dir=self._slstrdir) as ncfilename: + self.fake_dataset.to_netcdf(ncfilename.name) + xmlfile_path = tempfile.NamedTemporaryFile(prefix='xfdumanifest', suffix='.xml', + dir=Path(self._tmpdir.name) / slstr_fakename) + + with tarfile.open(name=self._tarfile_path, mode='w') as tar: + tar.add(ncfilename.name, arcname=Path(slstr_fakename) / os.path.basename(ncfilename.name)) + tar.add(xmlfile_path.name, arcname=Path(slstr_fakename) / os.path.basename(xmlfile_path.name)) + + def test_instantiate(self): """Test initialization of file handlers.""" filename_info = {} - tmp = MagicMock(start_time='20191120T125002Z', stop_time='20191120T125002Z') - tmp.rename.return_value = tmp - xr.open_dataset.return_value = tmp - GHRSSTL2FileHandler('somedir/somefile.nc', filename_info, None) - mocked_dataset.assert_called() - mocked_dataset.reset_mock() - - with patch('tarfile.open') as tf: - tf.return_value.__enter__.return_value = MagicMock(getnames=lambda *a: ["GHRSST-SSTskin.nc"]) - GHRSSTL2FileHandler('somedir/somefile.tar', filename_info, None) - mocked_dataset.assert_called() - mocked_dataset.reset_mock() - - @mock.patch('xarray.open_dataset') - def test_get_dataset(self, mocked_dataset): + GHRSSTL2FileHandler(self._tmpfile.name, filename_info, None) + GHRSSTL2FileHandler(os.fspath(self._tarfile_path), filename_info, None) + + def test_get_dataset(self): """Test retrieval of datasets.""" filename_info = {} - tmp = MagicMock(start_time='20191120T125002Z', stop_time='20191120T125002Z') - tmp.rename.return_value = tmp - xr.open_dataset.return_value = tmp - test = GHRSSTL2FileHandler('somedir/somefile.nc', filename_info, None) - test.nc = {'longitude': xr.Dataset(), - 'latitude': xr.Dataset(), - 'sea_surface_temperature': xr.Dataset(), - 'sea_ice_fraction': xr.Dataset(), - } + test = GHRSSTL2FileHandler(self._tmpfile.name, filename_info, None) test.get_dataset('longitude', {'standard_name': 'longitude'}) test.get_dataset('latitude', {'standard_name': 'latitude'}) test.get_dataset('sea_surface_temperature', {'standard_name': 'sea_surface_temperature'}) - test.get_dataset('sea_ice_fraction', {'standard_name': 'sea_ice_fraction'}) + with self.assertRaises(KeyError): test.get_dataset('erroneous dataset', {'standard_name': 'erroneous dataset'}) - mocked_dataset.assert_called() - mocked_dataset.reset_mock() - @mock.patch('xarray.open_dataset') - def test_get_sensor(self, mocked_dataset): + def test_get_sensor(self): """Test retrieval of the sensor name from the netCDF file.""" - mocked_dataset.return_value = self.fake_dataset dt_valid = datetime(2022, 3, 21, 11, 26, 40) # 202203211200Z filename_info = {'field_type': 'NARSST', 'generating_centre': 'FRA_', 'satid': 'NOAA20_', 'valid_time': dt_valid} - test = GHRSSTL2FileHandler('somedir/somefile.nc', filename_info, None) + test = GHRSSTL2FileHandler(self._tmpfile.name, filename_info, None) assert test.sensor == 'viirs' - @mock.patch('xarray.open_dataset') - def test_get_start_and_end_times(self, mocked_dataset): + def test_get_start_and_end_times(self): """Test retrieval of the sensor name from the netCDF file.""" - mocked_dataset.return_value = self.fake_dataset dt_valid = datetime(2022, 3, 21, 11, 26, 40) # 202203211200Z good_start_time = datetime(2022, 3, 21, 11, 26, 40) # 20220321T112640Z good_stop_time = datetime(2022, 3, 21, 14, 57, 11) # 20220321T145711Z @@ -115,7 +134,13 @@ def test_get_start_and_end_times(self, mocked_dataset): filename_info = {'field_type': 'NARSST', 'generating_centre': 'FRA_', 'satid': 'NOAA20_', 'valid_time': dt_valid} - test = GHRSSTL2FileHandler('somedir/somefile.nc', filename_info, None) + test = GHRSSTL2FileHandler(self._tmpfile.name, filename_info, None) assert test.start_time == good_start_time assert test.end_time == good_stop_time + + def tearDown(self): + """Clean up.""" + print(self._tmpfile.name) + self._tmpfile.close() + self._tmpdir.cleanup() From ddb083b39d79f58644fc0d1950cad4578eea3ec3 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 11 Apr 2022 13:04:28 +0200 Subject: [PATCH 11/17] Minor refactor Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/test_ghrsst_l2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_ghrsst_l2.py b/satpy/tests/reader_tests/test_ghrsst_l2.py index cd8f688977..2941c5f371 100644 --- a/satpy/tests/reader_tests/test_ghrsst_l2.py +++ b/satpy/tests/reader_tests/test_ghrsst_l2.py @@ -92,8 +92,7 @@ def _create_tarfile_with_testdata(self): with tempfile.NamedTemporaryFile(suffix='.nc', prefix='L2P_GHRSST-SSTskin', dir=self._slstrdir) as ncfilename: self.fake_dataset.to_netcdf(ncfilename.name) - xmlfile_path = tempfile.NamedTemporaryFile(prefix='xfdumanifest', suffix='.xml', - dir=Path(self._tmpdir.name) / slstr_fakename) + xmlfile_path = tempfile.NamedTemporaryFile(prefix='xfdumanifest', suffix='.xml', dir=self._slstrdir) with tarfile.open(name=self._tarfile_path, mode='w') as tar: tar.add(ncfilename.name, arcname=Path(slstr_fakename) / os.path.basename(ncfilename.name)) From 37c72a9d6f51f39f598f960d554c6c7949c5830f Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 11 Apr 2022 13:23:10 +0200 Subject: [PATCH 12/17] Minor refactor Signed-off-by: Adam.Dybbroe --- satpy/readers/ghrsst_l2.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/satpy/readers/ghrsst_l2.py b/satpy/readers/ghrsst_l2.py index 6b312a3d55..8dbe468e53 100644 --- a/satpy/readers/ghrsst_l2.py +++ b/satpy/readers/ghrsst_l2.py @@ -62,9 +62,7 @@ def __init__(self, filename, filename_info, filetype_info, engine=None): @staticmethod def _is_sst_file(name): """Check if file in the tar archive is a valid SST file.""" - if name.endswith('nc') and 'GHRSST-SSTskin' in name: - return True - return False + return name.endswith('nc') and 'GHRSST-SSTskin' in name def get_dataset(self, key, info): """Get any available dataset.""" From b13688704d9c350a8c44763da65a32f51e593440 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 11 Apr 2022 13:28:43 +0200 Subject: [PATCH 13/17] Remove debug printout Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/test_ghrsst_l2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_ghrsst_l2.py b/satpy/tests/reader_tests/test_ghrsst_l2.py index 2941c5f371..310e81efdd 100644 --- a/satpy/tests/reader_tests/test_ghrsst_l2.py +++ b/satpy/tests/reader_tests/test_ghrsst_l2.py @@ -140,6 +140,5 @@ def test_get_start_and_end_times(self): def tearDown(self): """Clean up.""" - print(self._tmpfile.name) self._tmpfile.close() self._tmpdir.cleanup() From 3f2e5c05209f8b540d1d07cd01068d29903c933a Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 11 Apr 2022 14:03:10 +0200 Subject: [PATCH 14/17] Fix copyright date span Signed-off-by: Adam.Dybbroe --- satpy/composites/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 8a0c03616f..a8cec78efc 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2015-2020, 2022 Satpy developers +# Copyright (c) 2015-2022 Satpy developers # # This file is part of satpy. # From e02abd4f727e6f9aefb51185fd8979b8119d7e1a Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 13 Apr 2022 23:18:16 +0200 Subject: [PATCH 15/17] Skip unittest module and use pytests tmp_path fixture instead In an attempt to make the tests pass on Windows Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/test_ghrsst_l2.py | 86 ++++++++++++---------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/satpy/tests/reader_tests/test_ghrsst_l2.py b/satpy/tests/reader_tests/test_ghrsst_l2.py index 310e81efdd..e33cec467a 100644 --- a/satpy/tests/reader_tests/test_ghrsst_l2.py +++ b/satpy/tests/reader_tests/test_ghrsst_l2.py @@ -19,23 +19,20 @@ import os import tarfile -import tempfile -import unittest from datetime import datetime from pathlib import Path -from unittest import mock import numpy as np +import pytest import xarray as xr from satpy.readers.ghrsst_l2 import GHRSSTL2FileHandler -class TestGHRSSTL2Reader(unittest.TestCase): +class TestGHRSSTL2Reader: """Test Sentinel-3 SST L2 reader.""" - @mock.patch('satpy.readers.ghrsst_l2.xr') - def setUp(self, xr_): + def setup_method(self, tmp_path): """Create a fake osisaf ghrsst dataset.""" self.base_data = np.array(([-32768, 1135, 1125], [1138, 1128, 1080])) self.lon_data = np.array(([-13.43, 1.56, 11.25], [-11.38, 1.28, 10.80])) @@ -74,57 +71,70 @@ def setUp(self, xr_): "sensor": "VIIRS", }, ) - self._tmpfile = tempfile.NamedTemporaryFile(suffix='.nc') - self.fake_dataset.to_netcdf(self._tmpfile.name) - self._create_tarfile_with_testdata() - - def _create_tarfile_with_testdata(self): + def _create_tarfile_with_testdata(self, mypath): """Create a 'fake' testdata set in a tar file.""" - self._tmpdir = tempfile.TemporaryDirectory() - slstr_fakename = "S3A_SL_2_WST_MAR_O_NR_003.SEN3" tarfile_fakename = "S3A_SL_2_WST_MAR_O_NR_003.SEN3.tar" - self._slstrdir = Path(self._tmpdir.name) / slstr_fakename - self._slstrdir.mkdir(parents=True, exist_ok=True) - self._tarfile_path = Path(self._tmpdir.name) / tarfile_fakename - with tempfile.NamedTemporaryFile(suffix='.nc', - prefix='L2P_GHRSST-SSTskin', dir=self._slstrdir) as ncfilename: - self.fake_dataset.to_netcdf(ncfilename.name) - xmlfile_path = tempfile.NamedTemporaryFile(prefix='xfdumanifest', suffix='.xml', dir=self._slstrdir) + slstrdir = mypath / slstr_fakename + slstrdir.mkdir(parents=True, exist_ok=True) + tarfile_path = mypath / tarfile_fakename + + ncfilename = slstrdir / 'L2P_GHRSST-SSTskin-202204131200.nc' + self.fake_dataset.to_netcdf(os.fspath(ncfilename)) + xmlfile_path = slstrdir / 'xfdumanifest.xml' + xmlfile_path.touch() + + with tarfile.open(name=tarfile_path, mode='w') as tar: + tar.add(os.fspath(ncfilename), arcname=Path(slstr_fakename) / ncfilename.name) + tar.add(os.fspath(xmlfile_path), arcname=Path(slstr_fakename) / xmlfile_path.name) + + return tarfile_path + + def test_instantiate_single_netcdf_file(self, tmp_path): + """Test initialization of file handlers - given a single netCDF file.""" + filename_info = {} + tmp_filepath = tmp_path / 'fake_dataset.nc' + self.fake_dataset.to_netcdf(os.fspath(tmp_filepath)) - with tarfile.open(name=self._tarfile_path, mode='w') as tar: - tar.add(ncfilename.name, arcname=Path(slstr_fakename) / os.path.basename(ncfilename.name)) - tar.add(xmlfile_path.name, arcname=Path(slstr_fakename) / os.path.basename(xmlfile_path.name)) + GHRSSTL2FileHandler(os.fspath(tmp_filepath), filename_info, None) - def test_instantiate(self): - """Test initialization of file handlers.""" + def test_instantiate_tarfile(self, tmp_path): + """Test initialization of file handlers - given a tar file as in the case of the SAFE format.""" filename_info = {} - GHRSSTL2FileHandler(self._tmpfile.name, filename_info, None) - GHRSSTL2FileHandler(os.fspath(self._tarfile_path), filename_info, None) + tarfile_path = self._create_tarfile_with_testdata(tmp_path) - def test_get_dataset(self): + GHRSSTL2FileHandler(os.fspath(tarfile_path), filename_info, None) + + def test_get_dataset(self, tmp_path): """Test retrieval of datasets.""" filename_info = {} - test = GHRSSTL2FileHandler(self._tmpfile.name, filename_info, None) + tmp_filepath = tmp_path / 'fake_dataset.nc' + self.fake_dataset.to_netcdf(os.fspath(tmp_filepath)) + + test = GHRSSTL2FileHandler(os.fspath(tmp_filepath), filename_info, None) + test.get_dataset('longitude', {'standard_name': 'longitude'}) test.get_dataset('latitude', {'standard_name': 'latitude'}) test.get_dataset('sea_surface_temperature', {'standard_name': 'sea_surface_temperature'}) - with self.assertRaises(KeyError): + with pytest.raises(KeyError): test.get_dataset('erroneous dataset', {'standard_name': 'erroneous dataset'}) - def test_get_sensor(self): + def test_get_sensor(self, tmp_path): """Test retrieval of the sensor name from the netCDF file.""" dt_valid = datetime(2022, 3, 21, 11, 26, 40) # 202203211200Z filename_info = {'field_type': 'NARSST', 'generating_centre': 'FRA_', 'satid': 'NOAA20_', 'valid_time': dt_valid} - test = GHRSSTL2FileHandler(self._tmpfile.name, filename_info, None) + tmp_filepath = tmp_path / 'fake_dataset.nc' + self.fake_dataset.to_netcdf(os.fspath(tmp_filepath)) + + test = GHRSSTL2FileHandler(os.fspath(tmp_filepath), filename_info, None) assert test.sensor == 'viirs' - def test_get_start_and_end_times(self): + def test_get_start_and_end_times(self, tmp_path): """Test retrieval of the sensor name from the netCDF file.""" dt_valid = datetime(2022, 3, 21, 11, 26, 40) # 202203211200Z good_start_time = datetime(2022, 3, 21, 11, 26, 40) # 20220321T112640Z @@ -133,12 +143,10 @@ def test_get_start_and_end_times(self): filename_info = {'field_type': 'NARSST', 'generating_centre': 'FRA_', 'satid': 'NOAA20_', 'valid_time': dt_valid} - test = GHRSSTL2FileHandler(self._tmpfile.name, filename_info, None) + tmp_filepath = tmp_path / 'fake_dataset.nc' + self.fake_dataset.to_netcdf(os.fspath(tmp_filepath)) + + test = GHRSSTL2FileHandler(os.fspath(tmp_filepath), filename_info, None) assert test.start_time == good_start_time assert test.end_time == good_stop_time - - def tearDown(self): - """Clean up.""" - self._tmpfile.close() - self._tmpdir.cleanup() From 8e79f963a55dd54a747b19faeb95bd728a01ec75 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 14 Apr 2022 08:06:50 +0200 Subject: [PATCH 16/17] Refactor constructor --- satpy/readers/ghrsst_l2.py | 43 ++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/satpy/readers/ghrsst_l2.py b/satpy/readers/ghrsst_l2.py index 8dbe468e53..c29148f7e1 100644 --- a/satpy/readers/ghrsst_l2.py +++ b/satpy/readers/ghrsst_l2.py @@ -20,6 +20,7 @@ import tarfile from contextlib import suppress from datetime import datetime +from functools import cached_property import xarray as xr @@ -33,32 +34,34 @@ class GHRSSTL2FileHandler(BaseFileHandler): def __init__(self, filename, filename_info, filetype_info, engine=None): """Initialize the file handler for GHRSST L2 netCDF data.""" super().__init__(filename, filename_info, filetype_info) + self._engine = engine + self._tarfile = None - if os.fspath(filename).endswith('tar'): - self._tarfile = tarfile.open(name=filename, mode='r') - sst_filename = next((name for name in self._tarfile.getnames() - if self._is_sst_file(name))) - file_obj = self._tarfile.extractfile(sst_filename) - self.nc = xr.open_dataset(file_obj, - decode_cf=True, - mask_and_scale=True, - engine=engine, - chunks={'ni': CHUNK_SIZE, - 'nj': CHUNK_SIZE}) - else: - self.nc = xr.open_dataset(filename, - decode_cf=True, - mask_and_scale=True, - engine=engine, - chunks={'ni': CHUNK_SIZE, - 'nj': CHUNK_SIZE}) - - self.nc = self.nc.rename({'ni': 'x', 'nj': 'y'}) self.filename_info['start_time'] = datetime.strptime( self.nc.start_time, '%Y%m%dT%H%M%SZ') self.filename_info['end_time'] = datetime.strptime( self.nc.stop_time, '%Y%m%dT%H%M%SZ') + @cached_property + def nc(self): + """Get the xarray Dataset for the filename.""" + if os.fspath(self.filename).endswith('tar'): + self._tarfile = tarfile.open(name=self.filename, mode='r') + sst_filename = next((name for name in self._tarfile.getnames() + if self._is_sst_file(name))) + file_obj = self._tarfile.extractfile(sst_filename) + else: + file_obj = self.filename + + nc = xr.open_dataset(file_obj, + decode_cf=True, + mask_and_scale=True, + engine=self._engine, + chunks={'ni': CHUNK_SIZE, + 'nj': CHUNK_SIZE}) + + return nc.rename({'ni': 'x', 'nj': 'y'}) + @staticmethod def _is_sst_file(name): """Check if file in the tar archive is a valid SST file.""" From e3d7dda2cc2e8c931274a8ceaf414237aa78a0f5 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 14 Apr 2022 08:21:12 +0200 Subject: [PATCH 17/17] Refactor nc --- satpy/readers/ghrsst_l2.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/satpy/readers/ghrsst_l2.py b/satpy/readers/ghrsst_l2.py index c29148f7e1..dde3ce7a71 100644 --- a/satpy/readers/ghrsst_l2.py +++ b/satpy/readers/ghrsst_l2.py @@ -46,10 +46,7 @@ def __init__(self, filename, filename_info, filetype_info, engine=None): def nc(self): """Get the xarray Dataset for the filename.""" if os.fspath(self.filename).endswith('tar'): - self._tarfile = tarfile.open(name=self.filename, mode='r') - sst_filename = next((name for name in self._tarfile.getnames() - if self._is_sst_file(name))) - file_obj = self._tarfile.extractfile(sst_filename) + file_obj = self._open_tarfile() else: file_obj = self.filename @@ -62,6 +59,13 @@ def nc(self): return nc.rename({'ni': 'x', 'nj': 'y'}) + def _open_tarfile(self): + self._tarfile = tarfile.open(name=self.filename, mode='r') + sst_filename = next((name for name in self._tarfile.getnames() + if self._is_sst_file(name))) + file_obj = self._tarfile.extractfile(sst_filename) + return file_obj + @staticmethod def _is_sst_file(name): """Check if file in the tar archive is a valid SST file."""