diff --git a/doc/source/index.rst b/doc/source/index.rst index 5d7da435c2..2a7d37e950 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -181,7 +181,7 @@ the base Satpy installation. - `slstr_l1b` - In development * - OSISAF SST data in GHRSST (netcdf) format - - `ghrsst_l3c_sst` + - `ghrsst_l2` - In development * - NUCAPS EDR Retrieval in NetCDF4 format - `nucaps` @@ -245,7 +245,7 @@ the base Satpy installation. - `glm_l2` - Beta * - Sentinel-3 SLSTR SST data in NetCDF4 format - - `slstr_l2` + - `ghrsst_l2` - Beta * - IASI level 2 SO2 in BUFR format - `iasi_l2_so2_bufr` diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index b81014dfca..a8cec78efc 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2015-2020 Satpy developers +# Copyright (c) 2015-2022 Satpy developers # # This file is part of satpy. # @@ -495,7 +495,7 @@ def build_colormap(palette, dtype, info): Colormaps come in different forms, but they are all supposed to have color values between 0 and 255. The following cases are considered: - - Palettes comprised of only a list on colors. If *dtype* is uint8, + - Palettes comprised of only a list of colors. If *dtype* is uint8, the values of the colormap are the enumeration of the colors. Otherwise, the colormap values will be spread evenly from the min to the max of the valid_range provided in `info`. diff --git a/satpy/etc/readers/ghrsst_l2.yaml b/satpy/etc/readers/ghrsst_l2.yaml new file mode 100644 index 0000000000..0fbcf3a4b6 --- /dev/null +++ b/satpy/etc/readers/ghrsst_l2.yaml @@ -0,0 +1,97 @@ +reader: + description: NC Reader for GHRSST Level 2 data + name: ghrsst_l2 + sensors: ['slstr', 'avhrr/3', 'viirs'] + default_channels: [] + reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader + +file_types: + GHRSST_OSISAF: + file_reader: !!python/name:satpy.readers.ghrsst_l2.GHRSSTL2FileHandler + # S-OSI_-FRA_-NPP_-NARSST_FIELD-202010141300Z.nc + file_patterns: ['S-OSI_-{generating_centre:4s}-{satid:s}-{field_type:s}_FIELD-{valid_time:%Y%m%d%H%M}Z.nc'] + + SLSTR: + file_reader: !!python/name:satpy.readers.ghrsst_l2.GHRSSTL2FileHandler + file_patterns: ['{dt1:%Y%m%d%H%M%S}-{generating_centre:3s}-{type_id:3s}_GHRSST-SSTskin-SLSTR{something:1s}-{dt2:%Y%m%d%H%M%S}-{version}.nc', + '{mission_id:3s}_SL_{processing_level:1s}_WST____{start_time:%Y%m%dT%H%M%S}_{end_time:%Y%m%dT%H%M%S}_{creation_time:%Y%m%dT%H%M%S}_{duration:4d}_{cycle:3d}_{relative_orbit:3d}_{frame:4s}_{centre:3s}_{mode:1s}_{timeliness:2s}_{collection:3s}.SEN3.tar'] + +datasets: + # SLSTR SST and Sea Ice products + longitude_slstr: + name: longitude_slstr + resolution: 1000 + view: nadir + file_type: SLSTR + standard_name: lon + units: degree + + latitude_slstr: + name: latitude_slstr + resolution: 1000 + view: nadir + file_type: SLSTR + standard_name: lat + units: degree + + sea_surface_temperature_slstr: + name: sea_surface_temperature + sensor: slstr + coordinates: [longitude_slstr, latitude_slstr] + file_type: SLSTR + resolution: 1000 + view: nadir + units: kelvin + standard_name: sea_surface_temperature + + sea_ice_fraction_slstr: + name: sea_ice_fraction + sensor: slstr + coordinates: [longitude_slstr, latitude_slstr] + file_type: SLSTR + resolution: 1000 + view: nadir + units: "%" + standard_name: sea_ice_fraction + + # Quality estimation 0-5: no data, cloud, worst, low, acceptable, best + quality_level_slstr: + name: quality_level + sensor: slstr + coordinates: [longitude_slstr, latitude_slstr] + file_type: SLSTR + resolution: 1000 + view: nadir + standard_name: quality_level + + + # OSISAF SST: + longitude_osisaf: + name: longitude_osisaf + resolution: 2000 + file_type: GHRSST_OSISAF + standard_name: lon + units: degree + + latitude_osisaf: + name: latitude_osisaf + resolution: 2000 + file_type: GHRSST_OSISAF + standard_name: lat + units: degree + + sea_surface_temperature_osisaf: + name: sea_surface_temperature + coordinates: [longitude_osisaf, latitude_osisaf] + file_type: GHRSST_OSISAF + resolution: 2000 + units: kelvin + standard_name: sea_surface_temperature + + sea_ice_fraction_osisaf: + name: sea_ice_fraction + coordinates: [longitude_osisaf, latitude_osisaf] + file_type: GHRSST_OSISAF + resolution: 2000 + units: "%" + standard_name: sea_ice_fraction diff --git a/satpy/etc/readers/ghrsst_l3c_sst.yaml b/satpy/etc/readers/ghrsst_l3c_sst.yaml deleted file mode 100644 index fd3ada064f..0000000000 --- a/satpy/etc/readers/ghrsst_l3c_sst.yaml +++ /dev/null @@ -1,17 +0,0 @@ -reader: - description: OSISAF SST GHRSST netCDF reader - name: ghrsst_l3c_sst - reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader - sensors: [avhrr/3, viirs] - -datasets: - - sea_surface_temperature: - name: sea_surface_temperature - file_type: ghrsst_osisaf_l2 - resolution: 1000 - -file_types: - ghrsst_osisaf_l2: - file_reader: !!python/name:satpy.readers.ghrsst_l3c_sst.GHRSST_OSISAFL2 - file_patterns: ['S-OSI_-FRA_-{satid:3s}_-NARSST_FIELD-{start_time:%Y%m%d%H00}Z.nc'] diff --git a/satpy/readers/ghrsst_l2.py b/satpy/readers/ghrsst_l2.py new file mode 100644 index 0000000000..dde3ce7a71 --- /dev/null +++ b/satpy/readers/ghrsst_l2.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 - 2022 Satpy developers +# +# This file is part of Satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Reader for the GHRSST level-2 formatted data.""" + +import os +import tarfile +from contextlib import suppress +from datetime import datetime +from functools import cached_property + +import xarray as xr + +from satpy import CHUNK_SIZE +from satpy.readers.file_handlers import BaseFileHandler + + +class GHRSSTL2FileHandler(BaseFileHandler): + """File handler for GHRSST L2 netCDF files.""" + + def __init__(self, filename, filename_info, filetype_info, engine=None): + """Initialize the file handler for GHRSST L2 netCDF data.""" + super().__init__(filename, filename_info, filetype_info) + self._engine = engine + self._tarfile = None + + self.filename_info['start_time'] = datetime.strptime( + self.nc.start_time, '%Y%m%dT%H%M%SZ') + self.filename_info['end_time'] = datetime.strptime( + self.nc.stop_time, '%Y%m%dT%H%M%SZ') + + @cached_property + def nc(self): + """Get the xarray Dataset for the filename.""" + if os.fspath(self.filename).endswith('tar'): + file_obj = self._open_tarfile() + else: + file_obj = self.filename + + nc = xr.open_dataset(file_obj, + decode_cf=True, + mask_and_scale=True, + engine=self._engine, + chunks={'ni': CHUNK_SIZE, + 'nj': CHUNK_SIZE}) + + return nc.rename({'ni': 'x', 'nj': 'y'}) + + def _open_tarfile(self): + self._tarfile = tarfile.open(name=self.filename, mode='r') + sst_filename = next((name for name in self._tarfile.getnames() + if self._is_sst_file(name))) + file_obj = self._tarfile.extractfile(sst_filename) + return file_obj + + @staticmethod + def _is_sst_file(name): + """Check if file in the tar archive is a valid SST file.""" + return name.endswith('nc') and 'GHRSST-SSTskin' in name + + def get_dataset(self, key, info): + """Get any available dataset.""" + stdname = info.get('standard_name') + return self.nc[stdname].squeeze() + + @property + def start_time(self): + """Get start time.""" + return self.filename_info['start_time'] + + @property + def end_time(self): + """Get end time.""" + return self.filename_info['end_time'] + + @property + def sensor(self): + """Get the sensor name.""" + return self.nc.attrs['sensor'].lower() + + def __del__(self): + """Close the tarfile object.""" + with suppress(AttributeError): + self._tarfile.close() diff --git a/satpy/readers/slstr_l2.py b/satpy/readers/slstr_l2.py deleted file mode 100644 index 8a23947ef3..0000000000 --- a/satpy/readers/slstr_l2.py +++ /dev/null @@ -1,77 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017 Satpy developers -# -# This file is part of satpy. -# -# satpy is free software: you can redistribute it and/or modify it under the -# terms of the GNU General Public License as published by the Free Software -# Foundation, either version 3 of the License, or (at your option) any later -# version. -# -# satpy is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# satpy. If not, see . -"""Reader for Sentinel-3 SLSTR SST data.""" - -from datetime import datetime - -import xarray as xr - -from satpy import CHUNK_SIZE -from satpy.readers.file_handlers import BaseFileHandler - - -class SLSTRL2FileHandler(BaseFileHandler): - """File handler for Sentinel-3 SSL L2 netCDF files.""" - - def __init__(self, filename, filename_info, filetype_info, engine=None): - """Initialize the file handler for Sentinel-3 SSL L2 netCDF data.""" - super(SLSTRL2FileHandler, self).__init__(filename, filename_info, filetype_info) - - if filename.endswith('tar'): - import os - import tarfile - import tempfile - with tempfile.TemporaryDirectory() as tempdir: - with tarfile.open(name=filename, mode='r') as tf: - sst_filename = next((name for name in tf.getnames() - if name.endswith('nc') and 'GHRSST-SSTskin' in name)) - tf.extract(sst_filename, tempdir) - fullpath = os.path.join(tempdir, sst_filename) - self.nc = xr.open_dataset(fullpath, - decode_cf=True, - mask_and_scale=True, - engine=engine, - chunks={'ni': CHUNK_SIZE, - 'nj': CHUNK_SIZE}) - else: - self.nc = xr.open_dataset(filename, - decode_cf=True, - mask_and_scale=True, - engine=engine, - chunks={'ni': CHUNK_SIZE, - 'nj': CHUNK_SIZE}) - - self.nc = self.nc.rename({'ni': 'x', 'nj': 'y'}) - self.filename_info['start_time'] = datetime.strptime( - self.nc.start_time, '%Y%m%dT%H%M%SZ') - self.filename_info['end_time'] = datetime.strptime( - self.nc.stop_time, '%Y%m%dT%H%M%SZ') - - def get_dataset(self, key, info): - """Get any available dataset.""" - stdname = info.get('standard_name') - return self.nc[stdname].squeeze() - - @property - def start_time(self): - """Get start time.""" - return self.filename_info['start_time'] - - @property - def end_time(self): - """Get end time.""" - return self.filename_info['end_time'] diff --git a/satpy/tests/reader_tests/test_ghrsst_l2.py b/satpy/tests/reader_tests/test_ghrsst_l2.py new file mode 100644 index 0000000000..e33cec467a --- /dev/null +++ b/satpy/tests/reader_tests/test_ghrsst_l2.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2018, 2022 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Module for testing the satpy.readers.ghrsst_l2 module.""" + +import os +import tarfile +from datetime import datetime +from pathlib import Path + +import numpy as np +import pytest +import xarray as xr + +from satpy.readers.ghrsst_l2 import GHRSSTL2FileHandler + + +class TestGHRSSTL2Reader: + """Test Sentinel-3 SST L2 reader.""" + + def setup_method(self, tmp_path): + """Create a fake osisaf ghrsst dataset.""" + self.base_data = np.array(([-32768, 1135, 1125], [1138, 1128, 1080])) + self.lon_data = np.array(([-13.43, 1.56, 11.25], [-11.38, 1.28, 10.80])) + self.lat_data = np.array(([43.43, 55.56, 61.25], [41.38, 50.28, 60.80])) + self.lon = xr.DataArray( + self.lon_data, + dims=('nj', 'ni'), + attrs={'standard_name': 'longitude', + 'units': 'degrees_east', + } + ) + self.lat = xr.DataArray( + self.lat_data, + dims=('nj', 'ni'), + attrs={'standard_name': 'latitude', + 'units': 'degrees_north', + } + ) + self.sst = xr.DataArray( + self.base_data, + dims=('nj', 'ni'), + attrs={'scale_factor': 0.01, 'add_offset': 273.15, + '_FillValue': -32768, 'units': 'kelvin', + } + ) + self.fake_dataset = xr.Dataset( + data_vars={ + 'sea_surface_temperature': self.sst, + 'longitude': self.lon, + 'latitude': self.lat, + }, + attrs={ + "start_time": "20220321T112640Z", + "stop_time": "20220321T145711Z", + "platform": 'NOAA20', + "sensor": "VIIRS", + }, + ) + + def _create_tarfile_with_testdata(self, mypath): + """Create a 'fake' testdata set in a tar file.""" + slstr_fakename = "S3A_SL_2_WST_MAR_O_NR_003.SEN3" + tarfile_fakename = "S3A_SL_2_WST_MAR_O_NR_003.SEN3.tar" + + slstrdir = mypath / slstr_fakename + slstrdir.mkdir(parents=True, exist_ok=True) + tarfile_path = mypath / tarfile_fakename + + ncfilename = slstrdir / 'L2P_GHRSST-SSTskin-202204131200.nc' + self.fake_dataset.to_netcdf(os.fspath(ncfilename)) + xmlfile_path = slstrdir / 'xfdumanifest.xml' + xmlfile_path.touch() + + with tarfile.open(name=tarfile_path, mode='w') as tar: + tar.add(os.fspath(ncfilename), arcname=Path(slstr_fakename) / ncfilename.name) + tar.add(os.fspath(xmlfile_path), arcname=Path(slstr_fakename) / xmlfile_path.name) + + return tarfile_path + + def test_instantiate_single_netcdf_file(self, tmp_path): + """Test initialization of file handlers - given a single netCDF file.""" + filename_info = {} + tmp_filepath = tmp_path / 'fake_dataset.nc' + self.fake_dataset.to_netcdf(os.fspath(tmp_filepath)) + + GHRSSTL2FileHandler(os.fspath(tmp_filepath), filename_info, None) + + def test_instantiate_tarfile(self, tmp_path): + """Test initialization of file handlers - given a tar file as in the case of the SAFE format.""" + filename_info = {} + tarfile_path = self._create_tarfile_with_testdata(tmp_path) + + GHRSSTL2FileHandler(os.fspath(tarfile_path), filename_info, None) + + def test_get_dataset(self, tmp_path): + """Test retrieval of datasets.""" + filename_info = {} + tmp_filepath = tmp_path / 'fake_dataset.nc' + self.fake_dataset.to_netcdf(os.fspath(tmp_filepath)) + + test = GHRSSTL2FileHandler(os.fspath(tmp_filepath), filename_info, None) + + test.get_dataset('longitude', {'standard_name': 'longitude'}) + test.get_dataset('latitude', {'standard_name': 'latitude'}) + test.get_dataset('sea_surface_temperature', {'standard_name': 'sea_surface_temperature'}) + + with pytest.raises(KeyError): + test.get_dataset('erroneous dataset', {'standard_name': 'erroneous dataset'}) + + def test_get_sensor(self, tmp_path): + """Test retrieval of the sensor name from the netCDF file.""" + dt_valid = datetime(2022, 3, 21, 11, 26, 40) # 202203211200Z + filename_info = {'field_type': 'NARSST', 'generating_centre': 'FRA_', + 'satid': 'NOAA20_', 'valid_time': dt_valid} + + tmp_filepath = tmp_path / 'fake_dataset.nc' + self.fake_dataset.to_netcdf(os.fspath(tmp_filepath)) + + test = GHRSSTL2FileHandler(os.fspath(tmp_filepath), filename_info, None) + assert test.sensor == 'viirs' + + def test_get_start_and_end_times(self, tmp_path): + """Test retrieval of the sensor name from the netCDF file.""" + dt_valid = datetime(2022, 3, 21, 11, 26, 40) # 202203211200Z + good_start_time = datetime(2022, 3, 21, 11, 26, 40) # 20220321T112640Z + good_stop_time = datetime(2022, 3, 21, 14, 57, 11) # 20220321T145711Z + + filename_info = {'field_type': 'NARSST', 'generating_centre': 'FRA_', + 'satid': 'NOAA20_', 'valid_time': dt_valid} + + tmp_filepath = tmp_path / 'fake_dataset.nc' + self.fake_dataset.to_netcdf(os.fspath(tmp_filepath)) + + test = GHRSSTL2FileHandler(os.fspath(tmp_filepath), filename_info, None) + + assert test.start_time == good_start_time + assert test.end_time == good_stop_time diff --git a/satpy/tests/reader_tests/test_slstr_l2.py b/satpy/tests/reader_tests/test_slstr_l2.py deleted file mode 100644 index 5330a10e3e..0000000000 --- a/satpy/tests/reader_tests/test_slstr_l2.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2018 Satpy developers -# -# This file is part of satpy. -# -# satpy is free software: you can redistribute it and/or modify it under the -# terms of the GNU General Public License as published by the Free Software -# Foundation, either version 3 of the License, or (at your option) any later -# version. -# -# satpy is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# satpy. If not, see . -"""Module for testing the satpy.readers.slstr_l2 module.""" - -import unittest -from unittest import mock -from unittest.mock import MagicMock, patch - -import xarray as xr - -from satpy.readers.slstr_l2 import SLSTRL2FileHandler - - -class TestSLSTRL2Reader(unittest.TestCase): - """Test Sentinel-3 SST L2 reader.""" - - @mock.patch('xarray.open_dataset') - def test_instantiate(self, mocked_dataset): - """Test initialization of file handlers.""" - filename_info = {} - tmp = MagicMock(start_time='20191120T125002Z', stop_time='20191120T125002Z') - tmp.rename.return_value = tmp - xr.open_dataset.return_value = tmp - SLSTRL2FileHandler('somedir/somefile.nc', filename_info, None) - mocked_dataset.assert_called() - mocked_dataset.reset_mock() - - with patch('tarfile.open') as tf: - tf.return_value.__enter__.return_value = MagicMock(getnames=lambda *a: ["GHRSST-SSTskin.nc"]) - SLSTRL2FileHandler('somedir/somefile.tar', filename_info, None) - mocked_dataset.assert_called() - mocked_dataset.reset_mock() - - @mock.patch('xarray.open_dataset') - def test_get_dataset(self, mocked_dataset): - """Test retrieval of datasets.""" - filename_info = {} - tmp = MagicMock(start_time='20191120T125002Z', stop_time='20191120T125002Z') - tmp.rename.return_value = tmp - xr.open_dataset.return_value = tmp - test = SLSTRL2FileHandler('somedir/somefile.nc', filename_info, None) - test.nc = {'longitude': xr.Dataset(), - 'latitude': xr.Dataset(), - 'sea_surface_temperature': xr.Dataset(), - 'sea_ice_fraction': xr.Dataset(), - } - test.get_dataset('longitude', {'standard_name': 'longitude'}) - test.get_dataset('latitude', {'standard_name': 'latitude'}) - test.get_dataset('sea_surface_temperature', {'standard_name': 'sea_surface_temperature'}) - test.get_dataset('sea_ice_fraction', {'standard_name': 'sea_ice_fraction'}) - with self.assertRaises(KeyError): - test.get_dataset('erroneous dataset', {'standard_name': 'erroneous dataset'}) - mocked_dataset.assert_called() - mocked_dataset.reset_mock()