From c993c6cbabc8cc62cde5a0f302097b30e218d1da Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Mon, 10 Oct 2022 10:19:43 +0000 Subject: [PATCH] Add support for gzipped HRIT files --- satpy/readers/hrit_base.py | 15 +++++++----- satpy/readers/utils.py | 3 +++ satpy/tests/reader_tests/test_hrit_base.py | 27 ++++++++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index 9bd0990f00..9c7e9b0e57 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -356,7 +356,8 @@ def __init__(self, filename, mda): self.bpp = mda['number_of_bits_per_pixel'] self.compressed = mda['compression_flag_for_data'] == 1 self.offset = mda['total_header_length'] - self.zipped = os.fspath(filename).endswith('.bz2') + self.bzipped = os.fspath(filename).endswith('.bz2') + self.gzipped = os.fspath(filename).endswith('.gz') def read_data(self): """Read the data.""" @@ -367,12 +368,14 @@ def read_data(self): return data def _read_data_from_file(self): - # check, if 'filename' is a file on disk, - # or a file like obj, possibly residing already in memory - try: + if self._should_read_from_disk(): return self._read_data_from_disk() - except (FileNotFoundError, AttributeError): - return self._read_file_like() + return self._read_file_like() + + def _should_read_from_disk(self): + is_bz2_or_wavelet = self.bzipped or self.compressed + is_uncompressed = not (is_bz2_or_wavelet or self.gzipped) + return is_bz2_or_wavelet or is_uncompressed def _read_data_from_disk(self): # For reading the image data, unzip_context is faster than generic_open diff --git a/satpy/readers/utils.py b/satpy/readers/utils.py index 289d73b382..cf95bafaff 100644 --- a/satpy/readers/utils.py +++ b/satpy/readers/utils.py @@ -18,6 +18,7 @@ """Helper functions for satpy readers.""" import bz2 +import gzip import logging import os import shutil @@ -288,6 +289,8 @@ def generic_open(filename, *args, **kwargs): """ if os.fspath(filename).endswith('.bz2'): fp = bz2.open(filename, *args, **kwargs) + elif os.fspath(filename).endswith(".gz"): + fp = gzip.open(filename) else: try: fp = filename.open(*args, **kwargs) diff --git a/satpy/tests/reader_tests/test_hrit_base.py b/satpy/tests/reader_tests/test_hrit_base.py index aa461d4237..76a26e5f96 100644 --- a/satpy/tests/reader_tests/test_hrit_base.py +++ b/satpy/tests/reader_tests/test_hrit_base.py @@ -18,6 +18,7 @@ """The HRIT base reader tests package.""" import bz2 +import gzip import os import unittest from datetime import datetime @@ -156,6 +157,14 @@ def stub_bzipped_hrit_file(tmp_path): return filename +@pytest.fixture +def stub_gzipped_hrit_file(tmp_path): + """Create a stub gzipped hrit file.""" + filename = tmp_path / "some_hrit_file.gz" + create_stub_hrit(filename, open_fun=gzip.open) + return filename + + @pytest.fixture def stub_compressed_hrit_file(tmp_path): """Create a stub compressed hrit file.""" @@ -244,6 +253,24 @@ def test_read_band_bzipped2_filepath(self, stub_bzipped_hrit_file): res = self.reader.read_band('VIS006', None) assert res.compute().shape == (464, 3712) + def test_read_band_gzipped_filepath(self, stub_gzipped_hrit_file): + """Test reading a single band from a gzipped file.""" + self.reader.filename = stub_gzipped_hrit_file + + res = self.reader.read_band('VIS006', None) + assert res.compute().shape == (464, 3712) + + def test_read_band_gzip_stream(self, stub_gzipped_hrit_file): + """Test reading a single band from a gzip stream.""" + import fsspec + filename = stub_gzipped_hrit_file + + fs_file = fsspec.open(filename) + self.reader.filename = FSFile(fs_file) + + res = self.reader.read_band('VIS006', None) + assert res.compute().shape == (464, 3712) + def fake_decompress(infile, outdir='.'): """Fake decompression."""