From c1559366e5b720ebb196e9c71821c94c7ed7697c Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Wed, 7 Dec 2022 17:07:34 +0100 Subject: [PATCH 01/14] using pillow for saving datasets --- satpy/tests/writer_tests/test_mitiff.py | 35 ++++++++++++++----------- satpy/writers/mitiff.py | 26 ++++++++++++++---- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index 498ed0e5a3..fa1c1d7f13 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -548,8 +548,7 @@ def test_save_datasets(self): import os import numpy as np - from libtiff import TIFF - + from PIL import Image from satpy.writers.mitiff import MITIFFWriter expected = np.full((100, 200), 0) dataset = self._get_test_datasets() @@ -557,16 +556,18 @@ def test_save_datasets(self): w.save_datasets(dataset) filename = (dataset[0].attrs['metadata_requirements']['file_pattern']).format( start_time=dataset[0].attrs['start_time']) - tif = TIFF.open(os.path.join(self.base_dir, filename)) - for image in tif.iter_images(): - np.testing.assert_allclose(image, expected, atol=1.e-6, rtol=0) + pillow_tif = Image.open(os.path.join(self.base_dir, filename)) + for frame_no in range(pillow_tif.n_frames): + pillow_tif.seek(frame_no) + np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((100, 200)), + expected, atol=1.e-6, rtol=0) def test_save_datasets_sensor_set(self): """Test basic writer operation save_datasets.""" import os import numpy as np - from libtiff import TIFF + from PIL import Image from satpy.writers.mitiff import MITIFFWriter expected = np.full((100, 200), 0) @@ -575,26 +576,30 @@ def test_save_datasets_sensor_set(self): w.save_datasets(dataset) filename = (dataset[0].attrs['metadata_requirements']['file_pattern']).format( start_time=dataset[0].attrs['start_time']) - tif = TIFF.open(os.path.join(self.base_dir, filename)) - for image in tif.iter_images(): - np.testing.assert_allclose(image, expected, atol=1.e-6, rtol=0) + pillow_tif = Image.open(os.path.join(self.base_dir, filename)) + for frame_no in range(pillow_tif.n_frames): + pillow_tif.seek(frame_no) + np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((100, 200)), + expected, atol=1.e-6, rtol=0) def test_save_one_dataset(self): """Test basic writer operation with one dataset ie. no bands.""" import os - from libtiff import TIFF + from PIL import Image from satpy.writers.mitiff import MITIFFWriter dataset = self._get_test_one_dataset() w = MITIFFWriter(base_dir=self.base_dir) w.save_dataset(dataset) - tif = TIFF.open(os.path.join(self.base_dir, os.listdir(self.base_dir)[0])) + pillow_tif = Image.open(os.path.join(self.base_dir, os.listdir(self.base_dir)[0])) + #tif = TIFF.open(os.path.join(self.base_dir, os.listdir(self.base_dir)[0])) IMAGEDESCRIPTION = 270 - imgdesc = (tif.GetField(IMAGEDESCRIPTION)).decode('utf-8').split('\n') - for key in imgdesc: - if 'In this file' in key: - self.assertEqual(key, ' Channels: 1 In this file: 1') + print(pillow_tif.tag_v2()) + # imgdesc = (tif.GetField(IMAGEDESCRIPTION)).decode('utf-8').split('\n') + # for key in imgdesc: + # if 'In this file' in key: + # self.assertEqual(key, ' Channels: 1 In this file: 1') def test_save_one_dataset_sesnor_set(self): """Test basic writer operation with one dataset ie. no bands.""" diff --git a/satpy/writers/mitiff.py b/satpy/writers/mitiff.py index ef07417527..9198e04318 100644 --- a/satpy/writers/mitiff.py +++ b/satpy/writers/mitiff.py @@ -25,6 +25,8 @@ from satpy.dataset import DataID, DataQuery from satpy.writers import ImageWriter, get_enhanced_image +from PIL import Image + IMAGEDESCRIPTION = 270 LOG = logging.getLogger(__name__) @@ -672,11 +674,20 @@ def _save_datasets_as_mitiff(self, datasets, image_description, """ from libtiff import TIFF - - tif = TIFF.open(gen_filename, mode='wb') - + import os + + bs, ex = os.path.splitext(gen_filename) + tmp_gen_filename = gen_filename + if ex.endswith('mitiff'): + bd = os.path.dirname(bs) + bn = os.path.basename(bs) + tmp_gen_filename = os.path.join(bd, '.' + bn + '.tif') + tif = TIFF.open("test.mitiff", mode='wb') tif.SetField(IMAGEDESCRIPTION, (image_description).encode('utf-8')) + tiffinfo = {} + tiffinfo[IMAGEDESCRIPTION] = (image_description).encode('utf-8') + mitiff_frames = [] cns = self.translate_channel_name.get(kwargs['sensor'], {}) if isinstance(datasets, list): LOG.debug("Saving datasets as list") @@ -688,8 +699,11 @@ def _save_datasets_as_mitiff(self, datasets, image_description, data = self._calibrate_data(dataset, dataset.attrs['calibration'], self.mitiff_config[kwargs['sensor']][cn]['min-val'], self.mitiff_config[kwargs['sensor']][cn]['max-val']) - tif.write_image(data.astype(np.uint8), compression='deflate') + #tif.write_image(data.astype(np.uint8), compression='deflate') + mitiff_frames.append(Image.fromarray(data.astype(np.uint8), mode='L')) break + mitiff_frames[0].save(tmp_gen_filename, save_all=True, append_images=mitiff_frames[1:], + compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) elif 'dataset' in datasets.attrs['name']: self._save_single_dataset(datasets, cns, tif, kwargs) elif self.palette: @@ -698,7 +712,9 @@ def _save_datasets_as_mitiff(self, datasets, image_description, else: LOG.debug("Saving datasets as enhanced image") self._save_as_enhanced(tif, datasets, **kwargs) - del tif + #del tif + os.rename(tmp_gen_filename, gen_filename) + def _save_single_dataset(self, datasets, cns, tif, kwargs): LOG.debug("Saving %s as a dataset.", datasets.attrs['name']) From 79240dd23bf6a0fb4507845a558c7df17c60ee5a Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Thu, 8 Dec 2022 08:28:16 +0100 Subject: [PATCH 02/14] ported the rest of the methods and updated the tests --- satpy/tests/writer_tests/test_mitiff.py | 133 +++++++++++++----------- satpy/writers/mitiff.py | 60 +++++------ 2 files changed, 95 insertions(+), 98 deletions(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index fa1c1d7f13..daaf430772 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -21,6 +21,7 @@ """ import unittest +from PIL import Image class TestMITIFFWriter(unittest.TestCase): @@ -593,27 +594,25 @@ def test_save_one_dataset(self): w = MITIFFWriter(base_dir=self.base_dir) w.save_dataset(dataset) pillow_tif = Image.open(os.path.join(self.base_dir, os.listdir(self.base_dir)[0])) - #tif = TIFF.open(os.path.join(self.base_dir, os.listdir(self.base_dir)[0])) IMAGEDESCRIPTION = 270 - print(pillow_tif.tag_v2()) - # imgdesc = (tif.GetField(IMAGEDESCRIPTION)).decode('utf-8').split('\n') - # for key in imgdesc: - # if 'In this file' in key: - # self.assertEqual(key, ' Channels: 1 In this file: 1') + imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') + for key in imgdesc: + if 'In this file' in key: + self.assertEqual(key, ' Channels: 1 In this file: 1') - def test_save_one_dataset_sesnor_set(self): + def test_save_one_dataset_sensor_set(self): """Test basic writer operation with one dataset ie. no bands.""" import os - from libtiff import TIFF + from PIL import Image from satpy.writers.mitiff import MITIFFWriter dataset = self._get_test_one_dataset_sensor_set() w = MITIFFWriter(base_dir=self.base_dir) w.save_dataset(dataset) - tif = TIFF.open(os.path.join(self.base_dir, os.listdir(self.base_dir)[0])) + pillow_tif = Image.open(os.path.join(self.base_dir, os.listdir(self.base_dir)[0])) IMAGEDESCRIPTION = 270 - imgdesc = (tif.GetField(IMAGEDESCRIPTION)).decode('utf-8').split('\n') + imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') for key in imgdesc: if 'In this file' in key: self.assertEqual(key, ' Channels: 1 In this file: 1') @@ -623,7 +622,7 @@ def test_save_dataset_with_calibration(self): import os import numpy as np - from libtiff import TIFF + from PIL import Image from satpy.writers.mitiff import MITIFFWriter @@ -760,9 +759,10 @@ def test_save_dataset_with_calibration(self): w.save_dataset(dataset) filename = (dataset.attrs['metadata_requirements']['file_pattern']).format( start_time=dataset.attrs['start_time']) - tif = TIFF.open(os.path.join(self.base_dir, filename)) + + pillow_tif = Image.open(os.path.join(self.base_dir, filename)) IMAGEDESCRIPTION = 270 - imgdesc = (tif.GetField(IMAGEDESCRIPTION)).decode('utf-8').split('\n') + imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') found_table_calibration = False number_of_calibrations = 0 for key in imgdesc: @@ -790,15 +790,18 @@ def test_save_dataset_with_calibration(self): self.fail("Not a valid channel description i the given key.") self.assertTrue(found_table_calibration, "Table_calibration is not found in the imagedescription.") self.assertEqual(number_of_calibrations, 6) - for i, image in enumerate(tif.iter_images()): - np.testing.assert_allclose(image, expected[i], atol=1.e-6, rtol=0) + # for i, image in enumerate(tif.iter_images()): + # np.testing.assert_allclose(image, expected[i], atol=1.e-6, rtol=0) + for frame_no in range(pillow_tif.n_frames): + pillow_tif.seek(frame_no) + np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((100, 200)), + expected[frame_no], atol=1.e-6, rtol=0) def test_save_dataset_with_calibration_one_dataset(self): """Test saving if mitiff as dataset with only one channel.""" import os import numpy as np - from libtiff import TIFF from satpy.writers.mitiff import MITIFFWriter @@ -831,9 +834,10 @@ def test_save_dataset_with_calibration_one_dataset(self): w.save_dataset(dataset) filename = (dataset.attrs['metadata_requirements']['file_pattern']).format( start_time=dataset.attrs['start_time']) - tif = TIFF.open(os.path.join(self.base_dir, filename)) + + pillow_tif = Image.open(os.path.join(self.base_dir, filename)) IMAGEDESCRIPTION = 270 - imgdesc = (tif.GetField(IMAGEDESCRIPTION)).decode('utf-8').split('\n') + imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') found_table_calibration = False number_of_calibrations = 0 for key in imgdesc: @@ -844,15 +848,18 @@ def test_save_dataset_with_calibration_one_dataset(self): number_of_calibrations += 1 self.assertTrue(found_table_calibration, "Expected table_calibration is not found in the imagedescription.") self.assertEqual(number_of_calibrations, 1) - for image in tif.iter_images(): - np.testing.assert_allclose(image, expected, atol=1.e-6, rtol=0) + for frame_no in range(pillow_tif.n_frames): + pillow_tif.seek(frame_no) + np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((100, 200)), + expected, atol=1.e-6, rtol=0) + # for image in tif.iter_images(): + # np.testing.assert_allclose(image, expected, atol=1.e-6, rtol=0) def test_save_dataset_with_bad_value(self): """Test writer operation with bad values.""" import os import numpy as np - from libtiff import TIFF from satpy.writers.mitiff import MITIFFWriter @@ -864,9 +871,11 @@ def test_save_dataset_with_bad_value(self): w.save_dataset(dataset) filename = "{:s}_{:%Y%m%d_%H%M%S}.mitiff".format(dataset.attrs['name'], dataset.attrs['start_time']) - tif = TIFF.open(os.path.join(self.base_dir, filename)) - for image in tif.iter_images(): - np.testing.assert_allclose(image, expected, atol=1.e-6, rtol=0) + pillow_tif = Image.open(os.path.join(self.base_dir, filename)) + for frame_no in range(pillow_tif.n_frames): + pillow_tif.seek(frame_no) + np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((2, 5)), + expected, atol=1.e-6, rtol=0) def test_convert_proj4_string(self): """Test conversion of geolocations.""" @@ -921,39 +930,38 @@ def test_save_dataset_palette(self): import os import numpy as np - from libtiff import TIFF from satpy.writers.mitiff import MITIFFWriter expected = np.full((100, 200), 0) - exp_c = ([0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) - - color_map = [[0, 3], [1, 4], [2, 5]] + exp_c = [0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + + color_map = (0, 1, 2, 3, 4, 5) pal_desc = ['test', 'test2'] unit = "Test" @@ -968,14 +976,15 @@ def test_save_dataset_palette(self): w.save_dataset(dataset, **palette) filename = "{:s}_{:%Y%m%d_%H%M%S}.mitiff".format(dataset.attrs['name'], dataset.attrs['start_time']) - tif = TIFF.open(os.path.join(self.base_dir, filename)) + pillow_tif = Image.open(os.path.join(self.base_dir, filename)) # Need to check PHOTOMETRIC is 3, ie palette - self.assertEqual(tif.GetField('PHOTOMETRIC'), 3) - colormap = tif.GetField('COLORMAP') + self.assertEqual(pillow_tif.tag_v2.get(262), 3) # Check the colormap of the palette image + palette = pillow_tif.palette + colormap = list((palette.getdata())[1]) self.assertEqual(colormap, exp_c) IMAGEDESCRIPTION = 270 - imgdesc = (tif.GetField(IMAGEDESCRIPTION)).decode('utf-8').split('\n') + imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') found_color_info = False unit_name_found = False name_length_found = False @@ -1002,8 +1011,10 @@ def test_save_dataset_palette(self): self.assertEqual(unit_name, ' Test') # Check the palette description of the palette self.assertEqual(names, [' test', ' test2']) - for image in tif.iter_images(): - np.testing.assert_allclose(image, expected, atol=1.e-6, rtol=0) + for frame_no in range(pillow_tif.n_frames): + pillow_tif.seek(frame_no) + np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((100, 200)), + expected, atol=1.e-6, rtol=0) def test_simple_write_two_bands(self): """Test basic writer operation with 3 bands from 2 prerequisites.""" @@ -1016,8 +1027,6 @@ def test_get_test_dataset_three_bands_prereq(self): """Test basic writer operation with 3 bands with DataQuery prerequisites with missing name.""" import os - from libtiff import TIFF - from satpy.writers.mitiff import MITIFFWriter IMAGEDESCRIPTION = 270 @@ -1026,8 +1035,8 @@ def test_get_test_dataset_three_bands_prereq(self): w.save_dataset(dataset) filename = "{:s}_{:%Y%m%d_%H%M%S}.mitiff".format(dataset.attrs['name'], dataset.attrs['start_time']) - tif = TIFF.open(os.path.join(self.base_dir, filename)) - imgdesc = (tif.GetField(IMAGEDESCRIPTION)).decode('utf-8').split('\n') + pillow_tif = Image.open(os.path.join(self.base_dir, filename)) + imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') for element in imgdesc: if ' Channels:' in element: self.assertEqual(element, ' Channels: 3 In this file: 1 2 3') diff --git a/satpy/writers/mitiff.py b/satpy/writers/mitiff.py index 9198e04318..a6bc7cabcf 100644 --- a/satpy/writers/mitiff.py +++ b/satpy/writers/mitiff.py @@ -25,7 +25,7 @@ from satpy.dataset import DataID, DataQuery from satpy.writers import ImageWriter, get_enhanced_image -from PIL import Image +from PIL import Image, ImagePalette IMAGEDESCRIPTION = 270 @@ -369,7 +369,7 @@ def _add_calibration_datasets(self, ch, datasets, reverse_offset, reverse_scale, elif ds.attrs['prerequisites'][i].get('calibration') == 'brightness_temperature': found_calibration = True _table_calibration += ', BT, ' - _table_calibration += u'\u00B0' # '\u2103' + _table_calibration += f"\N{DEGREE SIGN}" #u'\u00B0' # '\u2103' _table_calibration += u'[C]' _reverse_offset = 255. @@ -623,7 +623,7 @@ def _calibrate_data(self, dataset, calibration, min_val, max_val): (float(max_val) - float(min_val))) * 255. return _data.clip(0, 255) - def _save_as_palette(self, tif, datasets, **kwargs): + def _save_as_palette(self, datasets, tmp_gen_filename, tiffinfo, **kwargs): # MITIFF palette has only one data channel if len(datasets.dims) == 2: LOG.debug("Palette ok with only 2 dimensions. ie only x and y") @@ -631,40 +631,34 @@ def _save_as_palette(self, tif, datasets, **kwargs): # The value of the component is used as an index into the red, green and blue curves # in the ColorMap field to retrieve an RGB triplet that defines the color. When # PhotometricInterpretation=3 is used, ColorMap must be present and SamplesPerPixel must be 1. - tif.SetField('PHOTOMETRIC', 3) - - # As write_image can not save tiff image as palette, this has to be done basicly - # ie. all needed tags needs to be set. - tif.SetField('IMAGEWIDTH', datasets.sizes['x']) - tif.SetField('IMAGELENGTH', datasets.sizes['y']) - tif.SetField('BITSPERSAMPLE', 8) - tif.SetField('COMPRESSION', tif.get_tag_define('deflate')) + tiffinfo[270] = tiffinfo[270].decode('utf-8') + img = Image.fromarray(datasets.data.astype(np.uint8), mode='P') if 'palette_color_map' in kwargs: - tif.SetField('COLORMAP', kwargs['palette_color_map']) + img.putpalette(ImagePalette.ImagePalette('RGB', kwargs['palette_color_map'])) else: LOG.ERROR("In a mitiff palette image a color map must be provided: palette_color_map is missing.") - data_type = np.uint8 - # Looks like we need to pass the data to writeencodedstrip as ctypes - cont_data = np.ascontiguousarray(datasets.data, data_type) - tif.WriteEncodedStrip(0, cont_data.ctypes.data, - datasets.sizes['x'] * datasets.sizes['y']) - tif.WriteDirectory() + img.save(tmp_gen_filename, compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) - def _save_as_enhanced(self, tif, datasets, **kwargs): + def _save_as_enhanced(self, datasets, tmp_gen_filename, **kwargs): """Save datasets as an enhanced RGB image.""" img = get_enhanced_image(datasets.squeeze(), enhance=self.enhancer) + tiffinfo = {} if 'bands' in img.data.sizes and 'bands' not in datasets.sizes: LOG.debug("Datasets without 'bands' become image with 'bands' due to enhancement.") LOG.debug("Needs to regenerate mitiff image description") - image_description = self._make_image_description(img.data, **kwargs) - tif.SetField(IMAGEDESCRIPTION, (image_description).encode('utf-8')) + image_description = self._make_image_description(img.data, **kwargs) + tiffinfo[IMAGEDESCRIPTION] = (image_description).encode('utf-8') + + mitiff_frames = [] for band in img.data['bands']: chn = img.data.sel(bands=band) data = chn.values.clip(0, 1) * 254. + 1 data = data.clip(0, 255) - tif.write_image(data.astype(np.uint8), compression='deflate') + mitiff_frames.append(Image.fromarray(data.astype(np.uint8), mode='L')) + mitiff_frames[0].save(tmp_gen_filename, save_all=True, append_images=mitiff_frames[1:], + compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) def _save_datasets_as_mitiff(self, datasets, image_description, gen_filename, **kwargs): @@ -673,7 +667,6 @@ def _save_datasets_as_mitiff(self, datasets, image_description, Include the special tags making it a mitiff file. """ - from libtiff import TIFF import os bs, ex = os.path.splitext(gen_filename) @@ -682,15 +675,13 @@ def _save_datasets_as_mitiff(self, datasets, image_description, bd = os.path.dirname(bs) bn = os.path.basename(bs) tmp_gen_filename = os.path.join(bd, '.' + bn + '.tif') - tif = TIFF.open("test.mitiff", mode='wb') - tif.SetField(IMAGEDESCRIPTION, (image_description).encode('utf-8')) tiffinfo = {} - tiffinfo[IMAGEDESCRIPTION] = (image_description).encode('utf-8') + tiffinfo[IMAGEDESCRIPTION] = (image_description).encode('latin-1') - mitiff_frames = [] cns = self.translate_channel_name.get(kwargs['sensor'], {}) if isinstance(datasets, list): LOG.debug("Saving datasets as list") + mitiff_frames = [] for _cn in self.channel_order[kwargs['sensor']]: for dataset in datasets: if dataset.attrs['name'] == _cn: @@ -699,24 +690,22 @@ def _save_datasets_as_mitiff(self, datasets, image_description, data = self._calibrate_data(dataset, dataset.attrs['calibration'], self.mitiff_config[kwargs['sensor']][cn]['min-val'], self.mitiff_config[kwargs['sensor']][cn]['max-val']) - #tif.write_image(data.astype(np.uint8), compression='deflate') mitiff_frames.append(Image.fromarray(data.astype(np.uint8), mode='L')) break mitiff_frames[0].save(tmp_gen_filename, save_all=True, append_images=mitiff_frames[1:], compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) elif 'dataset' in datasets.attrs['name']: - self._save_single_dataset(datasets, cns, tif, kwargs) + self._save_single_dataset(datasets, cns, tmp_gen_filename, tiffinfo, kwargs) elif self.palette: LOG.debug("Saving dataset as palette.") - self._save_as_palette(tif, datasets, **kwargs) + self._save_as_palette(datasets, tmp_gen_filename, tiffinfo, **kwargs) else: LOG.debug("Saving datasets as enhanced image") - self._save_as_enhanced(tif, datasets, **kwargs) - #del tif + self._save_as_enhanced(datasets, tmp_gen_filename, **kwargs) os.rename(tmp_gen_filename, gen_filename) - def _save_single_dataset(self, datasets, cns, tif, kwargs): + def _save_single_dataset(self, datasets, cns, tmp_gen_filename, tiffinfo, kwargs): LOG.debug("Saving %s as a dataset.", datasets.attrs['name']) if len(datasets.dims) == 2 and (all('bands' not in i for i in datasets.dims)): # Special case with only one channel ie. no bands @@ -728,8 +717,7 @@ def _save_single_dataset(self, datasets, cns, tif, kwargs): data = self._calibrate_data(datasets, datasets.attrs['prerequisites'][0].get('calibration'), self.mitiff_config[kwargs['sensor']][cn]['min-val'], self.mitiff_config[kwargs['sensor']][cn]['max-val']) - - tif.write_image(data.astype(np.uint8), compression='deflate') + Image.fromarray(data.astype(np.uint8)).save(tmp_gen_filename, compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) else: for _cn_i, _cn in enumerate(self.channel_order[kwargs['sensor']]): for band in datasets['bands']: @@ -743,5 +731,5 @@ def _save_single_dataset(self, datasets, cns, tif, kwargs): self.mitiff_config[kwargs['sensor']][cn]['min-val'], self.mitiff_config[kwargs['sensor']][cn]['max-val']) - tif.write_image(data.astype(np.uint8), compression='deflate') + Image.fromarray(data.astype(np.uint8)).save(tmp_gen_filename, compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) break From 8f78ffff99196caa22c89e35ebcb169796d0ba05 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Thu, 8 Dec 2022 09:17:50 +0100 Subject: [PATCH 03/14] stickler --- satpy/writers/mitiff.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/satpy/writers/mitiff.py b/satpy/writers/mitiff.py index a6bc7cabcf..082c2c02f6 100644 --- a/satpy/writers/mitiff.py +++ b/satpy/writers/mitiff.py @@ -369,7 +369,7 @@ def _add_calibration_datasets(self, ch, datasets, reverse_offset, reverse_scale, elif ds.attrs['prerequisites'][i].get('calibration') == 'brightness_temperature': found_calibration = True _table_calibration += ', BT, ' - _table_calibration += f"\N{DEGREE SIGN}" #u'\u00B0' # '\u2103' + _table_calibration += f"\N{DEGREE SIGN}" _table_calibration += u'[C]' _reverse_offset = 255. @@ -650,14 +650,14 @@ def _save_as_enhanced(self, datasets, tmp_gen_filename, **kwargs): LOG.debug("Needs to regenerate mitiff image description") image_description = self._make_image_description(img.data, **kwargs) tiffinfo[IMAGEDESCRIPTION] = (image_description).encode('utf-8') - + mitiff_frames = [] for band in img.data['bands']: chn = img.data.sel(bands=band) data = chn.values.clip(0, 1) * 254. + 1 data = data.clip(0, 255) mitiff_frames.append(Image.fromarray(data.astype(np.uint8), mode='L')) - mitiff_frames[0].save(tmp_gen_filename, save_all=True, append_images=mitiff_frames[1:], + mitiff_frames[0].save(tmp_gen_filename, save_all=True, append_images=mitiff_frames[1:], compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) def _save_datasets_as_mitiff(self, datasets, image_description, @@ -692,7 +692,7 @@ def _save_datasets_as_mitiff(self, datasets, image_description, self.mitiff_config[kwargs['sensor']][cn]['max-val']) mitiff_frames.append(Image.fromarray(data.astype(np.uint8), mode='L')) break - mitiff_frames[0].save(tmp_gen_filename, save_all=True, append_images=mitiff_frames[1:], + mitiff_frames[0].save(tmp_gen_filename, save_all=True, append_images=mitiff_frames[1:], compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) elif 'dataset' in datasets.attrs['name']: self._save_single_dataset(datasets, cns, tmp_gen_filename, tiffinfo, kwargs) @@ -703,7 +703,6 @@ def _save_datasets_as_mitiff(self, datasets, image_description, LOG.debug("Saving datasets as enhanced image") self._save_as_enhanced(datasets, tmp_gen_filename, **kwargs) os.rename(tmp_gen_filename, gen_filename) - def _save_single_dataset(self, datasets, cns, tmp_gen_filename, tiffinfo, kwargs): LOG.debug("Saving %s as a dataset.", datasets.attrs['name']) @@ -717,7 +716,8 @@ def _save_single_dataset(self, datasets, cns, tmp_gen_filename, tiffinfo, kwargs data = self._calibrate_data(datasets, datasets.attrs['prerequisites'][0].get('calibration'), self.mitiff_config[kwargs['sensor']][cn]['min-val'], self.mitiff_config[kwargs['sensor']][cn]['max-val']) - Image.fromarray(data.astype(np.uint8)).save(tmp_gen_filename, compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) + Image.fromarray(data.astype(np.uint8)).save(tmp_gen_filename, compression='tiff_deflate', + compress_level=9, tiffinfo=tiffinfo) else: for _cn_i, _cn in enumerate(self.channel_order[kwargs['sensor']]): for band in datasets['bands']: @@ -731,5 +731,6 @@ def _save_single_dataset(self, datasets, cns, tmp_gen_filename, tiffinfo, kwargs self.mitiff_config[kwargs['sensor']][cn]['min-val'], self.mitiff_config[kwargs['sensor']][cn]['max-val']) - Image.fromarray(data.astype(np.uint8)).save(tmp_gen_filename, compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) + Image.fromarray(data.astype(np.uint8)).save(tmp_gen_filename, compression='tiff_deflate', + compress_level=9, tiffinfo=tiffinfo) break From e5983240d85d5598aa7ca05158c9bd9cc0e5e3b0 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Thu, 8 Dec 2022 09:24:08 +0100 Subject: [PATCH 04/14] stickler --- satpy/writers/mitiff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/writers/mitiff.py b/satpy/writers/mitiff.py index 082c2c02f6..f176552714 100644 --- a/satpy/writers/mitiff.py +++ b/satpy/writers/mitiff.py @@ -369,7 +369,7 @@ def _add_calibration_datasets(self, ch, datasets, reverse_offset, reverse_scale, elif ds.attrs['prerequisites'][i].get('calibration') == 'brightness_temperature': found_calibration = True _table_calibration += ', BT, ' - _table_calibration += f"\N{DEGREE SIGN}" + _table_calibration += "\N{DEGREE SIGN}" _table_calibration += u'[C]' _reverse_offset = 255. From 7bba0019245466772fc75cc22c16f05cdf35c0b0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 08:27:28 +0000 Subject: [PATCH 05/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- satpy/tests/writer_tests/test_mitiff.py | 2 ++ satpy/writers/mitiff.py | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index daaf430772..3d0564c31d 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -21,6 +21,7 @@ """ import unittest + from PIL import Image @@ -550,6 +551,7 @@ def test_save_datasets(self): import numpy as np from PIL import Image + from satpy.writers.mitiff import MITIFFWriter expected = np.full((100, 200), 0) dataset = self._get_test_datasets() diff --git a/satpy/writers/mitiff.py b/satpy/writers/mitiff.py index f176552714..648985ecd0 100644 --- a/satpy/writers/mitiff.py +++ b/satpy/writers/mitiff.py @@ -21,12 +21,11 @@ import dask import numpy as np +from PIL import Image, ImagePalette from satpy.dataset import DataID, DataQuery from satpy.writers import ImageWriter, get_enhanced_image -from PIL import Image, ImagePalette - IMAGEDESCRIPTION = 270 LOG = logging.getLogger(__name__) From 6a78f4c541fd769802aed8dd436e8bcfaee6e6e7 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Thu, 8 Dec 2022 21:26:45 +0100 Subject: [PATCH 06/14] add tests --- satpy/tests/writer_tests/test_mitiff.py | 60 ++++++++++++++++++++++++- satpy/writers/mitiff.py | 4 +- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index 3d0564c31d..d2c3e20702 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -21,9 +21,11 @@ """ import unittest - +import logging from PIL import Image +logger = logging.getLogger() + class TestMITIFFWriter(unittest.TestCase): """Test the MITIFF Writer class.""" @@ -1042,3 +1044,59 @@ def test_get_test_dataset_three_bands_prereq(self): for element in imgdesc: if ' Channels:' in element: self.assertEqual(element, ' Channels: 3 In this file: 1 2 3') + + def test_save_dataset_with_calibration_error_one_dataset(self): + """Test saving if mitiff as dataset with only one channel with invalid calibration.""" + import sys + + from satpy.tests.utils import make_dsq + from satpy.writers.mitiff import MITIFFWriter + logger.level = logging.DEBUG + + dataset = self._get_test_dataset_calibration_one_dataset() + prereqs = [make_dsq(name='4', calibration='not_valid_calibration_name')] + dataset.attrs['prerequisites'] = prereqs + w = MITIFFWriter(filename=dataset.attrs['metadata_requirements']['file_pattern'], base_dir=self.base_dir) + _reverse_offset = 0. + _reverse_scale = 1. + _decimals = 2 + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + try: + with self.assertLogs(logger) as lc: + w._add_calibration_datasets(4, dataset, _reverse_offset, _reverse_scale, _decimals) + for _op in lc.output: + self.assertIn("Unknown calib type. Must be Radiance, Reflectance or BT.", _op) + finally: + logger.removeHandler(stream_handler) + + def test_save_dataset_with_missing_palette(self): + """Test saving if mitiff missing palette.""" + import os + import sys + + from satpy.writers.mitiff import MITIFFWriter + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + logger.setLevel(logging.DEBUG) + + dataset = self._get_test_one_dataset() + pal_desc = ['test', 'test2'] + unit = "Test" + palette = {'palette': True, + 'palette_description': pal_desc, + 'palette_unit': unit, + 'palette_channel_name': dataset.attrs['name']} + w = MITIFFWriter(base_dir=self.base_dir) + tiffinfo = {} + tiffinfo[270] = "Just dummy image desc".encode('utf-8') + filename = "{:s}_{:%Y%m%d_%H%M%S}.mitiff".format(dataset.attrs['name'], + dataset.attrs['start_time']) + try: + with self.assertLogs(logger, logging.ERROR) as lc: + w._save_as_palette(dataset.compute(), os.path.join(self.base_dir, filename), tiffinfo, **palette) + for _op in lc.output: + self.assertIn(("In a mitiff palette image a color map must be provided: " + "palette_color_map is missing."), _op) + finally: + logger.removeHandler(stream_handler) diff --git a/satpy/writers/mitiff.py b/satpy/writers/mitiff.py index 648985ecd0..ed9d477de1 100644 --- a/satpy/writers/mitiff.py +++ b/satpy/writers/mitiff.py @@ -636,7 +636,8 @@ def _save_as_palette(self, datasets, tmp_gen_filename, tiffinfo, **kwargs): if 'palette_color_map' in kwargs: img.putpalette(ImagePalette.ImagePalette('RGB', kwargs['palette_color_map'])) else: - LOG.ERROR("In a mitiff palette image a color map must be provided: palette_color_map is missing.") + LOG.error("In a mitiff palette image a color map must be provided: palette_color_map is missing.") + return img.save(tmp_gen_filename, compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) @@ -694,6 +695,7 @@ def _save_datasets_as_mitiff(self, datasets, image_description, mitiff_frames[0].save(tmp_gen_filename, save_all=True, append_images=mitiff_frames[1:], compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) elif 'dataset' in datasets.attrs['name']: + LOG.debug("Saving dataset as single dataset.") self._save_single_dataset(datasets, cns, tmp_gen_filename, tiffinfo, kwargs) elif self.palette: LOG.debug("Saving dataset as palette.") From dfa2cca6fa8b301e6f890adaeee33d6ed6d8126a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 20:28:20 +0000 Subject: [PATCH 07/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- satpy/tests/writer_tests/test_mitiff.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index d2c3e20702..d871b4e5f4 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -20,8 +20,9 @@ Based on the test for geotiff writer """ -import unittest import logging +import unittest + from PIL import Image logger = logging.getLogger() From 8b7f9b69118c48ae1bd0bc1ed057c70cf5a28064 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Fri, 9 Dec 2022 12:28:16 +0100 Subject: [PATCH 08/14] Update satpy/tests/writer_tests/test_mitiff.py Co-authored-by: Martin Raspaud --- satpy/tests/writer_tests/test_mitiff.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index d871b4e5f4..873dac1a1f 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -553,7 +553,6 @@ def test_save_datasets(self): import os import numpy as np - from PIL import Image from satpy.writers.mitiff import MITIFFWriter expected = np.full((100, 200), 0) From e4c210a70b1e5a0bd3c62837960333e6f8546cb6 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Fri, 9 Dec 2022 12:28:32 +0100 Subject: [PATCH 09/14] Update satpy/tests/writer_tests/test_mitiff.py Co-authored-by: Martin Raspaud --- satpy/tests/writer_tests/test_mitiff.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index 873dac1a1f..96d1480189 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -572,7 +572,6 @@ def test_save_datasets_sensor_set(self): import os import numpy as np - from PIL import Image from satpy.writers.mitiff import MITIFFWriter expected = np.full((100, 200), 0) From ddf9bfea20f8d845f061f787b163b3b298eca2b4 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Fri, 9 Dec 2022 12:28:40 +0100 Subject: [PATCH 10/14] Update satpy/tests/writer_tests/test_mitiff.py Co-authored-by: Martin Raspaud --- satpy/tests/writer_tests/test_mitiff.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index 96d1480189..47d2bbcc49 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -590,7 +590,6 @@ def test_save_one_dataset(self): """Test basic writer operation with one dataset ie. no bands.""" import os - from PIL import Image from satpy.writers.mitiff import MITIFFWriter dataset = self._get_test_one_dataset() From 9766ff6237764be29caec2f976bf585e24eb93fa Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Fri, 9 Dec 2022 12:28:51 +0100 Subject: [PATCH 11/14] Update satpy/tests/writer_tests/test_mitiff.py Co-authored-by: Martin Raspaud --- satpy/tests/writer_tests/test_mitiff.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index 47d2bbcc49..b8242f01cf 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -606,7 +606,6 @@ def test_save_one_dataset_sensor_set(self): """Test basic writer operation with one dataset ie. no bands.""" import os - from PIL import Image from satpy.writers.mitiff import MITIFFWriter dataset = self._get_test_one_dataset_sensor_set() From e20b831793ba6f8a3011ba620b777497bb2ebbe9 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Fri, 9 Dec 2022 12:29:20 +0100 Subject: [PATCH 12/14] Update satpy/tests/writer_tests/test_mitiff.py Co-authored-by: Martin Raspaud --- satpy/tests/writer_tests/test_mitiff.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index b8242f01cf..fdbdf18c68 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -853,8 +853,6 @@ def test_save_dataset_with_calibration_one_dataset(self): pillow_tif.seek(frame_no) np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((100, 200)), expected, atol=1.e-6, rtol=0) - # for image in tif.iter_images(): - # np.testing.assert_allclose(image, expected, atol=1.e-6, rtol=0) def test_save_dataset_with_bad_value(self): """Test writer operation with bad values.""" From 3c763ed5a9a1764861002895179df044cb2cf28f Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Fri, 9 Dec 2022 13:12:57 +0100 Subject: [PATCH 13/14] refactor a small bit, and refactor some parts of the tests --- satpy/tests/writer_tests/test_mitiff.py | 122 +++++++----------------- satpy/writers/mitiff.py | 20 ++-- 2 files changed, 46 insertions(+), 96 deletions(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index fdbdf18c68..b9dd830503 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -20,8 +20,10 @@ Based on the test for geotiff writer """ +import os import logging import unittest +import numpy as np from PIL import Image @@ -275,7 +277,6 @@ def _get_test_dataset_with_bad_values(self, bands=3): """Create a single test dataset.""" from datetime import datetime - import numpy as np import xarray as xr from pyresample.geometry import AreaDefinition from pyresample.utils import proj4_str_to_dict @@ -536,6 +537,19 @@ def _get_test_dataset_three_bands_prereq(self, bands=3): 10.8]}) return ds1 + def _read_back_mitiff_and_check(self, filename, expected, test_shape=(100, 200)): + pillow_tif = Image.open(filename) + for frame_no in range(pillow_tif.n_frames): + pillow_tif.seek(frame_no) + np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape(test_shape), + expected[frame_no], atol=1.e-6, rtol=0) + + def _imagedescription_from_mitiff(self, filename): + pillow_tif = Image.open(filename) + IMAGEDESCRIPTION = 270 + imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') + return imgdesc + def test_init(self): """Test creating the writer with no arguments.""" from satpy.writers.mitiff import MITIFFWriter @@ -550,81 +564,50 @@ def test_simple_write(self): def test_save_datasets(self): """Test basic writer operation save_datasets.""" - import os - - import numpy as np - from satpy.writers.mitiff import MITIFFWriter - expected = np.full((100, 200), 0) + expected = [np.full((100, 200), 0)] dataset = self._get_test_datasets() w = MITIFFWriter(base_dir=self.base_dir) w.save_datasets(dataset) filename = (dataset[0].attrs['metadata_requirements']['file_pattern']).format( start_time=dataset[0].attrs['start_time']) - pillow_tif = Image.open(os.path.join(self.base_dir, filename)) - for frame_no in range(pillow_tif.n_frames): - pillow_tif.seek(frame_no) - np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((100, 200)), - expected, atol=1.e-6, rtol=0) + self._read_back_mitiff_and_check(os.path.join(self.base_dir, filename), expected) def test_save_datasets_sensor_set(self): """Test basic writer operation save_datasets.""" - import os - - import numpy as np - from satpy.writers.mitiff import MITIFFWriter - expected = np.full((100, 200), 0) + expected = [np.full((100, 200), 0)] dataset = self._get_test_datasets_sensor_set() w = MITIFFWriter(base_dir=self.base_dir) w.save_datasets(dataset) filename = (dataset[0].attrs['metadata_requirements']['file_pattern']).format( start_time=dataset[0].attrs['start_time']) - pillow_tif = Image.open(os.path.join(self.base_dir, filename)) - for frame_no in range(pillow_tif.n_frames): - pillow_tif.seek(frame_no) - np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((100, 200)), - expected, atol=1.e-6, rtol=0) + self._read_back_mitiff_and_check(os.path.join(self.base_dir, filename), expected) def test_save_one_dataset(self): """Test basic writer operation with one dataset ie. no bands.""" - import os - - from satpy.writers.mitiff import MITIFFWriter dataset = self._get_test_one_dataset() w = MITIFFWriter(base_dir=self.base_dir) w.save_dataset(dataset) - pillow_tif = Image.open(os.path.join(self.base_dir, os.listdir(self.base_dir)[0])) - IMAGEDESCRIPTION = 270 - imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') + imgdesc = self._imagedescription_from_mitiff(os.path.join(self.base_dir, os.listdir(self.base_dir)[0])) for key in imgdesc: if 'In this file' in key: self.assertEqual(key, ' Channels: 1 In this file: 1') def test_save_one_dataset_sensor_set(self): """Test basic writer operation with one dataset ie. no bands.""" - import os - - from satpy.writers.mitiff import MITIFFWriter dataset = self._get_test_one_dataset_sensor_set() w = MITIFFWriter(base_dir=self.base_dir) w.save_dataset(dataset) - pillow_tif = Image.open(os.path.join(self.base_dir, os.listdir(self.base_dir)[0])) - IMAGEDESCRIPTION = 270 - imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') + imgdesc = self._imagedescription_from_mitiff(os.path.join(self.base_dir, os.listdir(self.base_dir)[0])) for key in imgdesc: if 'In this file' in key: self.assertEqual(key, ' Channels: 1 In this file: 1') def test_save_dataset_with_calibration(self): """Test writer operation with calibration.""" - import os - - import numpy as np - from PIL import Image - from satpy.writers.mitiff import MITIFFWriter expected_ir = np.full((100, 200), 255) @@ -761,9 +744,7 @@ def test_save_dataset_with_calibration(self): filename = (dataset.attrs['metadata_requirements']['file_pattern']).format( start_time=dataset.attrs['start_time']) - pillow_tif = Image.open(os.path.join(self.base_dir, filename)) - IMAGEDESCRIPTION = 270 - imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') + imgdesc = self._imagedescription_from_mitiff(os.path.join(self.base_dir, filename)) found_table_calibration = False number_of_calibrations = 0 for key in imgdesc: @@ -791,22 +772,13 @@ def test_save_dataset_with_calibration(self): self.fail("Not a valid channel description i the given key.") self.assertTrue(found_table_calibration, "Table_calibration is not found in the imagedescription.") self.assertEqual(number_of_calibrations, 6) - # for i, image in enumerate(tif.iter_images()): - # np.testing.assert_allclose(image, expected[i], atol=1.e-6, rtol=0) - for frame_no in range(pillow_tif.n_frames): - pillow_tif.seek(frame_no) - np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((100, 200)), - expected[frame_no], atol=1.e-6, rtol=0) + self._read_back_mitiff_and_check(os.path.join(self.base_dir, filename), expected) def test_save_dataset_with_calibration_one_dataset(self): """Test saving if mitiff as dataset with only one channel.""" - import os - - import numpy as np - from satpy.writers.mitiff import MITIFFWriter - expected = np.full((100, 200), 255) + expected = [np.full((100, 200), 255)] expected_key_channel = [u'Table_calibration: BT, BT, °[C], 8, [ 50.00 49.22 48.43 47.65 46.86 46.08 45.29 ' '44.51 43.73 42.94 42.16 41.37 40.59 39.80 39.02 38.24 37.45 36.67 35.88 35.10 34.31 ' '33.53 32.75 31.96 31.18 30.39 29.61 28.82 28.04 27.25 26.47 25.69 24.90 24.12 23.33 ' @@ -836,9 +808,7 @@ def test_save_dataset_with_calibration_one_dataset(self): filename = (dataset.attrs['metadata_requirements']['file_pattern']).format( start_time=dataset.attrs['start_time']) - pillow_tif = Image.open(os.path.join(self.base_dir, filename)) - IMAGEDESCRIPTION = 270 - imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') + imgdesc = self._imagedescription_from_mitiff(os.path.join(self.base_dir, filename)) found_table_calibration = False number_of_calibrations = 0 for key in imgdesc: @@ -849,32 +819,21 @@ def test_save_dataset_with_calibration_one_dataset(self): number_of_calibrations += 1 self.assertTrue(found_table_calibration, "Expected table_calibration is not found in the imagedescription.") self.assertEqual(number_of_calibrations, 1) - for frame_no in range(pillow_tif.n_frames): - pillow_tif.seek(frame_no) - np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((100, 200)), - expected, atol=1.e-6, rtol=0) + self._read_back_mitiff_and_check(os.path.join(self.base_dir, filename), expected) def test_save_dataset_with_bad_value(self): """Test writer operation with bad values.""" - import os - - import numpy as np - from satpy.writers.mitiff import MITIFFWriter - expected = np.array([[0, 4, 1, 37, 73], - [110, 146, 183, 219, 255]]) - + _expected = np.array([[0, 4, 1, 37, 73], + [110, 146, 183, 219, 255]]) + expected = [_expected, _expected, _expected] dataset = self._get_test_dataset_with_bad_values() w = MITIFFWriter(base_dir=self.base_dir) w.save_dataset(dataset) filename = "{:s}_{:%Y%m%d_%H%M%S}.mitiff".format(dataset.attrs['name'], dataset.attrs['start_time']) - pillow_tif = Image.open(os.path.join(self.base_dir, filename)) - for frame_no in range(pillow_tif.n_frames): - pillow_tif.seek(frame_no) - np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((2, 5)), - expected, atol=1.e-6, rtol=0) + self._read_back_mitiff_and_check(os.path.join(self.base_dir, filename), expected, test_shape=(2, 5)) def test_convert_proj4_string(self): """Test conversion of geolocations.""" @@ -926,13 +885,9 @@ def test_convert_proj4_string(self): def test_save_dataset_palette(self): """Test writer operation as palette.""" - import os - - import numpy as np - from satpy.writers.mitiff import MITIFFWriter - expected = np.full((100, 200), 0) + expected = [np.full((100, 200), 0)] exp_c = [0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -982,8 +937,7 @@ def test_save_dataset_palette(self): palette = pillow_tif.palette colormap = list((palette.getdata())[1]) self.assertEqual(colormap, exp_c) - IMAGEDESCRIPTION = 270 - imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') + imgdesc = self._imagedescription_from_mitiff(os.path.join(self.base_dir, filename)) found_color_info = False unit_name_found = False name_length_found = False @@ -1010,10 +964,7 @@ def test_save_dataset_palette(self): self.assertEqual(unit_name, ' Test') # Check the palette description of the palette self.assertEqual(names, [' test', ' test2']) - for frame_no in range(pillow_tif.n_frames): - pillow_tif.seek(frame_no) - np.testing.assert_allclose(np.asarray(pillow_tif.getdata()).reshape((100, 200)), - expected, atol=1.e-6, rtol=0) + self._read_back_mitiff_and_check(os.path.join(self.base_dir, filename), expected) def test_simple_write_two_bands(self): """Test basic writer operation with 3 bands from 2 prerequisites.""" @@ -1024,18 +975,14 @@ def test_simple_write_two_bands(self): def test_get_test_dataset_three_bands_prereq(self): """Test basic writer operation with 3 bands with DataQuery prerequisites with missing name.""" - import os - from satpy.writers.mitiff import MITIFFWriter - IMAGEDESCRIPTION = 270 dataset = self._get_test_dataset_three_bands_prereq() w = MITIFFWriter(base_dir=self.base_dir) w.save_dataset(dataset) filename = "{:s}_{:%Y%m%d_%H%M%S}.mitiff".format(dataset.attrs['name'], dataset.attrs['start_time']) - pillow_tif = Image.open(os.path.join(self.base_dir, filename)) - imgdesc = (pillow_tif.tag_v2.get(IMAGEDESCRIPTION)).split('\n') + imgdesc = self._imagedescription_from_mitiff(os.path.join(self.base_dir, filename)) for element in imgdesc: if ' Channels:' in element: self.assertEqual(element, ' Channels: 3 In this file: 1 2 3') @@ -1067,7 +1014,6 @@ def test_save_dataset_with_calibration_error_one_dataset(self): def test_save_dataset_with_missing_palette(self): """Test saving if mitiff missing palette.""" - import os import sys from satpy.writers.mitiff import MITIFFWriter diff --git a/satpy/writers/mitiff.py b/satpy/writers/mitiff.py index ed9d477de1..1729b9ddba 100644 --- a/satpy/writers/mitiff.py +++ b/satpy/writers/mitiff.py @@ -17,6 +17,7 @@ # satpy. If not, see . """MITIFF writer objects for creating MITIFF files from `Dataset` objects.""" +import os import logging import dask @@ -660,6 +661,16 @@ def _save_as_enhanced(self, datasets, tmp_gen_filename, **kwargs): mitiff_frames[0].save(tmp_gen_filename, save_all=True, append_images=mitiff_frames[1:], compression='tiff_deflate', compress_level=9, tiffinfo=tiffinfo) + def _generate_intermediate_filename(self, gen_filename): + """Replace mitiff ext because pillow doesn't recognise the file type.""" + bs, ex = os.path.splitext(gen_filename) + tmp_gen_filename = gen_filename + if ex.endswith('mitiff'): + bd = os.path.dirname(bs) + bn = os.path.basename(bs) + tmp_gen_filename = os.path.join(bd, '.' + bn + '.tif') + return tmp_gen_filename + def _save_datasets_as_mitiff(self, datasets, image_description, gen_filename, **kwargs): """Put all together and save as a tiff file. @@ -667,14 +678,7 @@ def _save_datasets_as_mitiff(self, datasets, image_description, Include the special tags making it a mitiff file. """ - import os - - bs, ex = os.path.splitext(gen_filename) - tmp_gen_filename = gen_filename - if ex.endswith('mitiff'): - bd = os.path.dirname(bs) - bn = os.path.basename(bs) - tmp_gen_filename = os.path.join(bd, '.' + bn + '.tif') + tmp_gen_filename = self._generate_intermediate_filename(gen_filename) tiffinfo = {} tiffinfo[IMAGEDESCRIPTION] = (image_description).encode('latin-1') From d8fd2a2e0f353013b377c1c2cceac722ca48bf23 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Dec 2022 12:15:02 +0000 Subject: [PATCH 14/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- satpy/tests/writer_tests/test_mitiff.py | 4 ++-- satpy/writers/mitiff.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index b9dd830503..a7e9589f1f 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -20,11 +20,11 @@ Based on the test for geotiff writer """ -import os import logging +import os import unittest -import numpy as np +import numpy as np from PIL import Image logger = logging.getLogger() diff --git a/satpy/writers/mitiff.py b/satpy/writers/mitiff.py index 1729b9ddba..d85527e7a6 100644 --- a/satpy/writers/mitiff.py +++ b/satpy/writers/mitiff.py @@ -17,8 +17,8 @@ # satpy. If not, see . """MITIFF writer objects for creating MITIFF files from `Dataset` objects.""" -import os import logging +import os import dask import numpy as np