Merge pull request #1790 from mraspaud/feature-abi-benchmarks

Add ABI L1B benchmarks

asv.conf.json

@@ -22,7 +22,7 @@
     // Customizable commands for building, installing, and
     // uninstalling the project. See asv.conf.json documentation.
     //
-    "install_command": ["in-dir={env_dir} python -mpip install {wheel_file} pyspectral pyorbital s3fs rasterio h5py netCDF4 pyhdf"],
+    "install_command": ["in-dir={env_dir} python -mpip install {wheel_file} pyspectral pyorbital s3fs rasterio h5py netCDF4 pyhdf gcsfs"],
     // "install_command": ["in-dir={env_dir} conda install {wheel_file} s3fs rasterio"],
     // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"],
     // "build_command": [

benchmarks/abi_l1b_benchmarks.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 Satpy developers
+#
+# This file is part of satpy.
+#
+# satpy is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# satpy.  If not, see <http://www.gnu.org/licenses/>.
+"""Benchmark ABI L1B operations."""
+
+import os
+
+from pyspectral.rayleigh import check_and_download as download_luts
+from pyspectral.rsr_reader import check_and_download as download_rsr
+
+from benchmarks.utils import get_filenames, GeoBenchmarks
+
+
+class ABIL1B(GeoBenchmarks):
+    """Benchmark ABI L1B reading."""
+
+    timeout = 600
+    data_files = []
+    subdir = os.path.join("abi_l1b", "20190314_us_midlatitude_cyclone")
+    reader = "abi_l1b"
+
+    def setup_cache(self):
+        """Fetch the data files."""
+        try:
+            from satpy.demo import get_us_midlatitude_cyclone_abi
+            get_us_midlatitude_cyclone_abi()
+        except ImportError:
+            assert len(get_filenames(self.subdir)) == 16
+        download_rsr()
+        download_luts(aerosol_type='rayleigh_only')
+
+    def setup(self):
+        """Set up the benchmarks."""
+        import satpy
+        self.data_files = get_filenames(self.subdir)
+        satpy.CHUNK_SIZE = 2048
+
+    def time_load_one_channel(self):
+        """Time the loading of one channel."""
+        self.compute_channel("C01")
+
+    def peakmem_load_one_channel(self):
+        """Check peak memory usage of loading one channel."""
+        self.compute_channel("C01")
+
+    def time_load_true_color(self):
+        """Time the loading of the generation of true_color."""
+        self.compute_composite("true_color")
+
+    def peakmem_load_true_color(self):
+        """Check peak memory usage of the generation of true_color."""
+        self.compute_composite("true_color")
+
+    def time_save_true_color_nocorr_to_geotiff(self):
+        """Time the generation and saving of true_color_nocorr."""
+        self.save_composite_as_geotiff("true_color_nocorr")
+
+    def peakmem_save_true_color_to_geotiff(self):
+        """Check peak memory usage of the generation and saving of true_color_nocorr."""
+        self.save_composite_as_geotiff("true_color_nocorr")

benchmarks/ahi_hsd_benchmarks.py

@@ -51,47 +51,24 @@ def setup(self):
 
     def time_load_one_channel(self):
         """Time the loading of one channel."""
-        self.compute_B01()
+        self.compute_channel("B01")
 
     def peakmem_load_one_channel(self):
         """Check peak memory usage of loading one channel."""
-        self.compute_B01()
+        self.compute_channel("B01")
 
     def time_load_true_color(self):
         """Time the loading of the generation of true_color."""
-        self.compute_true_color()
+        self.compute_composite("true_color")
 
     def peakmem_load_true_color(self):
         """Check peak memory usage of the generation of true_color."""
-        self.compute_true_color()
+        self.compute_composite("true_color")
 
     def time_save_true_color_nocorr_to_geotiff(self):
         """Time the generation and saving of true_color_nocorr."""
-        self.save_true_color_nocorr_as_geotiff()
+        self.save_composite_as_geotiff("true_color_nocorr")
 
     def peakmem_save_true_color_to_geotiff(self):
         """Check peak memory usage of the generation and saving of true_color_nocorr."""
-        self.save_true_color_nocorr_as_geotiff()
-
-    def load_and_native_resample(self, composite):
-        """Load and native resample a composite."""
-        scn = self.load_no_padding(composite)
-        return scn.resample(resampler='native')
-
-    def compute_B01(self):
-        """Load and compute one channel."""
-        composite = "B01"
-        scn = self.load_no_padding(composite)
-        scn[composite].compute()
-
-    def compute_true_color(self):
-        """Compute a true color image."""
-        composite = "true_color"
-        lscn = self.load_and_native_resample(composite)
-        lscn[composite].compute()
-
-    def save_true_color_nocorr_as_geotiff(self):
-        """Save a true_color_nocorr to disk as geotiff."""
-        composite = "true_color_nocorr"
-        lscn = self.load_and_native_resample(composite)
-        lscn.save_dataset(composite, filename='test.tif', tiled=True)
+        self.save_composite_as_geotiff("true_color_nocorr")

benchmarks/seviri_hrit_benchmarks.py

@@ -51,42 +51,24 @@ def setup(self):
 
     def time_load_one_channel(self):
         """Time the loading of one channel."""
-        self.compute_vis006()
+        self.compute_channel("VIS006")
 
     def peakmem_load_one_channel(self):
         """Check peak memory usage of loading one channel."""
-        self.compute_vis006()
+        self.compute_channel("VIS006")
 
     def time_load_overview(self):
         """Time the loading of the generation of overview."""
-        self.compute_overview()
+        self.compute_composite("overview")
 
     def peakmem_load_overview(self):
         """Check peak memory usage of the generation of overview."""
-        self.compute_overview()
+        self.compute_composite("overview")
 
     def time_save_overview_to_geotiff(self):
         """Time the generation and saving of overview."""
-        self.save_overview_as_geotiff()
+        self.save_composite_as_geotiff("overview")
 
     def peakmem_save_overview_to_geotiff(self):
         """Check peak memory usage of the generation and saving of overview."""
-        self.save_overview_as_geotiff()
-
-    def compute_vis006(self):
-        """Load and compute one channel."""
-        composite = "VIS006"
-        scn = self.load_no_padding(composite)
-        scn[composite].compute()
-
-    def compute_overview(self):
-        """Compute an overview image."""
-        composite = "overview"
-        scn = self.load_no_padding(composite)
-        scn[composite].compute()
-
-    def save_overview_as_geotiff(self):
-        """Save a overview to disk as geotiff."""
-        composite = "overview"
-        scn = self.load_no_padding(composite)
-        scn.save_dataset(composite, filename='test.tif', tiled=True)
+        self.save_composite_as_geotiff("overview")

benchmarks/utils.py

@@ -36,3 +36,23 @@ def load_no_padding(self, composite):
         scn = Scene(filenames=self.data_files, reader=self.reader)
         scn.load([composite], pad_data=False)
         return scn
+
+    def load_and_native_resample(self, composite):
+        """Load and native resample a composite."""
+        scn = self.load_no_padding(composite)
+        return scn.resample(resampler='native')
+
+    def compute_composite(self, composite):
+        """Compute a true color image."""
+        lscn = self.load_and_native_resample(composite)
+        lscn[composite].compute()
+
+    def save_composite_as_geotiff(self, composite):
+        """Save a true_color_nocorr to disk as geotiff."""
+        lscn = self.load_and_native_resample(composite)
+        lscn.save_dataset(composite, filename='test.tif', tiled=True)
+
+    def compute_channel(self, channel):
+        """Load and compute one channel."""
+        scn = self.load_no_padding(channel)
+        scn[channel].compute()