Merge pull request #334 from djhoese/bugfix-geo-readers-proxy

Fix geo2grid readers with new ReaderProxy classes

integration_tests/features/geo2grid.feature

@@ -7,9 +7,18 @@ Feature: Test geo2grid output images
     When <command> runs
     Then the output matches with the files in <output>
 
-  Examples: ABI
-    | command                                                  | source          | output           |
-    | geo2grid.sh -r abi_l1b -w geotiff --num-workers 8 -vv -f | abi/input/test1 | abi/output/test1 |
-    | geo2grid.sh -r abi_l1b -w geotiff --num-workers 8 -vv -f | abi/input/test2 | abi/output/test2 |
-    | geo2grid.sh -r abi_l1b -w geotiff --num-workers 8 -vv -f | abi/input/test3 | abi/output/test3 |
-    | geo2grid.sh -r abi_l1b -w geotiff --num-workers 8 -vv -f | abi/input/test4 | abi/output/test4 |
+    Examples: ABI
+      | command                                                  | source          | output           |
+      | geo2grid.sh -r abi_l1b -w geotiff --num-workers 8 -vv -f | abi/input/test1 | abi/output/test1 |
+      | geo2grid.sh -r abi_l1b -w geotiff --num-workers 8 -vv -f | abi/input/test2 | abi/output/test2 |
+      | geo2grid.sh -r abi_l1b -w geotiff --num-workers 8 -vv -f | abi/input/test3 | abi/output/test3 |
+      | geo2grid.sh -r abi_l1b -w geotiff --num-workers 8 -vv -f | abi/input/test4 | abi/output/test4 |
+
+  Scenario Outline: Test list products output
+    Given input data from <source>
+    When <command> runs with --list-products
+    Then the printed output includes the products in <output>
+
+    Examples: ABI List Products
+      | command                                                                                      | source                      | output                       |
+      | geo2grid.sh -r abi_l1b -w geotiff -vv -f                                                     | abi_input/test1             | true_color,C01,natural_color |

integration_tests/features/polar2grid.feature

@@ -7,30 +7,39 @@ Feature: Test polar2grid output images
     When <command> runs
     Then the output matches with the files in <output>
 
-  Examples: ACSPO
-    | command                                              | source            |  output            |
-    | polar2grid.sh -r acspo -w geotiff -vv --grid-coverage=0.0 -f | acspo/input/test1 | acspo/output/test1 |
-    | polar2grid.sh -r acspo -w geotiff -vv --grid-coverage=0.0 -f | acspo/input/test2 | acspo/output/test2 |
-
- Examples: AMSR2_L1B
-    | command                                                                  | source            | output             |
-    | polar2grid.sh -r amsr2_l1b -w awips_tiled -vv --sector-id LCC --source-name SSEC --letters -g 211e -p btemp_36.5h btemp_89.0av -f | amsr2/input/test1 | amsr2/output/test1 |
-
-  Examples: AVHRR
-    | command                          | source            | output             |
-    | polar2grid.sh -r avhrr -w geotiff -vv -f | avhrr/input/test1 | avhrr/output/test1 |
-
-  Examples: MODIS
-    | command                                                                                                                                   | source            | output             |
-    | polar2grid.sh -r modis -w geotiff -vv -p true_color false_color --grid-configs ${datapath}/grid_configs/grid_example.conf -g miami -f | modis/input/test1 | modis/output/test1 |
-
-  Examples: VIIRS_L1B
-    | command                                                                                                                                                                                           | source                      | output                        |
-    | polar2grid.sh -r viirs_l1b -w geotiff -vv --grid-configs /data/dist/p2g_test_data/viirs_l1b_night/input/test1/my_grid.conf -g polar_europe -p adaptive_dnb dynamic_dnb histogram_dnb hncc_dnb -f  | viirs_l1b_night/input/test1 | viirs_l1b_night/output/test1  |
-
-  Examples: VIIRS_SDR
-    | command                                                                                      | source                      | output                       |
-    | polar2grid.sh -r viirs_sdr -w geotiff -vv --i-bands --m-bands -p adaptive_dnb dynamic_dnb -f | viirs_sdr_day/input/test1   | viirs_sdr_day/output/test1   |
-    | polar2grid.sh -r viirs_sdr -w geotiff -vv -f                                                 | viirs_sdr_day/input/test2   | viirs_sdr_day/output/test2   |
-    | polar2grid.sh -r viirs_sdr -w geotiff -vv -p adaptive_dnb dynamic_dnb -f                     | viirs_sdr_night/input/test1 | viirs_sdr_night/output/test1 |
-    | polar2grid.sh -r viirs_sdr -w geotiff -vv -p true_color false_color --grid-configs ${datapath}/grid_configs/grid_example.conf -g miami -f  | viirs/input/test1 | viirs/output/test1 |
+    Examples: ACSPO
+      | command                                              | source            |  output            |
+      | polar2grid.sh -r acspo -w geotiff -vv --grid-coverage=0.0 -f | acspo/input/test1 | acspo/output/test1 |
+      | polar2grid.sh -r acspo -w geotiff -vv --grid-coverage=0.0 -f | acspo/input/test2 | acspo/output/test2 |
+
+   Examples: AMSR2_L1B
+      | command                                                                  | source            | output             |
+      | polar2grid.sh -r amsr2_l1b -w awips_tiled -vv --sector-id LCC --source-name SSEC --letters -g 211e -p btemp_36.5h btemp_89.0av -f | amsr2/input/test1 | amsr2/output/test1 |
+
+    Examples: AVHRR
+      | command                          | source            | output             |
+      | polar2grid.sh -r avhrr -w geotiff -vv -f | avhrr/input/test1 | avhrr/output/test1 |
+
+    Examples: MODIS
+      | command                                                                                                                                   | source            | output             |
+      | polar2grid.sh -r modis -w geotiff -vv -p true_color false_color --grid-configs ${datapath}/grid_configs/grid_example.conf -g miami -f | modis/input/test1 | modis/output/test1 |
+
+    Examples: VIIRS_L1B
+      | command                                                                                                                                                                                           | source                      | output                        |
+      | polar2grid.sh -r viirs_l1b -w geotiff -vv --grid-configs /data/dist/p2g_test_data/viirs_l1b_night/input/test1/my_grid.conf -g polar_europe -p adaptive_dnb dynamic_dnb histogram_dnb hncc_dnb -f  | viirs_l1b_night/input/test1 | viirs_l1b_night/output/test1  |
+
+    Examples: VIIRS_SDR
+      | command                                                                                      | source                      | output                       |
+      | polar2grid.sh -r viirs_sdr -w geotiff -vv --i-bands --m-bands -p adaptive_dnb dynamic_dnb -f | viirs_sdr_day/input/test1   | viirs_sdr_day/output/test1   |
+      | polar2grid.sh -r viirs_sdr -w geotiff -vv -f                                                 | viirs_sdr_day/input/test2   | viirs_sdr_day/output/test2   |
+      | polar2grid.sh -r viirs_sdr -w geotiff -vv -p adaptive_dnb dynamic_dnb -f                     | viirs_sdr_night/input/test1 | viirs_sdr_night/output/test1 |
+      | polar2grid.sh -r viirs_sdr -w geotiff -vv -p true_color false_color --grid-configs ${datapath}/grid_configs/grid_example.conf -g miami -f  | viirs/input/test1 | viirs/output/test1 |
+
+  Scenario Outline: Test list products output
+    Given input data from <source>
+    When <command> runs with --list-products
+    Then the printed output includes the products in <output>
+
+    Examples: VIIRS_SDR List Products
+      | command                                                                                      | source                      | output                       |
+      | polar2grid.sh -r viirs_sdr -w geotiff -vv -f                                                 | viirs_sdr_day/input/test1   | true_color,i01,i01_rad,ifog  |

integration_tests/features/steps/compare_images.py

@@ -107,4 +107,6 @@ def step_impl(context, output):
     names_to_check = output.split(",")
     output = context.output
     for product_name in names_to_check:
-        assert product_name in output, f"Missing {product_name} in command output"
+        num_products_in_output = output.count(product_name + "\n")
+        assert num_products_in_output != 0, f"Missing {product_name} in command output"
+        assert num_products_in_output == 1, f"Too many of {product_name} in command output"

polar2grid/readers/_base.py

@@ -81,14 +81,14 @@ def _binary_name(self):
         return "polar2grid"
 
     @property
-    def _aliases(self):
+    def _aliases(self) -> dict[str, Union[DataQuery, str]]:
         return {}
 
     def get_default_products(self) -> list[str]:
         """Get products to load if user hasn't specified any others."""
         return []
 
-    def get_all_products(self):
+    def get_all_products(self) -> list[str]:
         """Get all polar2grid products that could be loaded."""
         return []
 

polar2grid/readers/abi_l1b.py

@@ -103,17 +103,34 @@
 
 """
 
-READER_PRODUCTS = ['C{:02d}'.format(x) for x in range(1, 17)]
+from __future__ import annotations
+
+from ._base import ReaderProxyBase
+
+READER_PRODUCTS = ["C{:02d}".format(x) for x in range(1, 17)]
 COMPOSITE_PRODUCTS = [
-    'true_color',
-    'natural_color',
-    'airmass',
-    'ash',
-    'dust',
-    'fog',
-    'night_microphysics',
+    "true_color",
+    "natural_color",
+    "airmass",
+    "ash",
+    "dust",
+    "fog",
+    "night_microphysics",
 ]
-DEFAULT_PRODUCTS = READER_PRODUCTS + COMPOSITE_PRODUCTS[:2]
+
+
+class ReaderProxy(ReaderProxyBase):
+    """Provide Polar2Grid-specific information about this reader's products."""
+
+    is_geo2grid_reader = True
+
+    def get_default_products(self) -> list[str]:
+        """Get products to load if users hasn't specified any others."""
+        return READER_PRODUCTS + COMPOSITE_PRODUCTS[:2]
+
+    def get_all_products(self) -> list[str]:
+        """Get all polar2grid products that could be loaded."""
+        return READER_PRODUCTS + COMPOSITE_PRODUCTS
 
 
 # def add_reader_argument_groups(parser):

polar2grid/readers/ahi_hrit.py

@@ -101,16 +101,33 @@
 
 """
 
-READER_PRODUCTS = ['B{:02d}'.format(x) for x in range(3, 17)]
+from __future__ import annotations
+
+from ._base import ReaderProxyBase
+
+READER_PRODUCTS = ["B{:02d}".format(x) for x in range(3, 17)]
 COMPOSITE_PRODUCTS = [
-    'natural_color',
-    'airmass',
-    'ash',
-    'dust',
-    'fog',
-    'night_microphysics',
+    "natural_color",
+    "airmass",
+    "ash",
+    "dust",
+    "fog",
+    "night_microphysics",
 ]
-DEFAULT_PRODUCTS = READER_PRODUCTS + COMPOSITE_PRODUCTS[:1]
+
+
+class ReaderProxy(ReaderProxyBase):
+    """Provide Polar2Grid-specific information about this reader's products."""
+
+    is_geo2grid_reader = True
+
+    def get_default_products(self) -> list[str]:
+        """Get products to load if users hasn't specified any others."""
+        return READER_PRODUCTS + COMPOSITE_PRODUCTS[:1]
+
+    def get_all_products(self) -> list[str]:
+        """Get all polar2grid products that could be loaded."""
+        return READER_PRODUCTS + COMPOSITE_PRODUCTS
 
 
 # def add_reader_argument_groups(parser):

polar2grid/readers/ahi_hsd.py

@@ -100,18 +100,38 @@
 
 """
 
-READER_PRODUCTS = ['B{:02d}'.format(x) for x in range(1, 17)]
+from __future__ import annotations
+
+from ._base import ReaderProxyBase
+
+from satpy import DataQuery
+
+READER_PRODUCTS = ["B{:02d}".format(x) for x in range(1, 17)]
 COMPOSITE_PRODUCTS = [
-    'true_color',
-    'natural_color',
-    'airmass',
-    'ash',
-    'dust',
-    'fog',
-    'night_microphysics',
+    "true_color",
+    "natural_color",
+    "airmass",
+    "ash",
+    "dust",
+    "fog",
+    "night_microphysics",
 ]
 DEFAULT_PRODUCTS = READER_PRODUCTS + COMPOSITE_PRODUCTS[:2]
 
 
+class ReaderProxy(ReaderProxyBase):
+    """Provide Polar2Grid-specific information about this reader's products."""
+
+    is_geo2grid_reader = True
+
+    def get_default_products(self) -> list[str]:
+        """Get products to load if users hasn't specified any others."""
+        return READER_PRODUCTS + COMPOSITE_PRODUCTS[:2]
+
+    def get_all_products(self) -> list[str]:
+        """Get all polar2grid products that could be loaded."""
+        return READER_PRODUCTS + COMPOSITE_PRODUCTS
+
+
 # def add_reader_argument_groups(parser):
 #     return group, None

polar2grid/utils/legacy_compat.py

@@ -117,9 +117,7 @@ def remove_unknown_user_products(
         new_user_products = []
         for user_name, satpy_name in zip(self._user_products, satpy_names):
             # convert DataID/DataQuery to string
-            satpy_name = (
-                satpy_name if isinstance(satpy_name, str) else satpy_name["name"]
-            )
+            satpy_name = satpy_name if isinstance(satpy_name, str) else satpy_name["name"]
             if satpy_name not in known_dataset_names:
                 continue
             new_user_products.append(user_name)
@@ -146,7 +144,13 @@ def convert_satpy_to_p2g_name(
         satpy_products: Iterable[Union[DataID]],
         possible_p2g_names: Optional[list[str]] = None,
     ):
-        """Get the P2G name for a series of Satpy names or DataIDs."""
+        """Get the P2G name for a series of Satpy names or DataIDs.
+
+        If a Satpy DataID does not have a Polar2Grid compatible name then
+        ``None`` is yielded. A name is not compatible if requesting it would
+        produce a different product than the original DataID.
+
+        """
         from satpy import DatasetDict
 
         if possible_p2g_names is None:
@@ -161,16 +165,26 @@ def convert_satpy_to_p2g_name(
                 continue
 
             if matching_satpy_id in satpy_id_to_p2g_name:
-                logger.warning(
-                    "Multiple product names map to the same identifier in Satpy"
-                )
-                print(
-                    matching_satpy_id, satpy_id_to_p2g_name[matching_satpy_id], p2g_name
-                )
+                logger.warning("Multiple product names map to the same identifier in Satpy")
+                print(matching_satpy_id, satpy_id_to_p2g_name[matching_satpy_id], p2g_name)
             satpy_id_to_p2g_name[matching_satpy_id] = p2g_name
 
         for satpy_product in satpy_products:
-            yield satpy_id_to_p2g_name.get(satpy_product, satpy_product["name"])
+            satpy_id_name = satpy_product["name"]
+            satpy_id_as_p2g_name = satpy_id_to_p2g_name.get(satpy_product)
+            satpy_name_is_p2g_name = satpy_id_name in possible_p2g_names
+            satpy_name_does_not_round_trip = satpy_id_dict[satpy_product] != satpy_product
+            if satpy_id_as_p2g_name is None:
+                # We can't yield this name if it is also a P2G name or if
+                # asking Satpy for the name doesn't return the same DataID
+                # product. Otherwise users would ask for X and not get X.
+                if satpy_name_is_p2g_name or satpy_name_does_not_round_trip:
+                    yield None
+                else:
+                    yield satpy_id_name
+                continue
+
+            yield satpy_id_to_p2g_name.get(satpy_product, satpy_id_name)
 
     def apply_p2g_name_to_scene(
         self,
@@ -186,23 +200,26 @@ def apply_p2g_name_to_scene(
         all_ids = list(scn.keys())
         all_p2g_names = list(self.convert_satpy_to_p2g_name(all_ids))
         for data_id, p2g_name in zip(all_ids, all_p2g_names):
+            if p2g_name is None:
+                # the Satpy ID doesn't have a Polar2Grid compatible name
+                logger.debug("Satpy DataID %s does not have a compatible polar2grid name.", data_id)
+                continue
             scn[data_id].attrs["p2g_name"] = p2g_name
             logger.debug("Mapping Satpy ID to P2G name: %s -> %s", data_id, p2g_name)
 
     def available_product_names(
         self, all_p2g_products: list[str], available_satpy_ids: list[DataID]
     ) -> tuple[list[str], list[str]]:
         """Get separate lists of available Satpy products and Polar2Grid products."""
-        available_ids_as_p2g_names = list(
-            self.convert_satpy_to_p2g_name(available_satpy_ids, all_p2g_products)
-        )
-        satpy_id_to_p2g_name = dict(
-            zip(available_satpy_ids, available_ids_as_p2g_names)
-        )
+        available_ids_as_p2g_names = list(self.convert_satpy_to_p2g_name(available_satpy_ids, all_p2g_products))
+        satpy_id_to_p2g_name = dict(zip(available_satpy_ids, available_ids_as_p2g_names))
         available_p2g_names = []
         available_satpy_names = []
         for satpy_id in available_satpy_ids:
             p2g_name = satpy_id_to_p2g_name[satpy_id]
+            if p2g_name is None:
+                # no Polar2Grid compatible name
+                continue
             if p2g_name in all_p2g_products:
                 available_p2g_names.append(p2g_name)
             else: