Thumbnail extension from s3 key (#91)

cbers2stac/layers/common/cbers_2_stac.py

@@ -7,7 +7,7 @@
 import typing
 import xml.etree.ElementTree as ET
 from collections import OrderedDict
-from typing import List
+from typing import Any, Dict, List
 
 import utm
 
@@ -528,22 +528,44 @@ def candidate_xml_files(xml_file: str) -> List[str]:
     return xml_options
 
 
-def convert_inpe_to_stac(inpe_metadata_filename, stac_metadata_filename, buckets):
+def convert_inpe_to_stac(
+    inpe_metadata_filename: str,
+    stac_metadata_filename: str,
+    buckets: Dict[str, Any],
+    thumbnail_extension: str = None,
+):
     """
     Generate STAC item in stac_metadata from inpe_metadata.
 
     Input:
-    inpe_metadata(string): CBERS metadata (INPE format) file
-    stac_metadata(string): STAC item metadata file to be written, if None then
-                           no file is written.
+    inpe_metadata: CBERS metadata (INPE format) file
+    stac_metadata: STAC item metadata file to be written, if None then
+                   no file is written.
     buckets: buckets dictionary
+    thumbnail_extension: force the thumbnail extension to be the one
+                         defined, e.g., "png" or "jpg". Required to
+                         work with CBERS4 data which may have both extensions.
 
     Return:
     Dictionary based on INPE's metadata
     """
 
     meta = get_keys_from_cbers_am(inpe_metadata_filename)
     stac_meta = build_stac_item_keys(meta, buckets)
+    if thumbnail_extension:
+        # Replace str after last "." with the override extension
+        pieces = stac_meta["assets"]["thumbnail"]["href"].split(".")
+        stac_meta["assets"]["thumbnail"]["href"] = ".".join(pieces[0:-1])
+        stac_meta["assets"]["thumbnail"]["href"] += f".{thumbnail_extension}"
+        # ... and check type also
+        ext_to_type: Dict[str, str] = {
+            "png": "image/png",
+            "jpg": "image/jpeg",
+        }
+        assert (
+            ext_to_type.get(thumbnail_extension) is not None
+        ), f"type unknown for {thumbnail_extension}"
+        stac_meta["assets"]["thumbnail"]["type"] = ext_to_type[thumbnail_extension]
     if stac_metadata_filename:
         create_json_item(stac_meta, stac_metadata_filename)
     return stac_meta

cbers2stac/process_new_scene_queue/code.py

@@ -171,6 +171,7 @@ def process_message(
 
     LOGGER.info(msg["key"])
     metadata_keys = get_s3_keys(msg["key"])
+    thumbnail_extension = msg["key"].split(".")[-1]
 
     assert metadata_keys["quicklook_keys"]["camera"] in (
         "MUX",
@@ -208,6 +209,7 @@ def process_message(
         inpe_metadata_filename=local_inpe_metadata,
         stac_metadata_filename=local_stac_item,
         buckets=buckets,
+        thumbnail_extension=thumbnail_extension,
     )
     # Upload STAC item file
     with open(local_stac_item, "rb") as data:

pytest.ini

@@ -4,6 +4,7 @@ filterwarnings =
     ignore:The 'body' parameter is deprecated for the 'update' API and will be removed in a future version:DeprecationWarning
     ignore:The 'body' parameter is deprecated for the 'search' API and will be removed in a future version:DeprecationWarning
     ignore:Using positional arguments for APIs is deprecated and will be disabled in 8.0.0:DeprecationWarning
+    ignore:the imp module is deprecated:DeprecationWarning
 
 markers =
     sqs_queue_args: SQS fixture params

test/cbers_am_2_stac_test.py

@@ -724,6 +724,19 @@ def test_convert_inpe_to_stac(tmp_path):  # pylint: disable=too-many-statements
     jsv.validate(output_filename)
     rescb4mux = diff_files(ref_output_filename, output_filename)
 
+    # MUX, CB4 with thumbnail extension override
+    output_filename = tmp_path / "CBERS_4_MUX_20170528_090_084_L2.json"
+    ref_output_filename = "test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2_PNG.json"
+    convert_inpe_to_stac(
+        inpe_metadata_filename="test/fixtures/CBERS_4_MUX_20170528"
+        "_090_084_L2_BAND6.xml",
+        stac_metadata_filename=output_filename,
+        buckets=buckets,
+        thumbnail_extension="png",
+    )
+    jsv.validate(output_filename)
+    rescb4mux = diff_files(ref_output_filename, output_filename)
+
     # AWFI, CB4
     output_filename = tmp_path / "CBERS_4_AWFI_20170409_167_123_L4.json"
     ref_output_filename = "test/fixtures/ref_CBERS_4_AWFI_20170409_167_123_L4.json"

test/fixtures/ref_CBERS_4_MUX_20170528_090_084_L2_PNG.json

@@ -0,0 +1,129 @@
+{
+  "stac_version": "1.0.0",
+  "stac_extensions": [
+    "https://stac-extensions.github.io/projection/v1.0.0/schema.json",
+    "https://stac-extensions.github.io/view/v1.0.0/schema.json",
+    "https://stac-extensions.github.io/eo/v1.0.0/schema.json",
+    "https://stac-extensions.github.io/sat/v1.0.0/schema.json"
+  ],
+  "id": "CBERS_4_MUX_20170528_090_084_L2",
+  "type": "Feature",
+  "geometry": {
+    "type": "MultiPolygon",
+    "coordinates": [
+      [
+        [
+          [
+            23.463987,
+            13.891487
+          ],
+          [
+            24.567134,
+            13.723733
+          ],
+          [
+            24.811268,
+            14.786396
+          ],
+          [
+            23.703021,
+            14.95489
+          ],
+          [
+            23.463987,
+            13.891487
+          ]
+        ]
+      ]
+    ]
+  },
+  "bbox": [
+    23.465111,
+    13.700498,
+    24.812825,
+    14.98818
+  ],
+  "collection": "CBERS4-MUX",
+  "properties": {
+    "datetime": "2017-05-28T09:01:17Z",
+    "platform": "cbers-4",
+    "instruments": [
+      "MUX"
+    ],
+    "gsd": 20.0,
+    "view:sun_azimuth": 66.2923,
+    "view:sun_elevation": 70.3079,
+    "view:off_nadir": 0.00744884,
+    "proj:epsg": 32635,
+    "sat:platform_international_designator": "2014-079A",
+    "sat:orbit_state": "descending",
+    "cbers:data_type": "L2",
+    "cbers:path": 90,
+    "cbers:row": 84
+  },
+  "links": [
+    {
+      "rel": "self",
+      "href": "https://cbers-stac.s3.amazonaws.com/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2.json"
+    },
+    {
+      "rel": "parent",
+      "href": "https://cbers-stac.s3.amazonaws.com/CBERS4/MUX/090/084/catalog.json"
+    },
+    {
+      "rel": "collection",
+      "href": "https://cbers-stac.s3.amazonaws.com/CBERS4/MUX/collection.json"
+    }
+  ],
+  "assets": {
+    "thumbnail": {
+      "href": "https://s3.amazonaws.com/cbers-meta-pds/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084.png",
+      "type": "image/png"
+    },
+    "metadata": {
+      "href": "s3://cbers-pds/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084_L2_BAND6.xml",
+      "title": "INPE original metadata",
+      "type": "text/xml"
+    },
+    "B5": {
+      "href": "s3://cbers-pds/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084_L2_BAND5.tif",
+      "type": "image/tiff; application=geotiff; profile=cloud-optimized",
+      "eo:bands": [
+        {
+          "name": "B5",
+          "common_name": "blue"
+        }
+      ]
+    },
+    "B6": {
+      "href": "s3://cbers-pds/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084_L2_BAND6.tif",
+      "type": "image/tiff; application=geotiff; profile=cloud-optimized",
+      "eo:bands": [
+        {
+          "name": "B6",
+          "common_name": "green"
+        }
+      ]
+    },
+    "B7": {
+      "href": "s3://cbers-pds/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084_L2_BAND7.tif",
+      "type": "image/tiff; application=geotiff; profile=cloud-optimized",
+      "eo:bands": [
+        {
+          "name": "B7",
+          "common_name": "red"
+        }
+      ]
+    },
+    "B8": {
+      "href": "s3://cbers-pds/CBERS4/MUX/090/084/CBERS_4_MUX_20170528_090_084_L2/CBERS_4_MUX_20170528_090_084_L2_BAND8.tif",
+      "type": "image/tiff; application=geotiff; profile=cloud-optimized",
+      "eo:bands": [
+        {
+          "name": "B8",
+          "common_name": "nir"
+        }
+      ]
+    }
+  }
+}