From 36d23bc0bab4315886d8d9501ffd369c3dc7e35c Mon Sep 17 00:00:00 2001 From: Anthony Cintron Roman Date: Mon, 20 Jul 2026 15:04:37 -0700 Subject: [PATCH] feat: Open Data Catalog explorer with server-side clip Add an Open Data Catalog side panel to the Create Image Layer page so analysts can browse open disaster imagery (Vantor/Maxar + Planet Open Data), preview it, and build an image layer without hunting for COG URLs. Catalog UI (ui/src/Components/OpenDataCatalog/): - Dynamic discovery + merge of every event across the Vantor S3 STAC and Planet Source-Cooperative STAC catalogs; framework-free module with a discoverEvents()/fetchEventCatalog() contract (swap-to-backend seam). - Azure Maps footprints + TiTiler COG preview (works with no Azure Maps key); preview tile loading indicator. - Add a scene to pre/post in one click with source-type + capture-date auto-fill; buttons are phase-scoped (pre->Pre, post->Post). - Two-way map<->list selection: click a footprint to scroll/highlight/expand the list row; footprints stay clickable during preview. Server-side clip AOI: - Draw a box that sets a layer-level clipBbox ([w,s,e,n] EPSG:4326); imagery prep clips the pre/post mosaics via gdalwarp -te/-te_srs (GDAL reprojects the bounds). The catalog filters scenes overlapping the AOI and badges scenes that fully cover it, so pre/post share one AOI. - Model ImageLayer.clipBbox + validate_clip_bbox; PutLayer validates/carries it; threaded through the imagery-prep config -> ImageryWorkflow -> mosaic_imagery. Verified end-to-end (output bounds match the drawn box). Supporting fixes: - Allow data.source.coop on the imagery allowlist (UI + hastelib) and route non-S3/Blob allowlisted hosts through the generic HTTP downloader, so Planet imagery downloads at prep time. - Local dev proxy (docker/nginx.conf): TiTiler timeouts + client_max_body_size for chunk uploads. - QUICKSTART: DOCKER_GID must be 0 on macOS Docker Desktop. Azure Maps auth is unchanged (existing anonymous/AAD Client-ID flow). Spec: spec/features/open-data-catalog/. Co-Authored-By: Claude Opus 4.8 (1M context) --- QUICKSTART.md | 20 +- api/hastefuncapi/function_app.py | 5 + docker/docker-compose.yml | 2 + docker/nginx.conf | 12 + hastelib/src/hastegeo/core/models/projects.py | 5 + .../src/hastegeo/core/processors/imagery.py | 3 + .../src/hastegeo/core/utils/downloader.py | 11 + hastelib/src/hastegeo/core/utils/imagery.py | 24 +- .../src/hastegeo/core/utils/url_allowlist.py | 43 +- .../src/hastegeo/workflows/prepare_imagery.py | 13 +- .../tests/core/utils/test_url_allowlist.py | 63 ++ spec/features/open-data-catalog/README.md | 78 +++ spec/features/open-data-catalog/data-model.md | 144 +++++ spec/features/open-data-catalog/design.md | 206 +++++++ .../open-data-catalog/impact-analysis.md | 74 +++ spec/features/open-data-catalog/plan.md | 78 +++ spec/features/open-data-catalog/rollout.md | 68 +++ spec/features/open-data-catalog/test-plan.md | 86 +++ .../open-data-catalog/user-stories.md | 180 ++++++ .../Components/CreateEditImageLayerForm.jsx | 44 ++ .../Components/CreateEditImageLayerHelper.js | 63 ++ .../OpenDataCatalog/OpenDataCatalogMap.jsx | 407 +++++++++++++ .../OpenDataCatalog/OpenDataCatalogPanel.jsx | 545 +++++++++++++++++ .../OpenDataCatalog/SceneListItem.jsx | 243 ++++++++ .../OpenDataCatalog/openDataCatalog.js | 567 ++++++++++++++++++ ui/src/util/validation.js | 8 +- 26 files changed, 2979 insertions(+), 13 deletions(-) create mode 100644 spec/features/open-data-catalog/README.md create mode 100644 spec/features/open-data-catalog/data-model.md create mode 100644 spec/features/open-data-catalog/design.md create mode 100644 spec/features/open-data-catalog/impact-analysis.md create mode 100644 spec/features/open-data-catalog/plan.md create mode 100644 spec/features/open-data-catalog/rollout.md create mode 100644 spec/features/open-data-catalog/test-plan.md create mode 100644 spec/features/open-data-catalog/user-stories.md create mode 100644 ui/src/Components/OpenDataCatalog/OpenDataCatalogMap.jsx create mode 100644 ui/src/Components/OpenDataCatalog/OpenDataCatalogPanel.jsx create mode 100644 ui/src/Components/OpenDataCatalog/SceneListItem.jsx create mode 100644 ui/src/Components/OpenDataCatalog/openDataCatalog.js diff --git a/QUICKSTART.md b/QUICKSTART.md index 032d81e..f1a59af 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -94,15 +94,23 @@ Compute the values: # the VM's PUBLIC IP if you'll reach it from another machine. HOST_IP=localhost -# DOCKER_GID: GID of the docker socket, so hastefuncqueues can spawn siblings. -# Linux: +# DOCKER_GID: GID that owns the docker socket *inside the container*, so +# hastefuncqueues can spawn sibling containers (imageryprep/training). +# Linux: the mounted socket keeps its host GID, so read it from the host: DOCKER_GID=$(stat -c '%g' /var/run/docker.sock) -# macOS (BSD stat — the -c form does NOT work). Docker Desktop puts the socket -# under $HOME, NOT /var/run, so resolve the real path from the docker context: -# SOCK=$(docker context inspect --format '{{.Endpoints.docker.Host}}' | sed 's|unix://||') -# DOCKER_GID=$(stat -f '%g' "$SOCK") # typically 20 (staff) on macOS +# macOS (Docker Desktop): the socket is bind-mounted into the container as +# root:root, so the container-visible GID is 0 — NOT the host `stat` value +# (which is typically 20/staff). Use 0 or the worker gets +# `PermissionError(13, 'Permission denied')` and every job spawn fails: +# DOCKER_GID=0 ``` +> **⚠️ macOS gotcha:** set `DOCKER_GID=0`. The host socket may show GID 20, but +> Docker Desktop presents it to the container as `root:root`, so `group_add` +> must be 0. A non-zero value here is the usual cause of jobs that submit, +> immediately fail, and show +> `Error while fetching server API version: ... PermissionError(13)`. + Write the file (adjust `HASTE_ENABLE_GPU` per your Gate-1 profile): ```bash diff --git a/api/hastefuncapi/function_app.py b/api/hastefuncapi/function_app.py index 112f43a..7683072 100644 --- a/api/hastefuncapi/function_app.py +++ b/api/hastefuncapi/function_app.py @@ -48,6 +48,7 @@ from hastegeo.core.utils.logs import Logger from hastegeo.core.utils.metadata import MetadataUtils from hastegeo.core.utils.url_allowlist import ( + validate_clip_bbox, validate_image_layer_imagery_urls, validate_image_layer_user_footprints_url, ) @@ -819,6 +820,10 @@ async def PutLayer(req: func.HttpRequest) -> func.HttpResponse: if footprint_url_error: return func.HttpResponse(footprint_url_error, status_code=400) + clip_bbox_error = validate_clip_bbox(image_data) + if clip_bbox_error: + return func.HttpResponse(clip_bbox_error, status_code=400) + if image_data.imageLayerId is None: image_data.imageLayerId = MetadataUtils.generate_id() if image_data.creationDate is None: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7755045..44680e3 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -173,6 +173,8 @@ services: # Memory settings for training containers (increase for large VMs) HASTE_DOCKER_SHM_SIZE: "${HASTE_DOCKER_SHM_SIZE:-64g}" HASTE_DOCKER_MEM_LIMIT: "${HASTE_DOCKER_MEM_LIMIT:-256g}" + # Debug: keep failed spawned containers so their logs can be inspected. + CLEANUP_CONTAINERS: "${CLEANUP_CONTAINERS:-1}" volumes: - ../data:/app/data - azurite-data:/shared/azurite # Mount azurite blob storage for LocalRunner access diff --git a/docker/nginx.conf b/docker/nginx.conf index f7f0bf2..1f159c5 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -7,6 +7,11 @@ events { } http { + # The image-layer file uploader (incl. Open Data Catalog clip export) + # POSTs 4 MB chunks to /api/UploadFileByChunk. nginx's default 1 MB body + # limit rejects those with 413, so raise it well above the chunk size. + client_max_body_size 64m; + upstream functions_backend { server hastefuncapi:8080; } @@ -42,6 +47,13 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + + # Clip/crop requests read from remote COGs and can take longer than + # nginx's default 60s to generate + stream. Give them headroom so + # the Open Data Catalog "Clip to area" export doesn't 504. + proxy_connect_timeout 30s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; } location /api/ { diff --git a/hastelib/src/hastegeo/core/models/projects.py b/hastelib/src/hastegeo/core/models/projects.py index b70c434..bb8732e 100644 --- a/hastelib/src/hastegeo/core/models/projects.py +++ b/hastelib/src/hastegeo/core/models/projects.py @@ -752,6 +752,11 @@ class ImageLayer(BaseModel): labelsUrl: Optional[str] = Field(default=None) buildingFootprintsUrl: Optional[str] = Field(default=None) userBuildingFootprintsUrl: Optional[str] = Field(default=None) + # Optional server-side clip AOI, [west, south, east, north] in EPSG:4326. + # When set, imagery prep clips the pre/post mosaics to this box (gdalwarp + # -te) so only the drawn area is processed/labeled. Set by the Open Data + # Catalog "clip to area" flow. + clipBbox: Optional[list[float]] = Field(default=None) validAreaMaskUrl: Optional[str] = Field(default=None) dependsOn: Optional[tuple[str, str]] = Field( default=("Project", "projectId") diff --git a/hastelib/src/hastegeo/core/processors/imagery.py b/hastelib/src/hastegeo/core/processors/imagery.py index 0196974..9497fad 100644 --- a/hastelib/src/hastegeo/core/processors/imagery.py +++ b/hastelib/src/hastegeo/core/processors/imagery.py @@ -292,6 +292,9 @@ def _execute_image_preprocess(self): "user_building_footprints_url": ( self.image_data.userBuildingFootprintsUrl ), + # Optional server-side clip AOI ([w, s, e, n] EPSG:4326). When set, + # prepare-imagery clips the pre/post mosaics to this box. + "clip_bbox": self.image_data.clipBbox, } self.storage.save( identifier=self.image_data.imageLayerId, diff --git a/hastelib/src/hastegeo/core/utils/downloader.py b/hastelib/src/hastegeo/core/utils/downloader.py index 8e42522..3a6ae03 100644 --- a/hastelib/src/hastegeo/core/utils/downloader.py +++ b/hastelib/src/hastegeo/core/utils/downloader.py @@ -45,6 +45,11 @@ def download_imagery(self, **kwargs): dst_directory = kwargs.get("dst_directory", "./downloads") azure_urls = [] aws_urls = [] + # Allowlisted hosts without a dedicated SDK path (e.g. Source + # Cooperative / data.source.coop, used by the Planet Open Data + # catalog) are fetched over plain HTTPS via + # download_imagery_from_urls. + http_urls = [] for url in urls: try: @@ -53,6 +58,8 @@ def download_imagery(self, **kwargs): azure_urls.append(url) elif url_type == "awss3": aws_urls.append(url) + else: + http_urls.append(url) except Exception as e: self.logger.warning(f"Skipping URL not in allowlist: {e}") @@ -65,6 +72,10 @@ def download_imagery(self, **kwargs): file_paths.extend( self.download_aws_s3_files(aws_urls, dst_directory) ) + if http_urls: + file_paths.extend( + self.download_imagery_from_urls(http_urls, dst_directory) + ) return file_paths @retry( diff --git a/hastelib/src/hastegeo/core/utils/imagery.py b/hastelib/src/hastegeo/core/utils/imagery.py index ee6c805..9ae8d31 100644 --- a/hastelib/src/hastegeo/core/utils/imagery.py +++ b/hastelib/src/hastegeo/core/utils/imagery.py @@ -456,12 +456,17 @@ def mosaic_imagery( tif_files: List[str], output_file_path: str = None, gdal_warp_params: str = "", + clip_bbox: List[float] = None, ) -> str: """ Parameters: tif_files (List[str]): List of paths to the input TIFF files. output_file_path (str, optional): Path to the output COG file. If not provided, a default path is generated. gdal_warp_params (str): GDAL warp parameters as a string. + clip_bbox (List[float], optional): [west, south, east, north] in + EPSG:4326. When provided, the output mosaic is clipped to this + box via gdalwarp ``-te``/``-te_srs`` (GDAL reprojects the bounds + to the mosaic's CRS), so only the drawn AOI is produced. Returns: str: Path to the output COG file. @@ -549,9 +554,22 @@ def mosaic_imagery( raise RuntimeError("VRT file does not exist on disk") # Use gdal.Warp to convert the VRT to a COG (Cloud Optimized GeoTIFF) - warp_options = gdal.WarpOptions( - options=gdal_warp_params.split() - ) + warp_option_list = gdal_warp_params.split() + if clip_bbox: + west, south, east, north = (float(v) for v in clip_bbox) + # -te bounds are given in EPSG:4326 (-te_srs); GDAL + # reprojects them to the mosaic's CRS before clipping. + warp_option_list += [ + "-te", + str(west), + str(south), + str(east), + str(north), + "-te_srs", + "EPSG:4326", + ] + logger.info(f"Clipping mosaic to AOI bbox {clip_bbox}") + warp_options = gdal.WarpOptions(options=warp_option_list) result = gdal.Warp( output_file_path, str(vrt_path), options=warp_options ) diff --git a/hastelib/src/hastegeo/core/utils/url_allowlist.py b/hastelib/src/hastegeo/core/utils/url_allowlist.py index 188cb5f..b70eba6 100644 --- a/hastelib/src/hastegeo/core/utils/url_allowlist.py +++ b/hastelib/src/hastegeo/core/utils/url_allowlist.py @@ -16,8 +16,9 @@ # Human-readable description used in user-facing error messages. Keep in # sync with the JS allowlist in ui/src/util/validation.js. ALLOWED_HOST_DESCRIPTION = ( - "Azure Blob Storage (*.blob.core.windows.net) " - "or AWS S3 (*.amazonaws.com)" + "Azure Blob Storage (*.blob.core.windows.net), " + "AWS S3 (*.amazonaws.com), " + "or Source Cooperative (data.source.coop)" ) # Building-footprint URLs additionally permit the host of the configured @@ -62,6 +63,13 @@ def validate_imagery_url(url: str) -> str: ): return "awss3" + # Source Cooperative — Planet Open Data STAC catalogs and COGs surfaced + # by the Open Data Catalog explorer are served from data.source.coop. + # This is a public, well-known open-data host (not user-controlled), + # so allowlisting it does not meaningfully widen SSRF exposure. + if host == "data.source.coop" or host.endswith(".source.coop"): + return "sourcecoop" + raise ValueError( f"URL host {host!r} is not on the allowlist of permitted imagery sources" ) @@ -232,3 +240,34 @@ def validate_image_layer_user_footprints_url(image_layer) -> Optional[str]: f"Rejected host: {host}." ) return None + + +def validate_clip_bbox(image_layer) -> Optional[str]: + """Validate the optional server-side clip AOI on an ImageLayer. + + Returns a user-facing error message if ``clipBbox`` is set but malformed, + or ``None`` when it is absent or valid. Expected shape: + ``[west, south, east, north]`` in EPSG:4326 with ``west < east``, + ``south < north`` and coordinates within valid lon/lat ranges. Callers + (PutLayer) treat a non-None return as a hard 400. + + Lives here (not in utils/aoi.py) so the lightweight PutLayer validation + path doesn't pull rasterio/GDAL into the API process just to bounds-check + four numbers. + """ + bbox = getattr(image_layer, "clipBbox", None) + if bbox is None: + return None + if not isinstance(bbox, (list, tuple)) or len(bbox) != 4: + return "clipBbox must be a [west, south, east, north] array." + try: + west, south, east, north = (float(v) for v in bbox) + except (TypeError, ValueError): + return "clipBbox values must be numbers." + if not (-180 <= west <= 180 and -180 <= east <= 180): + return "clipBbox longitudes must be within [-180, 180]." + if not (-90 <= south <= 90 and -90 <= north <= 90): + return "clipBbox latitudes must be within [-90, 90]." + if west >= east or south >= north: + return "clipBbox must have west < east and south < north." + return None diff --git a/hastelib/src/hastegeo/workflows/prepare_imagery.py b/hastelib/src/hastegeo/workflows/prepare_imagery.py index a3ae768..5e852f7 100644 --- a/hastelib/src/hastegeo/workflows/prepare_imagery.py +++ b/hastelib/src/hastegeo/workflows/prepare_imagery.py @@ -82,6 +82,7 @@ def __init__( fine_tune=False, dst_directory=None, user_building_footprints_url=None, + clip_bbox=None, ): """Initialize the ImageryWorkflow class. Args: @@ -107,6 +108,9 @@ def __init__( self.fine_tune = fine_tune self.dst_directory = dst_directory or os.path.join(".", "outputs") self.user_building_footprints_url = user_building_footprints_url + # Optional server-side clip AOI ([w, s, e, n] EPSG:4326). Applied to + # both pre and post mosaics so only the drawn area is processed. + self.clip_bbox = clip_bbox self.pre_event_raw_paths = [] self.pre_event_preview_paths = [] @@ -205,6 +209,7 @@ def process_pre_event(self): self.pre_event_raw_paths, dst_directory=self.dst_directory, save_as_prefix=save_as_prefix, + clip_bbox=self.clip_bbox, ) cog_file_name = self.generate_prefix(PRE_EVENT_PROCESSED_COG_PREFIX) @@ -243,6 +248,7 @@ def process_post_event(self): self.post_event_raw_paths, dst_directory=self.dst_directory, save_as_prefix=save_as_prefix, + clip_bbox=self.clip_bbox, ) # The RGB band and scale adjusted imagery serves as supporting input to training and inference @@ -763,12 +769,15 @@ def _save_raw_imagery( return raw_image_paths, preview_image_paths -def _create_mosaic_cog(tif_files, dst_directory=None, save_as_prefix=None): +def _create_mosaic_cog( + tif_files, dst_directory=None, save_as_prefix=None, clip_bbox=None +): """Create a mosaic of TIFF files and convert to COG. Args: tif_files (list): List of TIFF files to mosaic (combine). dst_directory (str): (Optional) Directory to save the mosaic file. Defaults to current directory save_as_prefix (str): (Optional) Prefix for the saved mosaic file. + clip_bbox (list): (Optional) [w, s, e, n] EPSG:4326 AOI to clip to. Returns: str: Path to the created mosaic file.""" dst_directory = dst_directory or "." @@ -776,6 +785,7 @@ def _create_mosaic_cog(tif_files, dst_directory=None, save_as_prefix=None): tif_files, os.path.join(dst_directory, f"{save_as_prefix}.tif"), GDAL_WARP_PARAMS, + clip_bbox=clip_bbox, ) @@ -891,6 +901,7 @@ def main(): user_building_footprints_url=config.get( "user_building_footprints_url" ), + clip_bbox=config.get("clip_bbox"), ) log_progress("Downloading imagery") diff --git a/hastelib/tests/core/utils/test_url_allowlist.py b/hastelib/tests/core/utils/test_url_allowlist.py index 7452eb5..2d87968 100644 --- a/hastelib/tests/core/utils/test_url_allowlist.py +++ b/hastelib/tests/core/utils/test_url_allowlist.py @@ -41,6 +41,17 @@ def test_accepts_aws_s3_host(self): "awss3", ) + def test_accepts_source_cooperative_host(self): + # Planet Open Data STAC catalogs/COGs surfaced by the Open Data + # Catalog explorer are served from data.source.coop. + self.assertEqual( + validate_imagery_url( + "https://data.source.coop/planet/venezuela-earthquake" + "-2026-06-24/post-event/scene.tif" + ), + "sourcecoop", + ) + def test_rejects_arbitrary_host(self): with self.assertRaises(ValueError): validate_imagery_url("https://evil.example.com/x.tif") @@ -322,5 +333,57 @@ def test_bad_url_reports_rejected_host(self): self.assertIn("evil.example", msg) +class TestValidateClipBbox(unittest.TestCase): + """Tests for validate_clip_bbox (server-side clip AOI on an ImageLayer).""" + + def _layer(self, bbox): + from types import SimpleNamespace + + return SimpleNamespace(clipBbox=bbox) + + def test_none_returns_none(self): + from hastegeo.core.utils.url_allowlist import validate_clip_bbox + + self.assertIsNone(validate_clip_bbox(self._layer(None))) + + def test_valid_bbox_returns_none(self): + from hastegeo.core.utils.url_allowlist import validate_clip_bbox + + self.assertIsNone( + validate_clip_bbox(self._layer([-67.03, 10.54, -66.97, 10.61])) + ) + + def test_wrong_length_rejected(self): + from hastegeo.core.utils.url_allowlist import validate_clip_bbox + + self.assertIsNotNone(validate_clip_bbox(self._layer([1, 2, 3]))) + + def test_non_numeric_rejected(self): + from hastegeo.core.utils.url_allowlist import validate_clip_bbox + + self.assertIsNotNone( + validate_clip_bbox(self._layer([-67, "x", -66, 11])) + ) + + def test_out_of_range_rejected(self): + from hastegeo.core.utils.url_allowlist import validate_clip_bbox + + self.assertIsNotNone( + validate_clip_bbox(self._layer([-67, 10, -66, 99])) + ) + + def test_inverted_bounds_rejected(self): + from hastegeo.core.utils.url_allowlist import validate_clip_bbox + + # west >= east + self.assertIsNotNone( + validate_clip_bbox(self._layer([-66, 10.5, -67, 10.6])) + ) + # south >= north + self.assertIsNotNone( + validate_clip_bbox(self._layer([-67, 10.6, -66, 10.5])) + ) + + if __name__ == "__main__": unittest.main() diff --git a/spec/features/open-data-catalog/README.md b/spec/features/open-data-catalog/README.md new file mode 100644 index 0000000..c094e7b --- /dev/null +++ b/spec/features/open-data-catalog/README.md @@ -0,0 +1,78 @@ +# Feature: Open Data Catalog Explorer + +**Status:** implemented +**Author:** HASTE engineering team +**Date:** 2026-07-20 +**Target Release:** TBD +**Priority:** P2 +**Work Item:** — + +## Summary + +A side panel on the **Create Image Layer** page that lets disaster analysts +browse open disaster-response imagery from the **Vantor/Maxar** and **Planet** +Open Data Programs, preview a scene's imagery on a map, and add scenes directly +into an image layer's pre-/post-event inputs — including drawing a clip Area of +Interest (AOI) that the imagery-prep workflow applies to the mosaics. This +removes the manual, error-prone step of hunting down Cloud-Optimized GeoTIFF +(COG) URLs on S3/STAC catalogs and pasting them into the form. + +## Motivation + +- Analysts currently must locate a disaster's open imagery on external S3/STAC + catalogs, copy exact COG URLs, and paste them into the layer form — slow and + easy to get wrong (wrong host, wrong phase, no AOI). +- Open data (Vantor/Maxar, Planet) is the primary imagery source for many + responses; making it first-class in the app shortens time-to-assessment. +- Reference prototype (self-contained, OpenLayers): the **Open Disaster + Response Data Visualizer** — + + +## Success Criteria + +- [x] Analysts can discover every available disaster event across Vantor + Planet from within the app. +- [x] A scene's COG URL can be added to pre/post imagery in one click, with source-type + capture-date auto-filled. +- [x] A drawn AOI clips the produced imagery to just that area (verified end-to-end). +- [x] Planet (`data.source.coop`) imagery downloads successfully through the imagery-prep workflow. +- [x] Pre imagery can only be added to Pre and post to Post from the catalog UI. + +## HASTE Components Affected + +| Component | Impact | +|---|---| +| `hastelib/src/hastegeo/core/models/` | `ImageLayer.clipBbox` field | +| `hastelib/src/hastegeo/core/utils/` | `url_allowlist` (source.coop + `validate_clip_bbox`); `downloader` HTTP route; `imagery.mosaic_imagery` clip | +| `hastelib/src/hastegeo/core/processors/` | `imagery` — `clip_bbox` in the prep config | +| `hastelib/src/hastegeo/workflows/prepare_imagery.py` | thread `clip_bbox` into the mosaic step | +| `api/hastefuncapi/` | `PutLayer` validates + carries `clipBbox` | +| `ui/src/Components/` | new `OpenDataCatalog/`; integration into `CreateEditImageLayer*`; `util/validation.js` | +| `docker/` | `nginx.conf` (titiler timeouts + body size); `docker-compose.yml` (cleanup knob) | + +## Related Specs + +| Spec | Relationship | +|---|---| +| [gdal-compensating-controls](../gdal-compensating-controls/) | related — imagery prep runs under the GDAL driver allowlist | + +## Document Index + +| Document | Purpose | Status | +|---|---|---| +| [plan.md](plan.md) | Execution plan, milestones, phases | implemented | +| [impact-analysis.md](impact-analysis.md) | Risk, dependencies, blast radius | implemented | +| [user-stories.md](user-stories.md) | User stories & acceptance criteria | implemented | +| [design.md](design.md) | Technical design & API contracts | implemented | +| [data-model.md](data-model.md) | Schema / model / blob changes | implemented | +| [test-plan.md](test-plan.md) | Test strategy & coverage matrix | implemented | +| [rollout.md](rollout.md) | Rollout strategy, flags, rollback | draft | + +## Decision Log + +| Date | Decision | Rationale | +|---|---|---| +| 2026-07-17 | Client-side discovery, isolated in one framework-free module | Fast to ship; a stable `discoverEvents()`/`fetchEventCatalog()` contract can move behind an Azure Function later with no UI change. | +| 2026-07-17 | Support Vantor + Planet | Full parity with the reference; required allowlisting `data.source.coop`. | +| 2026-07-17 | Azure Maps for the map (not OpenLayers) | Reuse the app's existing `window.atlas` stack — no new dependency. | +| 2026-07-18 | Preview + clip via TiTiler | HASTE already runs TiTiler; streams any remote COG without an Azure Maps subscription. | +| 2026-07-19 | **Server-side clip** (layer-level `clipBbox` applied at imagery prep) chosen over client-side crop+upload | Instant add (no client crop/upload wait); the clip runs where imagery is already processed. The client-side TiTiler-crop approach was prototyped first (see [design.md](design.md#clip-approaches)). | +| 2026-07-19 | Route non-S3/Blob allowlisted hosts through the generic HTTP downloader | `data.source.coop` was validated but silently dropped by `ImageryDownloader`, failing Planet downloads. | diff --git a/spec/features/open-data-catalog/data-model.md b/spec/features/open-data-catalog/data-model.md new file mode 100644 index 0000000..5788a26 --- /dev/null +++ b/spec/features/open-data-catalog/data-model.md @@ -0,0 +1,144 @@ +# Data Model: Open Data Catalog Explorer + +> HASTE's local/dev stack stores image-layer metadata as JSON in Blob Storage +> (`METADATA_STORAGE_TYPE=blob`), not Cosmos DB. The "Cosmos" section below +> describes the logical `ImageLayer` document change; it applies equally to the +> blob-backed metadata store. + +## Image Layer document change + +### Modified schema + +| Store | Field | Before | After | Notes | +|---|---|---|---|---| +| ImageLayer metadata | `clipBbox` | (absent) | `Optional[list[float]]` = `null` | `[west, south, east, north]` EPSG:4326. Optional, default `null`; fully backward compatible. | + +```jsonc +// ImageLayer (relevant fields) +{ + "imageLayerId": "uuid", + "projectId": "uuid", + "preEventImageryUrls": ["https url", "..."], + "postEventImageryUrls": ["https url", "..."], + "sourceTypePreEvent": "maxar | planet_scope | planet_skysat | n/a | ...", + "sourceTypePostEvent": "…", + "clipBbox": [-66.859534, 10.4039, -66.816345, 10.424412] // NEW, or null +} +``` + +**Migration needed?** No. New optional field; existing documents read as +`clipBbox=null` (no clip). + +**RU / cost:** negligible — one small array field on an existing document. + +## Imagery URL allowlist + +Imagery URLs are validated (UI + backend) against a host allowlist. This +feature adds one host so Planet Open Data can be submitted: + +| Host pattern | Source type label | Added | +|---|---|---| +| `*.blob.core.windows.net` | `azureblobstorage` | existing | +| `*.amazonaws.com` | `awss3` | existing (Vantor lives here) | +| `data.source.coop` / `*.source.coop` | `sourcecoop` | **new** | + +Kept in sync between [`ui/src/util/validation.js`](../../../ui/src/util/validation.js) +and [`hastelib/.../core/utils/url_allowlist.py`](../../../hastelib/src/hastegeo/core/utils/url_allowlist.py). + +## Normalized scene (in-memory, UI only) + +Not persisted — produced by `openDataCatalog.js` from STAC, consumed by the +panel/map/list. + +```js +{ + uid, // `${source}:${id}:${index}` — unique key/feature id + id, source: "Vantor" | "Planet", + phase: "pre" | "post" | null, + cogUrl, thumbUrl, + bbox, geometry, // GeoJSON, EPSG:4326 + datetime, // ISO string + title, place, sensor, constellation, + gsd, cloud, offNadir, sunElev, cogSize, + sourceUrl, // browse/attribution link + sourceTypeKey, // HASTE dropdown key: maxar | planet_scope | planet_skysat +} +``` + +## Blob Storage changes + +No new containers. The imagery-prep workflow writes the same artifacts as +before (mosaic COG, processed RGB COG, valid-area-mask GeoJSON, building +footprints GPKG) under the existing per-layer path — but when `clipBbox` is set, +those artifacts cover only the clipped AOI. + +``` +{container=data}/ + {hash}/ + {task_id}/ + raw_imagery_post_event_mosaic_cog_{projectId}_{layerId}.tif # clipped when clipBbox set + processed_imagery_post_event_cog_{projectId}_{layerId}.tif + valid_area_mask_{projectId}_{layerId}.geojson + building_footprints_{projectId}_{layerId}.gpkg +``` + +## Queue Storage changes + +No new queues and no message-shape change. `ImageryPostProcessor` adds +`clip_bbox` to the YAML imagery-prep **config** (blob-staged) that the +`prepare-imagery` container reads: + +```yaml +project_id: "…" +image_layer_id: "…" +post_event_imagery_urls: ["https://data.source.coop/…/scene_visual.tif"] +source_type_post_event: "planet_scope" +clip_bbox: [-66.859534, 10.4039, -66.816345, 10.424412] # NEW (null when unset) +``` + +## Data Flow + +### Write path + +``` +UI → PutLayer (validate URLs + clipBbox) → ImageLayer metadata (Blob) + → local-image-queue (job msg) + hastefuncqueues → prepare-imagery (docker) → gdalwarp -te clip + → Blob (clipped COGs + artifacts) +``` + +### Read path + +``` +UI → GetLayerDetailView → ImageLayer metadata (Blob) +UI catalog → Vantor S3 STAC / Planet source.coop STAC (direct HTTPS) +UI catalog → TiTiler (/api/titiler) → remote COG tiles/crop (preview) +``` + +## Migration Plan + +### Forward + +1. Deploy backend (`clipBbox` on the model, `validate_clip_bbox`, downloader + route, mosaic clip) — additive. +2. Deploy UI (catalog panel). + +### Backward + +- Fully reversible. Reverting the backend leaves `clipBbox` ignored (unknown + field on the document is harmless); imagery is produced unclipped. No blob + cleanup required. + +## Data Volume Estimates + +| Entity | Size | Notes | +|---|---|---| +| `clipBbox` field | ~64 bytes | per ImageLayer | +| Clipped artifacts | **smaller** than unclipped | clip reduces mosaic/COG size | + +## Caching Strategy + +| Data | Cache Layer | TTL | Invalidation | +|---|---|---|---| +| STAC catalogs | Browser (per panel open) | session | Re-fetched on event change | +| TiTiler tiles | Browser tile cache | default | — | diff --git a/spec/features/open-data-catalog/design.md b/spec/features/open-data-catalog/design.md new file mode 100644 index 0000000..7245c40 --- /dev/null +++ b/spec/features/open-data-catalog/design.md @@ -0,0 +1,206 @@ +# Technical Design: Open Data Catalog Explorer + +## Overview + +A Fluent UI side panel on the Create Image Layer page browses open imagery +(Vantor/Maxar S3 STAC + Planet Source-Cooperative STAC), previews scenes on an +Azure Maps map via TiTiler, and adds scene COG URLs into the layer's pre/post +imagery. A drawn clip AOI is stored on the layer and applied server-side by the +imagery-prep workflow. The reference prototype is the standalone Open Disaster +Response Data Visualizer: + + +## Architecture + +### Component Diagram + +``` +┌───────────────────────────┐ direct HTTPS (CORS *) ┌──────────────────────────┐ +│ React UI │───────────────────────────▶│ Vantor S3 STAC │ +│ OpenDataCatalog panel │ │ Planet source.coop STAC │ +│ (Fluent UI + Azure Maps) │◀──── COG tiles/crop ───────│ TiTiler (api/titiler) │ +└─────────────┬─────────────┘ via /api/titiler proxy └──────────────────────────┘ + │ PutLayer (clipBbox + COG URLs) + ▼ + ┌────────────────────┐ queue msg ┌────────────────────┐ docker run ┌──────────────────┐ + │ hastefuncapi │──────────────▶│ hastefuncqueues │───────────────▶│ haste-imageryprep │ + │ PutLayer (validate)│ │ LocalRunner/Batch │ │ prepare_imagery │ + └────────────────────┘ └────────────────────┘ └────────┬─────────┘ + │ gdalwarp -te (clip) + ▼ Blob (clipped COGs) +``` + +### New Components + +| Component | Path | Responsibility | Technology | +|---|---|---|---| +| Catalog module | `ui/src/Components/OpenDataCatalog/openDataCatalog.js` | Framework-free discovery + STAC normalization + TiTiler URL builders + AOI geometry helpers | JS | +| Catalog panel | `ui/src/Components/OpenDataCatalog/OpenDataCatalogPanel.jsx` | Fluent ``: event picker, filters, list + map, clip toolbar | React / Fluent UI | +| Catalog map | `ui/src/Components/OpenDataCatalog/OpenDataCatalogMap.jsx` | Azure Maps footprints, TiTiler COG preview, rectangle clip drawing, loading indicator | React / Azure Maps | +| Scene row | `ui/src/Components/OpenDataCatalog/SceneListItem.jsx` | One scene: thumbnail, badges, expandable metadata, phase-scoped add buttons | React / Fluent UI | + +### Modified Components + +| Component | Path | Change Description | +|---|---|---| +| Image layer form | `ui/src/Components/CreateEditImageLayerForm.jsx` | "Browse Open Data Catalog" button + panel; `handleAddScene`, `handleClipAoiChange`; `clipBbox` in PutLayer body | +| Form helper | `ui/src/Components/CreateEditImageLayerHelper.js` | `addSceneToEventImagery()`; `clipBbox` in default state | +| Imagery allowlist (UI) | `ui/src/util/validation.js` | allow `data.source.coop` | +| Imagery allowlist (backend) | `hastelib/.../core/utils/url_allowlist.py` | allow `data.source.coop`; `validate_clip_bbox` | +| Downloader | `hastelib/.../core/utils/downloader.py` | route non-S3/Blob allowlisted hosts through generic HTTP | +| Mosaic | `hastelib/.../core/utils/imagery.py` | `mosaic_imagery(..., clip_bbox)` → gdalwarp `-te`/`-te_srs` | +| Imagery processor | `hastelib/.../core/processors/imagery.py` | `clip_bbox` in the prep config | +| Prep workflow | `hastelib/.../workflows/prepare_imagery.py` | thread `clip_bbox` through `ImageryWorkflow` → `_create_mosaic_cog` | +| Image layer model | `hastelib/.../core/models/projects.py` | `ImageLayer.clipBbox` | +| PutLayer | `api/hastefuncapi/function_app.py` | `validate_clip_bbox` | +| Dev proxy | `docker/nginx.conf` | `/api/titiler` timeouts (300s) + `client_max_body_size 64m` | + +## Event discovery & scene normalization + +Both catalogs are STAC. `openDataCatalog.js` exposes two contracts: + +- `discoverEvents()` → `{ events, errors }`. Crawls two root catalogs and + merges events present in both: + - **Vantor:** `https://vantor-opendata.s3.amazonaws.com/events/catalog.json` + → `child` links → each event `collection.json` (id, `odp:event_date`, + `item` links). + - **Planet:** `https://data.source.coop/planet/disasterdata/catalog.json` + → `child` links → each event `catalog.json` → `pre-event`/`post-event` + collections → items (or a pre-event `mosaic` asset). + - **Merge:** event ids are tokenized into place vs hazard words (dates/months + dropped); two events merge when their place tokens overlap and hazards + agree (e.g. Vantor `Venezuela-Earthquake-Jun-2026` ↔ Planet + `venezuela-earthquake-2026-06-24`). Each unified event carries + `sources: { vantor?, planet? }`. +- `fetchEventCatalog(event)` → `{ scenes, errors }`. Fetches whichever sources + the event has, resiliently (a failing source is dropped with a warning), and + dedupes + assigns a unique `uid`. + +Scene model — see [data-model.md](data-model.md#normalized-scene-in-memory-ui-only). +Vantor items carry an explicit `phase` property and a `visual` COG asset; +Planet phase comes from the `pre-event`/`post-event` collection id. + +## Map: preview, clip, indicators (Azure Maps + TiTiler) + +- **Footprints:** a `DataSource` + `PolygonLayer` (fill) + `LineLayer` (outline), + colored by source, highlight driven by an `_active` feature property. +- **COG preview:** selecting a scene adds an `atlas.layer.TileLayer` pointed at + `titilerTileUrl(cogUrl)` → `/cog/tiles/{z}/{x}/{y}.png?url=…`, inserted + beneath the footprint layers. Works with **no Azure Maps subscription** + (blank basemap) because the imagery itself is served by TiTiler. +- **Loading indicator:** the preview TileLayer is given id `odcPreview`; its GL + source id is resolved via a `findGlMap` duck-type, and a `sourcedata` / + `isSourceLoaded` listener clears a "Loading imagery…" chip once its tiles are + in (15s timeout fallback). +- **Clip drawing:** `atlas.drawing.DrawingManager` in `draw-rectangle` mode; + on completion the EPSG:4326 bbox is handed up and rendered as a persistent + dashed AOI rectangle. +- **Two-way selection:** clicking a footprint selects the scene (list scrolls to + it, highlights, expands metadata). During preview the fill is set fully + transparent (not hidden) so footprints stay clickable to switch scenes. + +TiTiler is reached from the browser through the api-proxy at `VITE_TITILER_URL` +(`…/api/titiler/`). Both source catalogs send `Access-Control-Allow-Origin: *`, +so browser fetches and tiles work directly. + +## Clip approaches + +Two were built; the **server-side** approach is the current direction. + +### Server-side clip (current) + +A single **layer-level** `clipBbox` (`[west, south, east, north]`, EPSG:4326) is +drawn in the catalog and stored on the `ImageLayer`. Because it applies to both +pre and post mosaics, the catalog filters scenes to those overlapping the AOI +(`bboxIntersects`) and badges scenes that fully contain it (`bboxContains`). + +Flow: draw AOI → `PutLayer` carries `clipBbox` → prep config `clip_bbox` → +`ImageryWorkflow` → `_create_mosaic_cog(..., clip_bbox)` → +`mosaic_imagery(..., clip_bbox)` appends `-te -te_srs EPSG:4326` +to the gdalwarp options. GDAL reprojects the EPSG:4326 bounds to the mosaic's +CRS, so no manual reprojection is needed. The AOI polygon + Overture footprints +are then derived from the already-clipped mosaic. + +### Client-side clip (prototype, sibling branch) + +Draw box → TiTiler `/cog/crop/{minx},{miny},{maxx},{maxy}/{w}x{h}.tif?url=…` +returns a clipped GeoTIFF → wrapped as a `File` → uploaded via the existing +chunked uploader → added as a normal blob URL. Instant clip visibility but a +download→reupload round-trip and a client-side wait. Superseded by the +server-side approach for the AOI-per-layer model. + +## API Design + +### `PUT /api/PutLayer` (modified) + +Adds one optional field on the `ImageLayer` request body: + +```json +{ + "clipBbox": "[number, number, number, number] | null — [west, south, east, north] EPSG:4326" +} +``` + +Validated by `validate_clip_bbox` (in `url_allowlist.py`): must be 4 finite +numbers, lon in [-180,180], lat in [-90,90], `west Optional[str]` | None if valid/absent, else user-facing error | +| `core/utils/imagery.py` | `ImageryUtils.mosaic_imagery` | `(tif_files, output_file_path, gdal_warp_params, clip_bbox=None)` | Appends `-te`/`-te_srs` when `clip_bbox` set | +| `core/utils/downloader.py` | `ImageryDownloader.download_imagery` | routes `sourcecoop`/other allowlisted hosts to `download_imagery_from_urls` | generic streamed HTTPS + size cap | +| `workflows/prepare_imagery.py` | `ImageryWorkflow.__init__` / `_create_mosaic_cog` | `clip_bbox=None` | threaded to `mosaic_imagery` | + +## Behavior & Logic + +### Core flow (add a scene) + +1. Analyst opens the catalog on the Create Image Layer page. +2. `discoverEvents()` lists events; selecting one runs `fetchEventCatalog()`. +3. Analyst previews scenes (TiTiler) and optionally draws a clip AOI. +4. Phase-scoped **＋ Pre/＋ Post** appends the COG URL (auto-fills source-type + + date); the AOI sets `clipBbox`. +5. Submit → `PutLayer` validates URLs + `clipBbox`, enqueues imagery prep. +6. `hastefuncqueues` spawns `haste-imageryprep`; the mosaic is clipped to the AOI. + +### Edge Cases + +| Case | Expected Behavior | +|---|---| +| One source catalog fails (CORS/network) | Other source's scenes still load; per-source warning banner | +| Duplicate STAC ids across a catalog | Deduped by `source + cogUrl`; unique `uid` for keys/features | +| Scene has no COG (footprint only) | Add buttons disabled; "COG not linked yet" note; no preview | +| Clip AOI doesn't overlap a scene | Scene hidden by the "only scenes in clip area" filter (default on) | +| No Azure Maps key (placeholder) | Blank basemap; footprints + TiTiler preview still work | +| Planet `data.source.coop` imagery | Downloaded via generic HTTP route (not S3/Blob SDK) | +| Large clip / slow TiTiler crop (client approach) | 4096px cap + client timeout; nginx 300s + friendly 504 msg | + +## Configuration + +| Config Key | Type | Default | Where Set | Description | +|---|---|---|---|---| +| `VITE_TITILER_URL` | url | `…/api/titiler/` | `docker-compose.yml` (ui) | Browser-reachable TiTiler base for preview/clip | +| `VITE_AZURE_MAPS_CLIENT_ID` | uuid | `placeholder` | `docker-compose.yml` (ui) | Azure Maps anonymous/AAD Client ID; placeholder → blank basemap (footprints + TiTiler preview still work) | +| `CLEANUP_CONTAINERS` | bool | `1` | `docker-compose.yml` (queues) | Debug: retain a failed spawned container's logs | +| `client_max_body_size` / `proxy_read_timeout` | nginx | `64m` / `300s` | `docker/nginx.conf` | Chunk uploads + titiler crop headroom (local dev) | + +## Observability + +- UI: per-source discovery/fetch error banners; preview loading chip. +- Backend: structured logging in `prepare_imagery` (`Clipping mosaic to AOI bbox …`); layer `statusMessage` surfaces failures. + +## Open Questions + +- [ ] Move discovery/normalization behind an Azure Function (`GetOpenDataCatalog`)? The module contract is designed for it. +- [ ] Polygon-exact (vs bbox) overlap filtering for oblique footprints? +- [ ] Hard backend guard for pre/post correctness (UI-only today)? diff --git a/spec/features/open-data-catalog/impact-analysis.md b/spec/features/open-data-catalog/impact-analysis.md new file mode 100644 index 0000000..3f9b6af --- /dev/null +++ b/spec/features/open-data-catalog/impact-analysis.md @@ -0,0 +1,74 @@ +# Impact Analysis: Open Data Catalog Explorer + +## Scope of Change + +| Component | Path | Type | Severity | +|---|---|---|---| +| Core library | `hastelib/.../core/models/projects.py` | modified (`clipBbox`) | low | +| Core library | `hastelib/.../core/utils/{url_allowlist,downloader,imagery}.py` | modified | medium | +| Core library | `hastelib/.../workflows/prepare_imagery.py`, `core/processors/imagery.py` | modified | medium | +| REST API | `api/hastefuncapi/function_app.py` (`PutLayer`) | modified | low | +| React UI | `ui/src/Components/OpenDataCatalog/*` (new) + `CreateEditImageLayer*` | new/modified | medium | +| React UI | `ui/src/util/validation.js` | modified | low | +| Docker config | `docker/nginx.conf`, `docker/docker-compose.yml` | modified | low (local dev) | + +## Azure Service Impact + +| Service | Change | Cost Impact | +|---|---|---| +| Blob Storage | Clipped artifacts (same paths, **smaller** when clipped) | neutral/negative | +| Queue Storage | Same queue; `clip_bbox` added to prep config | none | +| Azure Batch / imageryprep | Same image; extra `gdalwarp -te` (clip usually reduces work) | neutral | +| Static Web Apps | New UI + external fetches (Vantor/Planet/TiTiler) | none | + +## Dependency Analysis + +### Upstream + +| Dependency | Type | Risk if Unavailable | +|---|---|---| +| Vantor S3 STAC (`vantor-opendata.s3.amazonaws.com`) | external | that source's events don't list; other source still works | +| Planet STAC (`data.source.coop`) | external | Planet events don't list/download; Vantor still works | +| TiTiler (`api/titiler`) | infra | no imagery preview/crop; footprints + add still work | +| Azure Maps | infra | with a key: satellite basemap; without: blank basemap (feature still works) | + +### Downstream + +| Consumer | How Affected | Breaking? | Migration? | +|---|---|---|---| +| `PutLayer` callers | new optional `clipBbox` field | no | no | +| Existing ImageLayer docs | unknown field read as `null` | no | no | +| imagery-prep workflow | clips when `clip_bbox` present | no | no | + +## Risk Assessment + +| Risk | Likelihood | Impact | Mitigation | +|---|---|---|---| +| External catalog schema/URL drift | medium | medium | Per-source resilient fetch; discovery isolated in one module (swap-to-backend seam) | +| `data.source.coop` on the imagery allowlist → server-side fetch (SSRF surface) | low | low | Public, well-known open-data host; explicit host match; redirects not followed; size-capped | +| Clip bounds outside imagery / degenerate box | low | low | `validate_clip_bbox` (ranges + `w Status reflects work already implemented on the `feat-data-catalog*` branches. + +## Phases + +### Phase 1: Core Library — done + +**Goal:** allowlist + clip plumbing in `hastelib`. + +| Task | Agent | Story | Status | +|---|---|---|---| +| Allow `data.source.coop` in `url_allowlist` + `validate_clip_bbox` (+ tests) | `backend-dev`/`gis` | US-004 | done | +| `ImageLayer.clipBbox` model field | `backend-dev` | US-004 | done | +| `mosaic_imagery(..., clip_bbox)` → gdalwarp `-te`/`-te_srs` | `gis` | US-004 | done | +| Route non-S3/Blob allowlisted hosts through generic HTTP downloader | `gis` | US-004 | done | + +**Exit Criteria:** [x] unit tests pass; [x] clip applies independently of the API. + +### Phase 2: API Layer — done + +**Goal:** carry + validate `clipBbox`; thread into imagery prep. + +| Task | Agent | Story | Status | +|---|---|---|---| +| `PutLayer` validates + persists `clipBbox` | `backend-dev` | US-004 | done | +| `clip_bbox` into imagery-prep config; `ImageryWorkflow` → `_create_mosaic_cog` | `backend-dev`/`gis` | US-004 | done | + +**Exit Criteria:** [x] end-to-end prep run clips to the AOI in Docker Compose. + +### Phase 3: UI — done + +**Goal:** the catalog experience. + +| Task | Agent | Story | Status | +|---|---|---|---| +| `openDataCatalog.js` discovery/normalization + TiTiler/AOI helpers | `ui` | US-001 | done | +| Panel + Map + SceneListItem (footprints, preview, filters) | `ui` | US-001/002 | done | +| Add scene → pre/post with auto-fill; phase-scoped buttons | `ui` | US-003 | done | +| Clip drawing + layer `clipBbox`; AOI-overlap filter + "covers AOI" badge | `ui` | US-004/005 | done | +| Preview loading indicator; two-way map↔list selection | `ui` | US-002/006 | done | + +**Exit Criteria:** [x] feature usable in the SWA/Docker local stack. + +### Phase 4: Integration & Deployment — partial + +| Task | Agent | Status | +|---|---|---| +| Local nginx proxy: titiler timeouts + `client_max_body_size` | `backend-dev` | done | +| E2E validation in Docker Compose | `backend-dev` | done | +| Deploy to dev1/testing/prod SWA + Function Apps | `backend-dev` | not-started | +| Docs / CHANGELOG | `backend-dev` | not-started | + +## Milestones + +| Milestone | Status | Deliverable | +|---|---|---| +| Core library done | done | allowlist + clip in `hastelib` | +| API layer done | done | `PutLayer` + prep config | +| UI done | done | catalog visible in the app | +| Release | not-started | deployed to production SWA | + +## Agent Summary + +| Agent | Phases | +|---|---| +| `gis` | 1, 2 | +| `backend-dev` | 1, 2, 4 | +| `ui` | 3 | + +## Resource Requirements + +- **Azure services:** none new (reuses TiTiler, Blob, queues, imageryprep image). +- **External data:** Vantor Open Data (S3), Planet Open Data (Source Cooperative). Both public, CC BY-NC 4.0. + +## Open Questions + +- [ ] Production rollout + any provider-catalog change monitoring. diff --git a/spec/features/open-data-catalog/rollout.md b/spec/features/open-data-catalog/rollout.md new file mode 100644 index 0000000..7eb14dc --- /dev/null +++ b/spec/features/open-data-catalog/rollout.md @@ -0,0 +1,68 @@ +# Rollout Plan: Open Data Catalog Explorer + +## Rollout Strategy + +**Type:** phased (dev1 → testing → production) +**Target date:** TBD + +> Verified in the local Docker Compose stack; not yet deployed to Azure. + +## Deployment Targets + +| Component | Deployment Method | Target | +|---|---|---| +| `hastelib` | Docker rebuild (imageryprep + Function Apps) | all Function Apps + imageryprep image | +| `hastefuncapi` | `deploy-apps.yml` | Azure Functions | +| `hastefuncqueues` | `deploy-apps.yml` | Azure Functions | +| React UI | `deploy-apps.yml` | Azure Static Web Apps | + +## Feature Flags + +| Flag | Location | Default | Notes | +|---|---|---|---| +| (none) | — | — | No flag; the "Browse Open Data Catalog" button is additive and low-risk. A flag could gate it if desired. | + +## Prerequisites (production) + +| Item | Requirement | +|---|---| +| Imagery allowlist | `data.source.coop` allowed by the deployed API/downloader (ships with this change) | +| TiTiler public endpoint | `VITE_TITILER_URL` reachable from the browser (already used by the labeling tool) | +| Azure Maps | Client-ID/AAD auth configured (satellite basemap). Blank basemap works without it. | +| Ingress/APIM | Body-size + timeout headroom for chunk uploads / any TiTiler crop (the local `nginx.conf` changes are dev-only and must be mirrored on the real gateway if needed) | + +## Rollout Phases + +### Phase 1: Dev1 — TBD +- Merge to `main` → `deploy-apps.yml`. +- Verify: catalog lists events; add scene; draw AOI; process a layer and confirm clipped output; Planet download succeeds. + +### Phase 2: Testing — TBD +- E2E scenarios (see [test-plan.md](test-plan.md)) pass; no ImageLayer metadata regressions. + +### Phase 3: Production — TBD +- Health checks green; error rate stable; imagery-prep clip jobs succeed. + +## Rollback Plan + +| Step | Action | ETA | +|---|---|---| +| 1 | Revert PR / deploy previous commit | <15 min | +| 2 | Confirm ImageLayers process unclipped (unknown `clipBbox` ignored) | — | + +**Cosmos/metadata rollback required?** No — `clipBbox` optional/additive. +**Blob cleanup?** No. + +## Monitoring + +| Metric | Source | Watch | +|---|---|---| +| PutLayer 400 rate | Functions metrics | spikes → bad `clipBbox`/URLs | +| imagery-prep failures | layer `statusMessage` / queue logs | download or clip errors | +| External catalog reachability | UI per-source warning banners | Vantor/Planet outages | + +## Post-Rollout Checklist + +- [ ] `docs/` updated (catalog usage). +- [ ] `CHANGELOG.md` updated. +- [ ] Production ingress body-size/timeout mirrored if clip-upload path is used. diff --git a/spec/features/open-data-catalog/test-plan.md b/spec/features/open-data-catalog/test-plan.md new file mode 100644 index 0000000..f7cfb6a --- /dev/null +++ b/spec/features/open-data-catalog/test-plan.md @@ -0,0 +1,86 @@ +# Test Plan: Open Data Catalog Explorer + +## Test Strategy + +| Level | Scope | Tool/Framework | Coverage Target | +|---|---|---|---| +| Unit | allowlist + clip-bbox validation | pytest/unittest (`hastelib/tests/`) | validators covered | +| Manual/integration | discovery, preview, clip end-to-end | Docker Compose local stack | key flows | +| E2E | imagery-prep clip job | real `haste-imageryprep` run | 1 verified run | + +## Test Scenarios + +### Unit Tests (`hastelib/tests/`) + +| ID | Module | Scenario | Expected | Story | +|---|---|---|---|---| +| UT-001 | `url_allowlist` | `data.source.coop` imagery URL | returns `"sourcecoop"` | US-001/004 | +| UT-002 | `url_allowlist` | `validate_clip_bbox` valid `[w,s,e,n]` | `None` | US-004 | +| UT-003 | `url_allowlist` | wrong length / non-numeric / out-of-range / inverted bbox | error string | US-004 | + +> Implemented in `hastelib/tests/core/utils/test_url_allowlist.py` (32 tests passing). + +### API Integration Tests + +| ID | Endpoint | Scenario | Expected | Story | +|---|---|---|---|---| +| IT-001 | `PUT /api/PutLayer` | Planet post URL + valid `clipBbox` | 200; layer persisted with `clipBbox` | US-003/004 | +| IT-002 | `PUT /api/PutLayer` | invalid `clipBbox` (e.g. `[1,2,3]`) | 400 with message | US-004 | +| IT-003 | `PUT /api/PutLayer` | non-allowlisted imagery host | 400 | US-003 | + +### Queue / Workflow Tests + +| ID | Scenario | Expected | Story | +|---|---|---|---| +| QT-001 | Layer with `clipBbox` processed by `haste-imageryprep` | mosaic clipped to AOI; `Clipping mosaic to AOI bbox …` logged | US-004 | +| QT-002 | Planet (`source.coop`) post URL | downloaded via generic HTTP route (not S3/Blob SDK) | US-004 | + +### UI Component Scenarios + +| ID | Component | User Action | Expected | Story | +|---|---|---|---|---| +| UI-001 | Panel | open catalog | events discovered + merged | US-001 | +| UI-002 | Map | select scene | TiTiler preview + loading chip until tiles in | US-002 | +| UI-003 | SceneListItem | view a `post` scene | only ＋ Post-event button shown | US-003 | +| UI-004 | Map | draw clip box | persistent AOI rectangle; `clipBbox` set | US-004 | +| UI-005 | Panel | with AOI set | only overlapping scenes shown; "covers AOI" badges | US-005 | +| UI-006 | Map→list | click a footprint | list scrolls to + highlights + expands the row | US-006 | + +### End-to-End (Docker Compose) — executed + +| ID | Flow | Result | Story | +|---|---|---|---| +| E2E-001 | PutLayer (Planet Caracas scene + `clipBbox`) → imagery prep → inspect output | **Passed** — output mosaic WGS84 bounds match the drawn box (lon to 5 dp; ~30 m lat delta from UTM reprojection); AOI + 5,374 footprints derived from the clipped area | US-004 | + +### Edge Case & Negative Tests + +| ID | Scenario | Expected | +|---|---|---| +| NEG-001 | One source catalog down | other source loads; warning banner | +| NEG-002 | Duplicate STAC ids | deduped; unique `uid` keys (no React duplicate-key warning) | +| EDGE-001 | Scene without a COG | add disabled; footprint-only note | +| EDGE-002 | Clip AOI not overlapping a scene | scene hidden by default AOI filter | + +## Coverage Matrix + +| Story | Unit | API | Queue/WF | UI | E2E | +|---|---|---|---|---|---| +| US-001 | UT-001 | — | — | UI-001 | — | +| US-002 | — | — | — | UI-002 | — | +| US-003 | — | IT-001/003 | — | UI-003 | — | +| US-004 | UT-002/003 | IT-001/002 | QT-001/002 | UI-004 | E2E-001 | +| US-005 | — | — | — | UI-005 | — | +| US-006 | — | — | — | UI-006 | — | + +## Environment Requirements + +| Environment | Purpose | Config | +|---|---|---| +| Local (Docker Compose) | dev + E2E | `docker/docker-compose.yml` (Azurite, TiTiler, imageryprep) | + +## Sign-off Criteria + +- [x] Allowlist + clip-bbox unit tests pass (32). +- [x] Clip verified end-to-end against a real imagery-prep run. +- [x] `docker-compose` stack runs the catalog + prep clean. +- [ ] Automated UI/API integration tests (currently manual). diff --git a/spec/features/open-data-catalog/user-stories.md b/spec/features/open-data-catalog/user-stories.md new file mode 100644 index 0000000..7351395 --- /dev/null +++ b/spec/features/open-data-catalog/user-stories.md @@ -0,0 +1,180 @@ +# User Stories: Open Data Catalog Explorer + +## Personas + +| Persona | Description | Key Goals | +|---|---|---| +| Disaster Analyst | Domain expert who assembles imagery and produces damage maps | Find the right open imagery for an event fast; get a clean AOI | + +--- + +## Stories + +### US-001: Browse open disaster imagery in-app + +**As a** Disaster Analyst, +**I want to** browse open imagery for a disaster event from within the Create Image Layer page, +**So that** I don't have to hunt for COG URLs on external S3/STAC catalogs. + +**Priority:** P1 · **Component(s):** `ui/src/Components/OpenDataCatalog/` + +```gherkin +Given I am on the Create Image Layer page +When I click "Browse Open Data Catalog" +Then I see a panel listing every disaster event discovered across Vantor and Planet +And events present in both sources are shown once, tagged "Vantor + Planet" +``` + +```gherkin +Given one source catalog is unreachable +When the catalog loads +Then the other source's scenes still appear +And a per-source warning banner explains what failed +``` + +--- + +### US-002: Preview a scene's imagery on the map + +**As a** Disaster Analyst, +**I want to** preview a scene's actual imagery on the map, +**So that** I can judge coverage/quality before adding it. + +**Priority:** P1 · **Component(s):** `OpenDataCatalog/OpenDataCatalogMap.jsx` + +```gherkin +Given a scene with a COG is listed +When I select it +Then its imagery streams onto the map via TiTiler and the view flies to it +And a "Loading imagery…" indicator shows until its tiles have rendered +``` + +```gherkin +Given Azure Maps is in placeholder mode (no real Client ID configured) +When I preview a scene +Then the imagery still renders (on a blank basemap) +``` + +--- + +### US-003: Add a scene to the correct phase + +**As a** Disaster Analyst, +**I want to** add a scene to pre- or post-event imagery in one click, +**So that** the layer is populated correctly without copy/paste. + +**Priority:** P0 · **Component(s):** `OpenDataCatalog/SceneListItem.jsx`, `CreateEditImageLayerHelper.js` + +```gherkin +Given a scene whose phase is "post" +When I look at its actions +Then only "＋ Post-event" is offered (not "＋ Pre-event") +When I click it +Then the COG URL is appended to post-event imagery and source-type + capture date are auto-filled when empty +``` + +```gherkin +Given a scene already added to post-event +When I view it +Then its button shows "Added to Post" and is disabled +``` + +--- + +### US-004: Clip imagery to a drawn AOI + +**As a** Disaster Analyst, +**I want to** draw an area on the map and have the produced imagery clipped to it, +**So that** the layer covers only the area I care about. + +**Priority:** P1 · **Component(s):** `OpenDataCatalogMap.jsx`, `hastelib` imagery prep + +```gherkin +Given I have selected a scene +When I click "Set clip area" and drag a box +Then a persistent AOI rectangle is drawn and stored on the layer (clipBbox) +When the image layer is processed +Then the pre/post mosaics are clipped to that AOI (gdalwarp -te) and the derived AOI/footprints cover only the clipped area +``` + +```gherkin +Given a clipBbox with west>=east or out-of-range coordinates +When I submit the layer +Then PutLayer returns 400 with a clear message +``` + +--- + +### US-005: Keep pre/post on the same AOI + +**As a** Disaster Analyst, +**I want to** see only scenes that cover my clip area when picking pre and post, +**So that** both phases share the same AOI and nothing clips away to nothing. + +**Priority:** P2 · **Component(s):** `OpenDataCatalogPanel.jsx` + +```gherkin +Given I have drawn a clip AOI +When I browse scenes +Then by default only scenes whose footprint overlaps the AOI are shown +And scenes whose footprint fully contains the AOI get a "covers AOI" badge +And I can toggle the filter off to see all scenes +``` + +--- + +### US-006: Select from the map and see it in the list + +**As a** Disaster Analyst, +**I want to** click a footprint on the map and have the list jump to that scene, +**So that** I can browse spatially and still see the scene's details/actions. + +**Priority:** P2 · **Component(s):** `OpenDataCatalogMap.jsx`, `OpenDataCatalogPanel.jsx`, `SceneListItem.jsx` + +```gherkin +Given several footprints are on the map +When I click one (including while another scene is previewing) +Then that scene is selected: the list scrolls to it, highlights it, and expands it with extra metadata +``` + +--- + +## Agent Assignment Map + +### Available Agents + +| Agent | Scope | Touches Code? | +|---|---|---| +| `gis` | Satellite imagery, GDAL/rasterio, provider adapters | Yes | +| `backend-dev` | Python backend, API, processors, data layers | Yes | +| `ui` | React/FluentUI/Azure Maps, frontend | Yes | +| `backend-validation` | Validates backend against specs/tests | No | +| `ui-validation` | Validates frontend behavior | No | + +### Story → Agent Mapping + +| Story | Implementing Agent(s) | Validating Agent(s) | Notes | +|---|---|---|---| +| US-001 | `ui` | `ui-validation` | STAC discovery in a UI module | +| US-002 | `ui`, `gis` | `ui-validation` | TiTiler preview | +| US-003 | `ui` | `ui-validation` | phase-scoped add | +| US-004 | `gis`, `backend-dev` | `backend-validation` | mosaic clip + PutLayer | +| US-005 | `ui` | `ui-validation` | AOI overlap filter | +| US-006 | `ui` | `ui-validation` | two-way selection | + +## Story Map + +| Priority | Story | Phase | Implementing Agent | Component | +|---|---|---|---|---| +| P0 | US-003 | Phase 3 — UI | `ui` | `ui/src/Components/OpenDataCatalog/` | +| P1 | US-001 | Phase 3 — UI | `ui` | `ui/src/Components/OpenDataCatalog/` | +| P1 | US-002 | Phase 3 — UI | `ui`/`gis` | `OpenDataCatalogMap.jsx` | +| P1 | US-004 | Phase 1/2 — Core/API | `gis`/`backend-dev` | `hastelib`, `hastefuncapi` | +| P2 | US-005 | Phase 3 — UI | `ui` | `OpenDataCatalogPanel.jsx` | +| P2 | US-006 | Phase 3 — UI | `ui` | `OpenDataCatalog/` | + +## Out of Scope + +- [ ] Live COG streaming preview with per-tile progress (reference used OpenLayers + geotiff); TiTiler tile preview is used instead. +- [ ] Hard backend enforcement of pre/post correctness (UI-guarded only). +- [ ] Per-scene (rather than layer-level) clip AOIs. diff --git a/ui/src/Components/CreateEditImageLayerForm.jsx b/ui/src/Components/CreateEditImageLayerForm.jsx index 42c924b..1f5a90e 100644 --- a/ui/src/Components/CreateEditImageLayerForm.jsx +++ b/ui/src/Components/CreateEditImageLayerForm.jsx @@ -4,6 +4,7 @@ import { useState, useContext, useEffect } from "react"; import { TextField, PrimaryButton, + DefaultButton, Text, DatePicker, Dropdown, @@ -13,11 +14,13 @@ import { useParams } from "react-router-dom"; import SectionHeader from "./Section/SectionHeader"; import CreateEditImageLayerFormImagerySources from "./CreateEditImageLayerFormImagerySources"; import CreateEditImageLayerFormBuildingFootprints from "./CreateEditImageLayerFormBuildingFootprints"; +import OpenDataCatalogPanel from "./OpenDataCatalog/OpenDataCatalogPanel"; import { createComponentDefaultState, onFormChange, getUrlList, + addSceneToEventImagery, } from "./CreateEditImageLayerHelper"; import { apiPut } from "../util/api"; @@ -39,6 +42,21 @@ const CreateEditImageLayerModal = () => { const projectId = useParams().projectId; const imageLayerId = useParams().imageLayerId; const [isUploading, setIsUploading] = useState(false); + const [isCatalogOpen, setIsCatalogOpen] = useState(false); + + // Add a scene picked from the Open Data Catalog explorer into the pre/post + // imagery array (with source-type + capture-date auto-fill). Returns the + // helper's { ok, error } result so the panel can surface a message inline. + function handleAddScene(scene, field) { + return addSceneToEventImagery(setComponentState, componentState, scene, field); + } + + // Server-side clip AOI: the Open Data Catalog draws a box and sets a single + // layer-level clip bbox ([w, s, e, n] EPSG:4326). Imagery prep clips the + // pre/post mosaics to it — no client-side crop/upload wait. + function handleClipAoiChange(bbox) { + onFormChange(bbox, "clipBbox", setComponentState, componentState); + } useEffect(() => { async function initComponent() { @@ -209,6 +227,7 @@ const CreateEditImageLayerModal = () => { imageryCaptureDatePreEvent: imageryCaptureDatePreEvent, imageryCaptureDatePostEvent: imageryCaptureDatePostEvent, userBuildingFootprintsUrl: userBuildingFootprintsUrl, + clipBbox: componentState.clipBbox || null, userId: appParams.userId, }; @@ -316,6 +335,19 @@ const CreateEditImageLayerModal = () => { You can combine files from both a URL and a local directory if needed. All files must be valid GeoTIFF (.tif) files. + {!imageLayerId && ( +
+ setIsCatalogOpen(true)} + /> + + Explore Vantor/Maxar and Planet open imagery for a disaster + and add scenes straight into the sections below. + +
+ )} @@ -511,6 +543,18 @@ const CreateEditImageLayerModal = () => { + + {!imageLayerId && ( + setIsCatalogOpen(false)} + onAddScene={handleAddScene} + clipAoi={componentState.clipBbox} + onClipAoiChange={handleClipAoiChange} + preUrls={getUrlList(componentState.preEventImageryUrls)} + postUrls={getUrlList(componentState.postEventImageryUrls)} + /> + )} ); }; diff --git a/ui/src/Components/CreateEditImageLayerHelper.js b/ui/src/Components/CreateEditImageLayerHelper.js index 867c192..d182d04 100644 --- a/ui/src/Components/CreateEditImageLayerHelper.js +++ b/ui/src/Components/CreateEditImageLayerHelper.js @@ -73,6 +73,8 @@ export async function createComponentDefaultState(imageLayerId, projectId) { : [], currentUserBuildingFootprintsControl: imageryOriginOptions[0].key, currentUserBuildingFootprintsControlError: "", + // Server-side clip AOI [w, s, e, n] EPSG:4326 (Open Data Catalog). + clipBbox: imageLayerToEdit.clipBbox || null, } : { imageLayerId: "", @@ -109,6 +111,8 @@ export async function createComponentDefaultState(imageLayerId, projectId) { userBuildingFootprintsUrls: [], currentUserBuildingFootprintsControl: imageryOriginOptions[0].key, currentUserBuildingFootprintsControlError: "", + // Server-side clip AOI [w, s, e, n] EPSG:4326 (Open Data Catalog). + clipBbox: null, } return tempState; @@ -223,6 +227,65 @@ export const addUrlToFootprintArray = (setComponentState, componentState, URL, f }; +// Add a scene picked from the Open Data Catalog explorer to a pre/post +// imagery array. Appends the COG URL AND (v1 auto-fill) sets the matching +// source-type dropdown and imagery capture date — but only when those fields +// are still empty/default, so a user's own entries are never clobbered. +// +// `field` is "preEventImageryUrls" | "postEventImageryUrls"; the sibling +// source-type / capture-date field names are derived from it. Returns a +// { ok, error } result so the caller can surface a message inline. +export const addSceneToEventImagery = (setComponentState, componentState, scene, field) => { + const url = (scene?.cogUrl || "").trim(); + if (!url) { + return { ok: false, error: "This scene has no downloadable COG yet." }; + } + + const urlIsValid = validateURL(url); + if (!urlIsValid[0]) { + return { ok: false, error: urlIsValid[1] }; + } + + const hostIsValid = validateImageryUrlHost(url); + if (!hostIsValid[0]) { + return { ok: false, error: hostIsValid[1] }; + } + + if (componentState[field].some((item) => item.value === url)) { + return { ok: false, error: "This scene is already added." }; + } + + const isPre = field === "preEventImageryUrls"; + const sourceTypeField = isPre ? "sourceTypePreEvent" : "sourceTypePostEvent"; + const captureDateField = isPre + ? "imageryCaptureDatePreEvent" + : "imageryCaptureDatePostEvent"; + + const patch = { + ...componentState, + [field]: [ + ...componentState[field], + { id: uuidv4(), type: "url", value: url, name: scene.title || "" }, + ], + }; + + // Only auto-fill source type when still at the "Unknown" default. + if (scene.sourceTypeKey && (componentState[sourceTypeField] === "n/a" || !componentState[sourceTypeField])) { + patch[sourceTypeField] = scene.sourceTypeKey; + } + + // Only auto-fill capture date when the user hasn't set one. + if (scene.datetime && !componentState[captureDateField]) { + const parsed = new Date(scene.datetime); + if (!isNaN(parsed.getTime())) { + patch[captureDateField] = parsed; + } + } + + setComponentState(patch); + return { ok: true, error: "" }; +}; + export const addFileToEventImageryArray = (files, acceptedFileTypes, componentState, setComponentState, field, errorField) => { var invalidFiles = []; diff --git a/ui/src/Components/OpenDataCatalog/OpenDataCatalogMap.jsx b/ui/src/Components/OpenDataCatalog/OpenDataCatalogMap.jsx new file mode 100644 index 0000000..f43f38c --- /dev/null +++ b/ui/src/Components/OpenDataCatalog/OpenDataCatalogMap.jsx @@ -0,0 +1,407 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// Azure Maps footprint view for the Open Data Catalog explorer. Reuses the +// app's global `window.atlas` map stack (loaded in index.html) — no new +// dependency. Footprints are colored by source and stay in sync with the +// list's hover/selection state. +import { useEffect, useRef, useCallback, useState } from "react"; +import PropTypes from "prop-types"; +import { Spinner, SpinnerSize } from "@fluentui/react"; + +import { + getAzureMapsAuthOptions, + isAzureMapsPlaceholder, +} from "../../util/azureMapsAuth"; +import { SOURCE_COLORS, titilerTileUrl } from "./openDataCatalog"; + +// atlas.Map hides the underlying Mapbox-GL fork; reach it (for getLayer().source) +// via this duck-typed scan. Mirrors findGlMap in InteractiveLabeler.jsx. +// Normal footprint fill opacity (active scene emphasized). Kept as a constant +// so the preview effect can restore it after transparently disabling the fill. +const FILL_OPACITY_EXPR = ["case", ["get", "_active"], 0.35, 0.12]; + +function findGlMap(atlasMap) { + if (!atlasMap) return null; + const direct = [atlasMap.map, atlasMap._map, atlasMap.gl, atlasMap._gl]; + for (const c of direct) { + if (c && typeof c.getLayer === "function") return c; + } + for (const k of Object.keys(atlasMap)) { + const v = atlasMap[k]; + if (v && typeof v === "object" && typeof v.getLayer === "function") return v; + } + return null; +} + +// Build a GeoJSON FeatureCollection from normalized scenes, tagging each +// feature with the id + a per-source color + an "active" flag the layer +// style expressions read (so hover/select highlight is data-driven). +function scenesToFeatureCollection(scenes, activeId) { + const features = []; + for (const scene of scenes) { + if (!scene.geometry) continue; + const uid = scene.uid || scene.id; + features.push({ + type: "Feature", + geometry: scene.geometry, + properties: { + sceneId: uid, + _color: SOURCE_COLORS[scene.source] || "#616161", + _active: uid === activeId, + }, + }); + } + return { type: "FeatureCollection", features }; +} + +const OpenDataCatalogMap = ({ + scenes, + activeId, + previewScene, + clipMode, + clipAoi, + onHover, + onSelect, + onClipDrawn, + center, +}) => { + const containerRef = useRef(null); + const mapRef = useRef(null); + const dataSourceRef = useRef(null); + const previewLayerRef = useRef(null); + const fillLayerRef = useRef(null); + const clipSourceRef = useRef(null); + const drawingRef = useRef(null); + // Preview tile-loading indicator: the GL source id of the current COG + // preview layer, and whether its tiles are still loading. + const previewSourceIdRef = useRef(null); + const [tilesLoading, setTilesLoading] = useState(false); + const readyRef = useRef(false); + + // Latest props addressable from the (once-bound) map events / camera fit + // without rebinding. Updated in an effect (not during render) so the + // rules-of-hooks ref guidance is respected. + const scenesRef = useRef(scenes); + const activeIdRef = useRef(activeId); + const clipAoiRef = useRef(clipAoi); + const handlersRef = useRef({ onHover, onSelect, onClipDrawn }); + useEffect(() => { + scenesRef.current = scenes; + activeIdRef.current = activeId; + clipAoiRef.current = clipAoi; + handlersRef.current = { onHover, onSelect, onClipDrawn }; + }); + + // Draw (or clear) the persistent clip-AOI rectangle from a [w,s,e,n] bbox. + const renderClipAoi = useCallback((bbox) => { + const ds = clipSourceRef.current; + if (!ds) return; + ds.clear(); + if (bbox && bbox.length === 4) { + const [w, s, e, n] = bbox; + ds.add({ + type: "Feature", + geometry: { + type: "Polygon", + coordinates: [[[w, s], [e, s], [e, n], [w, n], [w, s]]], + }, + }); + } + }, []); + + // Push scene data into the datasource and fit the camera to the footprints. + const syncData = useCallback(() => { + const ds = dataSourceRef.current; + if (!ds || !readyRef.current) return; + const fc = scenesToFeatureCollection(scenesRef.current, activeIdRef.current); + ds.clear(); + ds.add(fc); + if (fc.features.length > 0 && mapRef.current) { + const bounds = window.atlas.data.BoundingBox.fromData(fc); + mapRef.current.setCamera({ + bounds, + padding: 40, + type: "fly", + duration: 500, + }); + } + }, []); + + // Create the map once. + useEffect(() => { + if (!window.atlas || !containerRef.current) return undefined; + + const map = new window.atlas.Map(containerRef.current, { + center: center || [0, 0], + zoom: center ? 6 : 2, + maxPitch: 0, + pitch: 0, + style: isAzureMapsPlaceholder ? "blank" : "satellite", + language: "en-US", + authOptions: getAzureMapsAuthOptions(), + }); + mapRef.current = map; + + map.events.add("ready", () => { + map.setUserInteraction({ + dragRotateInteraction: false, + scrollZoomInteraction: true, + pinchRotateInteraction: false, + }); + map.controls.add(new window.atlas.control.ZoomControl(), { + position: "bottom-left", + }); + + const dataSource = new window.atlas.source.DataSource(); + map.sources.add(dataSource); + dataSourceRef.current = dataSource; + + const fillLayer = new window.atlas.layer.PolygonLayer(dataSource, "odcFill", { + fillColor: ["get", "_color"], + fillOpacity: FILL_OPACITY_EXPR, + }); + fillLayerRef.current = fillLayer; + const lineLayer = new window.atlas.layer.LineLayer(dataSource, "odcLine", { + strokeColor: ["case", ["get", "_active"], "#ffffff", ["get", "_color"]], + strokeWidth: ["case", ["get", "_active"], 3, 1.7], + }); + map.layers.add([fillLayer, lineLayer]); + + // Persistent clip-AOI rectangle (drawn above footprints/preview). + const clipSource = new window.atlas.source.DataSource(); + map.sources.add(clipSource); + clipSourceRef.current = clipSource; + map.layers.add( + new window.atlas.layer.LineLayer(clipSource, "odcClip", { + strokeColor: "#d83b01", + strokeWidth: 3, + strokeDashArray: [3, 2], + }) + ); + + map.events.add("mousemove", fillLayer, (e) => { + const shapeProps = e.shapes?.[0]?.getProperties?.(); + const id = shapeProps?.sceneId; + map.getCanvasContainer().style.cursor = id ? "pointer" : ""; + if (id) handlersRef.current.onHover(id); + }); + map.events.add("mouseleave", fillLayer, () => { + map.getCanvasContainer().style.cursor = ""; + handlersRef.current.onHover(null); + }); + map.events.add("click", fillLayer, (e) => { + const shapeProps = e.shapes?.[0]?.getProperties?.(); + const id = shapeProps?.sceneId; + if (!id) return; + const scene = (scenesRef.current || []).find( + (s) => (s.uid || s.id) === id + ); + if (scene) handlersRef.current.onSelect(scene); + }); + + // Clear the preview loading indicator once the COG preview source has + // finished loading its tiles for the current view. + map.events.add("sourcedata", (e) => { + if ( + e && + e.isSourceLoaded && + e.sourceId && + e.sourceId === previewSourceIdRef.current + ) { + setTilesLoading(false); + } + }); + + readyRef.current = true; + syncData(); + renderClipAoi(clipAoiRef.current); + }); + + return () => { + readyRef.current = false; + dataSourceRef.current = null; + if (drawingRef.current) { + drawingRef.current.dispose?.(); + drawingRef.current = null; + } + if (mapRef.current) { + mapRef.current.dispose(); + mapRef.current = null; + } + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // Rectangle clip-box drawing. When clipMode turns on we put the drawing + // manager into draw-rectangle mode; on completion we hand the box's + // EPSG:4326 bbox ([w,s,e,n]) back to the panel and idle the tool. + useEffect(() => { + const map = mapRef.current; + if (!map || !readyRef.current || !window.atlas?.drawing) return; + + if (clipMode) { + if (!drawingRef.current) { + const dm = new window.atlas.drawing.DrawingManager(map, {}); + map.events.add("drawingcomplete", dm, (shape) => { + try { + const geo = + typeof shape.toJson === "function" ? shape.toJson() : shape; + const b = window.atlas.data.BoundingBox.fromData(geo); + handlersRef.current.onClipDrawn?.([b[0], b[1], b[2], b[3]]); + } catch (err) { + console.warn("Failed to read clip bbox:", err); + } + dm.setOptions({ mode: "idle" }); + }); + drawingRef.current = dm; + } + drawingRef.current.getSource().clear(); + drawingRef.current.setOptions({ mode: "draw-rectangle" }); + } else if (drawingRef.current) { + drawingRef.current.setOptions({ mode: "idle" }); + drawingRef.current.getSource().clear(); + } + }, [clipMode]); + + // Re-sync features (and refit) whenever the scene set changes. + useEffect(() => { + syncData(); + }, [scenes, syncData]); + + // Update only the active-highlight flag on hover/select without refitting. + useEffect(() => { + const ds = dataSourceRef.current; + if (!ds || !readyRef.current) return; + for (const shape of ds.getShapes()) { + const shapeProps = shape.getProperties(); + const active = shapeProps.sceneId === activeId; + if (shapeProps._active !== active) { + shape.setProperties({ ...shapeProps, _active: active }); + } + } + }, [activeId]); + + // Render the persistent clip-AOI rectangle whenever it changes. + useEffect(() => { + if (!readyRef.current) return; + renderClipAoi(clipAoi); + }, [clipAoi, renderClipAoi]); + + // Preview the selected scene's actual imagery by streaming its COG through + // TiTiler as a tile layer (no Azure Maps subscription needed), inserted + // below the footprint outlines so they stay visible on top. + useEffect(() => { + const map = mapRef.current; + if (!map || !readyRef.current) return; + + if (previewLayerRef.current) { + map.layers.remove(previewLayerRef.current); + previewLayerRef.current = null; + } + previewSourceIdRef.current = null; + + const tileUrl = titilerTileUrl(previewScene?.cogUrl); + + // While previewing imagery, make the footprint fill fully transparent + // (not hidden) so its colored tint doesn't wash out the scene BUT the + // footprints stay clickable — the user can click another footprint on the + // map to switch scenes. Outlines (the line layer) remain visible. + if (fillLayerRef.current) { + fillLayerRef.current.setOptions({ + fillOpacity: tileUrl ? 0 : FILL_OPACITY_EXPR, + }); + } + + if (!tileUrl) { + setTilesLoading(false); + return; + } + + const layer = new window.atlas.layer.TileLayer( + { + tileUrl, + opacity: 1, + minSourceZoom: 1, + maxSourceZoom: 22, + bounds: previewScene.bbox, + }, + "odcPreview" + ); + // Insert beneath the footprint fill so outlines remain on top. + map.layers.add(layer, "odcFill"); + previewLayerRef.current = layer; + + // Track this preview's GL source id so the "sourcedata" listener knows + // when ITS tiles have loaded, and show the loading indicator meanwhile. + setTilesLoading(true); + const glMap = findGlMap(map); + previewSourceIdRef.current = + glMap?.getLayer?.("odcPreview")?.source ?? null; + // Fallback so the spinner never sticks if the source id can't be resolved + // or tiles error out silently. + const sceneUid = previewScene.uid || previewScene.id; + const timer = setTimeout(() => { + if ((previewScene.uid || previewScene.id) === sceneUid) { + setTilesLoading(false); + } + }, 15000); + + if (previewScene.bbox) { + map.setCamera({ + bounds: previewScene.bbox, + padding: 60, + type: "fly", + duration: 600, + }); + } + + return () => clearTimeout(timer); + }, [previewScene]); + + return ( +
+
+ {tilesLoading && ( +
+ Loading imagery… +
+ )} +
+ ); +}; + +OpenDataCatalogMap.propTypes = { + scenes: PropTypes.array.isRequired, + activeId: PropTypes.string, + previewScene: PropTypes.object, + clipMode: PropTypes.bool, + clipAoi: PropTypes.array, + onHover: PropTypes.func.isRequired, + onSelect: PropTypes.func.isRequired, + onClipDrawn: PropTypes.func, + center: PropTypes.array, +}; + +export default OpenDataCatalogMap; diff --git a/ui/src/Components/OpenDataCatalog/OpenDataCatalogPanel.jsx b/ui/src/Components/OpenDataCatalog/OpenDataCatalogPanel.jsx new file mode 100644 index 0000000..1a1bf69 --- /dev/null +++ b/ui/src/Components/OpenDataCatalog/OpenDataCatalogPanel.jsx @@ -0,0 +1,545 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// Open Data Catalog explorer — a side panel on the Create Image Layer form +// that browses Vantor/Maxar and Planet open disaster imagery and adds a +// scene's COG URL straight into the pre/post imagery inputs. +// See spec/features/open-data-catalog/. +// +// Layout: opens from the left (customNear) and reads left→right — a scene +// list on the left, a map on the right that shows footprints and previews +// the selected scene's imagery (via TiTiler) so you never scroll to preview. +import { useState, useEffect, useMemo, useCallback, useRef } from "react"; +import PropTypes from "prop-types"; +import { + Panel, + PanelType, + Dropdown, + Spinner, + SpinnerSize, + MessageBar, + MessageBarType, + Pivot, + PivotItem, + Text, + DefaultButton, + Checkbox, +} from "@fluentui/react"; + +import { + discoverEvents, + fetchEventCatalog, + bboxIntersects, + bboxContains, +} from "./openDataCatalog"; +import { isAzureMapsPlaceholder } from "../../util/azureMapsAuth"; +import OpenDataCatalogMap from "./OpenDataCatalogMap"; +import SceneListItem from "./SceneListItem"; + +const OpenDataCatalogPanel = ({ + isOpen, + onDismiss, + onAddScene, + clipAoi, + onClipAoiChange, + preUrls, + postUrls, +}) => { + const [events, setEvents] = useState([]); + const [eventKey, setEventKey] = useState(null); + const [discovering, setDiscovering] = useState(false); + const [discoverErrors, setDiscoverErrors] = useState([]); + + const [scenes, setScenes] = useState([]); + const [loading, setLoading] = useState(false); + const [errors, setErrors] = useState([]); + const [loadError, setLoadError] = useState(""); + const [addError, setAddError] = useState(""); + + const [sourceFilter, setSourceFilter] = useState("all"); + const [phaseFilter, setPhaseFilter] = useState("all"); + const [hoveredId, setHoveredId] = useState(null); + const [selectedScene, setSelectedScene] = useState(null); + + // Server-side clip: drawing a box sets a single layer-level AOI ([w,s,e,n] + // EPSG:4326). Imagery prep clips the pre/post mosaics to it — the add is + // instant here (no client-side crop/upload). The AOI persists across scene + // selection (it's a property of the layer, not a scene). + const [clipMode, setClipMode] = useState(false); + + // Leaving draw mode when the previewed scene changes (the AOI itself stays). + useEffect(() => { + setClipMode(false); + }, [selectedScene]); + + const handleClipDrawn = useCallback( + (bbox) => { + onClipAoiChange(bbox); + setClipMode(false); + }, + [onClipAoiChange] + ); + + const event = useMemo( + () => events.find((e) => e.key === eventKey), + [events, eventKey] + ); + + // Discover all available events when the panel first opens. Keyed on + // `isOpen` ONLY — deliberately not on `events.length`/`discovering`, so + // flipping those bits of state mid-fetch doesn't tear down and cancel the + // in-flight discovery (which would leave the spinner stuck forever). + useEffect(() => { + if (!isOpen || events.length > 0) return undefined; + let cancelled = false; + setDiscovering(true); + setDiscoverErrors([]); + discoverEvents() + .then((result) => { + if (cancelled) return; + setEvents(result.events); + setDiscoverErrors(result.errors); + if (result.events.length > 0) setEventKey(result.events[0].key); + }) + .catch((err) => { + if (!cancelled) setLoadError(err.message || "Failed to discover events."); + }) + .finally(() => { + if (!cancelled) setDiscovering(false); + }); + return () => { + cancelled = true; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [isOpen]); + + // Load the catalog whenever the selected event changes. + useEffect(() => { + if (!isOpen || !event) return undefined; + let cancelled = false; + setLoading(true); + setScenes([]); + setErrors([]); + setLoadError(""); + setAddError(""); + setSelectedScene(null); + setHoveredId(null); + + fetchEventCatalog(event) + .then((result) => { + if (cancelled) return; + setScenes(result.scenes); + setErrors(result.errors); + if (result.scenes.length === 0 && result.errors.length > 0) { + setLoadError( + "Could not load any open data for this event. See the source errors below." + ); + } + }) + .catch((err) => { + if (cancelled) return; + setLoadError(err.message || "Failed to load the open data catalog."); + }) + .finally(() => { + if (!cancelled) setLoading(false); + }); + + return () => { + cancelled = true; + }; + }, [isOpen, event]); + + // Restrict to scenes whose footprint overlaps the drawn clip AOI. A scene + // that doesn't overlap the layer-level AOI contributes nothing to the + // clipped mosaic, so hide it (default on whenever an AOI is set). + const [aoiOnly, setAoiOnly] = useState(true); + const aoiScenes = useMemo(() => { + if (!clipAoi || !aoiOnly) return scenes; + return scenes.filter((s) => bboxIntersects(s.bbox, clipAoi)); + }, [scenes, clipAoi, aoiOnly]); + + const counts = useMemo(() => { + const c = { Vantor: 0, Planet: 0, pre: 0, post: 0 }; + for (const s of aoiScenes) { + c[s.source] = (c[s.source] || 0) + 1; + if (s.phase) c[s.phase] += 1; + } + return c; + }, [aoiScenes]); + + const filtered = useMemo( + () => + aoiScenes.filter( + (s) => + (sourceFilter === "all" || s.source === sourceFilter) && + (phaseFilter === "all" || s.phase === phaseFilter) + ), + [aoiScenes, sourceFilter, phaseFilter] + ); + + const preSet = useMemo(() => new Set(preUrls || []), [preUrls]); + const postSet = useMemo(() => new Set(postUrls || []), [postUrls]); + + const handleAdd = useCallback( + (scene, field) => { + const result = onAddScene(scene, field); + setAddError(result && result.ok ? "" : (result && result.error) || ""); + }, + [onAddScene] + ); + + const activeId = hoveredId || selectedScene?.uid || null; + + // When a scene is selected (notably by clicking a footprint on the map), + // scroll the matching list row into view so the highlighted/expanded item + // is visible without hunting for it. + const listRef = useRef(null); + useEffect(() => { + if (!selectedScene || !listRef.current) return; + const uid = selectedScene.uid || selectedScene.id; + const el = listRef.current.querySelector( + `[data-scene-uid="${window.CSS?.escape ? window.CSS.escape(uid) : uid}"]` + ); + el?.scrollIntoView({ block: "nearest", behavior: "smooth" }); + }, [selectedScene]); + + return ( + +
+ {/* Header controls (full width) */} +
+ + Browse open disaster imagery from the Vantor/Maxar and Planet Open + Data Programs, then add a scene directly to your pre- or post-event + imagery. Imagery is licensed CC BY-NC 4.0. + + +
+ ({ + key: e.key, + text: `${e.name}${ + e.sources.vantor && e.sources.planet + ? " · Vantor + Planet" + : e.sources.vantor + ? " · Vantor" + : " · Planet" + }`, + }))} + onChange={(e, opt) => setEventKey(opt.key)} + styles={{ root: { minWidth: 320 } }} + /> + setSourceFilter(item.props.itemKey)} + headersOnly + styles={{ root: { minHeight: 32 } }} + > + + + + + setPhaseFilter(item.props.itemKey)} + headersOnly + styles={{ root: { minHeight: 32 } }} + > + + + + +
+ + {/* When a clip AOI is set, offer to restrict the catalog to scenes + that actually overlap it — so pre/post picks share the AOI. */} + {clipAoi && ( + setAoiOnly(!!v)} + styles={{ text: { fontSize: 12 } }} + /> + )} +
+ + {addError && ( + setAddError("")} + className="mx-3 mb-2" + > + {addError} + + )} + {loadError && ( + + {loadError} + + )} + {discoverErrors.map((e) => ( + + Could not list {e.source} events: {e.message} + + ))} + {errors.map((e) => ( + + {e.source} imagery could not be loaded: {e.message} + + ))} + + {/* Body: list (left) + map (right) */} +
+ {/* Scene list */} +
+ {loading || discovering ? ( +
+ +
+ ) : filtered.length === 0 ? ( + + {scenes.length === 0 + ? "No imagery available for this event." + : "No scenes match the current filters."} + + ) : ( + filtered.map((scene) => ( + + )) + )} +
+ + {/* Map / preview */} +
+ {isOpen && ( + + )} + + {/* Server-side clip AOI toolbar */} + {((selectedScene && selectedScene.cogUrl) || clipAoi) && ( +
+ {clipMode ? ( + + Drag a box to set the clip area. + setClipMode(false)} + styles={{ root: { height: 24, minWidth: 0, padding: "0 8px" } }} + /> + + ) : clipAoi ? ( + + Clip area set — imagery is clipped to it during processing. + setClipMode(true)} + styles={{ root: { height: 26, minWidth: 0, padding: "0 8px" } }} + /> + onClipAoiChange(null)} + styles={{ root: { height: 26, minWidth: 0, padding: "0 8px" } }} + /> + + ) : ( + setClipMode(true)} + styles={{ root: { background: "#fff" } }} + /> + )} +
+ )} +
+ {selectedScene ? ( + + Previewing: {selectedScene.place || selectedScene.title || selectedScene.id} + + ) : ( + + Select a scene to preview its imagery here. + + )} + {isAzureMapsPlaceholder && ( + + Satellite basemap disabled (no Azure Maps key) — footprints and + scene previews still work. + + )} +
+
+
+
+ + ); +}; + +OpenDataCatalogPanel.propTypes = { + isOpen: PropTypes.bool.isRequired, + onDismiss: PropTypes.func.isRequired, + onAddScene: PropTypes.func.isRequired, + clipAoi: PropTypes.array, + onClipAoiChange: PropTypes.func.isRequired, + preUrls: PropTypes.array, + postUrls: PropTypes.array, +}; + +export default OpenDataCatalogPanel; diff --git a/ui/src/Components/OpenDataCatalog/SceneListItem.jsx b/ui/src/Components/OpenDataCatalog/SceneListItem.jsx new file mode 100644 index 0000000..03d921c --- /dev/null +++ b/ui/src/Components/OpenDataCatalog/SceneListItem.jsx @@ -0,0 +1,243 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import PropTypes from "prop-types"; +import { DefaultButton, Icon, Text, Link } from "@fluentui/react"; + +import { SOURCE_COLORS } from "./openDataCatalog"; + +function formatDate(iso) { + if (!iso) return "—"; + return iso.replace("T", " ").replace(/\.\d+Z?$/, "").replace("Z", "").slice(0, 16) + " UTC"; +} + +function formatGsd(gsd) { + if (gsd == null || gsd === "") return null; + return `${Number(gsd).toFixed(2)} m GSD`; +} + +function formatSize(bytes) { + if (!bytes || !Number.isFinite(Number(bytes))) return null; + let n = Number(bytes); + const units = ["B", "KB", "MB", "GB", "TB"]; + let i = 0; + while (n >= 1024 && i < units.length - 1) { + n /= 1024; + i += 1; + } + return `${n.toFixed(n < 10 && i > 0 ? 1 : 0)} ${units[i]}`; +} + +const Badge = ({ label, color, filled }) => ( + + {label} + +); +Badge.propTypes = { + label: PropTypes.string.isRequired, + color: PropTypes.string.isRequired, + filled: PropTypes.bool, +}; + +// Extra metadata shown only when a scene is selected. Rows with no value +// are omitted so the panel stays compact. +function ExpandedMeta({ scene }) { + const rows = [ + ["Captured", formatDate(scene.datetime)], + [ + "Sensor", + [scene.sensor, scene.constellation].filter(Boolean).join(" · ") || null, + ], + ["GSD", formatGsd(scene.gsd)], + ["Cloud", scene.cloud == null ? null : `${scene.cloud}%`], + [ + "Off-nadir", + scene.offNadir == null ? null : `${Number(scene.offNadir).toFixed(1)}°`, + ], + [ + "Sun elev.", + scene.sunElev == null ? null : `${Number(scene.sunElev).toFixed(1)}°`, + ], + ["Size", formatSize(scene.cogSize)], + ].filter(([, v]) => v != null && v !== "" && v !== "—"); + + return ( +
e.stopPropagation()} + > +
+ {rows.map(([k, v]) => ( +
+ {k} + + {v} + +
+ ))} +
+ {scene.sourceUrl && ( + + ↗ View source + + )} +
+ ); +} +ExpandedMeta.propTypes = { scene: PropTypes.object.isRequired }; + +const SceneListItem = ({ + scene, + isHovered, + isSelected, + onHover, + onSelect, + onAdd, + addedPre, + addedPost, + coversAoi, + disabled, +}) => { + const color = SOURCE_COLORS[scene.source] || "#616161"; + const gsd = formatGsd(scene.gsd); + const hasCog = !!scene.cogUrl; + + // A scene can only be added to its own phase — pre imagery to Pre, post to + // Post. When the phase is unknown, offer both as a fallback. + const showPre = scene.phase === "pre" || !scene.phase; + const showPost = scene.phase === "post" || !scene.phase; + + return ( +
onHover(scene.uid || scene.id)} + onMouseLeave={() => onHover(null)} + onClick={() => onSelect(scene)} + > +
+ {!scene.thumbUrl && ( + + )} +
+ +
+
+ + {scene.phase && ( + + )} + {coversAoi && } +
+ + {scene.place || scene.title || scene.id} + + + {formatDate(scene.datetime)} + {scene.sensor ? ` · ${scene.sensor}` : ""} + {gsd ? ` · ${gsd}` : ""} + + +
e.stopPropagation()}> + {showPre && ( + onAdd(scene, "preEventImageryUrls")} + styles={{ root: { minWidth: 0, height: 26, padding: "0 8px", fontSize: 12 } }} + /> + )} + {showPost && ( + onAdd(scene, "postEventImageryUrls")} + styles={{ root: { minWidth: 0, height: 26, padding: "0 8px", fontSize: 12 } }} + /> + )} +
+ {!hasCog && ( + + COG not linked yet — footprint only. + + )} + + {isSelected && } +
+
+ ); +}; + +SceneListItem.propTypes = { + scene: PropTypes.object.isRequired, + isHovered: PropTypes.bool, + isSelected: PropTypes.bool, + onHover: PropTypes.func.isRequired, + onSelect: PropTypes.func.isRequired, + onAdd: PropTypes.func.isRequired, + addedPre: PropTypes.bool, + addedPost: PropTypes.bool, + coversAoi: PropTypes.bool, + disabled: PropTypes.bool, +}; + +export default SceneListItem; diff --git a/ui/src/Components/OpenDataCatalog/openDataCatalog.js b/ui/src/Components/OpenDataCatalog/openDataCatalog.js new file mode 100644 index 0000000..c276dac --- /dev/null +++ b/ui/src/Components/OpenDataCatalog/openDataCatalog.js @@ -0,0 +1,567 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// Open Data Catalog — discover + fetch + normalize open disaster-response +// imagery from the Vantor/Maxar Open Data Program (S3 STAC) and the Planet +// Open Data Program (Source Cooperative STAC). Adapted from the standalone +// prototype (Open Disaster Response Data Visualizer): +// https://visualizers.aiforgood.ai/damage-assessment/venezuela_earthqake_data_explorer.html +// +// This module is deliberately framework-free (no React, no atlas): it is the +// single seam where the catalog is produced. Today it runs client-side; the +// `discoverEvents()` / `fetchEventCatalog(event)` contract can later be moved +// behind an Azure Function with no change to the UI components that consume +// it. See spec/features/open-data-catalog/. + +// Root STAC catalogs that enumerate every available disaster event. +const VANTOR_ROOT_CATALOG = + "https://vantor-opendata.s3.amazonaws.com/events/catalog.json"; +const PLANET_ROOT_CATALOG = + "https://data.source.coop/planet/disasterdata/catalog.json"; + +export const SOURCE_COLORS = { Vantor: "#0078d4", Planet: "#00b294" }; + +// Maps a normalized scene to the HASTE source-type dropdown key +// (see sourceTypeOptions in CreateEditImageLayerHelper.js). +const SOURCE_TYPE_KEYS = { + vantor: "maxar", // Vantor is the rebranded Maxar Open Data Program + planetSkysat: "planet_skysat", + planet: "planet_scope", +}; + +// ── COG preview tiles (TiTiler) ───────────────────────────────────────────── +// +// HASTE runs a TiTiler instance that streams tiles from any remote COG, reached +// from the browser through the api-proxy at VITE_TITILER_URL (…/api/titiler/). +// This lets the explorer preview a scene's actual imagery on the map without an +// Azure Maps subscription. Returns an Azure-Maps-style {z}/{x}/{y} template. +function titilerBase() { + const raw = + (typeof import.meta !== "undefined" && import.meta.env?.VITE_TITILER_URL) || + "/api/titiler/"; + return raw.replace(/\/+$/, ""); +} + +export function titilerTileUrl(cogUrl) { + if (!cogUrl) return null; + return `${titilerBase()}/cog/tiles/{z}/{x}/{y}.png?url=${encodeURIComponent( + cogUrl + )}`; +} + +// Maximum width/height (px) of a clipped export. A clip at native GSD over a +// large box would be enormous (an 8k² RGB GeoTIFF is ~190 MB) and TiTiler +// can't generate + stream it from the remote COG before the proxy's timeout, +// yielding a 504. Cap the longest side so the crop stays fast and a +// reasonable upload size; TiTiler downscales larger areas. +const CLIP_MAX_DIM = 4096; + +// Build a TiTiler crop URL that returns a georeferenced GeoTIFF of `bbox` +// ([west, south, east, north], EPSG:4326) from `cogUrl`, sized to roughly the +// scene's native resolution (`gsd`, metres/px) up to CLIP_MAX_DIM. +export function titilerCropUrl(cogUrl, bbox, gsd) { + if (!cogUrl || !bbox || bbox.length < 4) return null; + const [w, s, e, n] = bbox; + const midLat = (s + n) / 2; + const mPerDegLat = 111320; + const mPerDegLon = 111320 * Math.cos((midLat * Math.PI) / 180); + const res = gsd && gsd > 0 ? gsd : 0.5; + let width = Math.max(1, Math.round(((e - w) * mPerDegLon) / res)); + let height = Math.max(1, Math.round(((n - s) * mPerDegLat) / res)); + const scale = Math.min(1, CLIP_MAX_DIM / Math.max(width, height)); + width = Math.max(1, Math.round(width * scale)); + height = Math.max(1, Math.round(height * scale)); + const box = [w, s, e, n].map((v) => v.toFixed(8)).join(","); + return `${titilerBase()}/cog/crop/${box}/${width}x${height}.tif?url=${encodeURIComponent( + cogUrl + )}`; +} + +export function clipFileName(scene) { + const base = (scene?.id || "scene").replace(/[^A-Za-z0-9_-]+/g, "_"); + return `${base}_clip.tif`; +} + +// ── AOI overlap helpers (bbox = [west, south, east, north]) ───────────────── +// +// Used to filter the catalog to scenes that actually cover the drawn clip AOI: +// a scene whose footprint doesn't overlap the layer-level AOI contributes +// nothing to the clipped mosaic, so it shouldn't be offered. + +export function bboxIntersects(a, b) { + if (!a || !b || a.length < 4 || b.length < 4) return false; + return a[0] < b[2] && a[2] > b[0] && a[1] < b[3] && a[3] > b[1]; +} + +// True when `outer` fully contains `inner` (scene footprint covers the AOI). +export function bboxContains(outer, inner) { + if (!outer || !inner || outer.length < 4 || inner.length < 4) return false; + return ( + outer[0] <= inner[0] && + outer[1] <= inner[1] && + outer[2] >= inner[2] && + outer[3] >= inner[3] + ); +} + +// ── shared helpers ────────────────────────────────────────────────────────── + +function absUrl(href, base) { + return new URL(href, base).href; +} + +function delay(ms) { + return new Promise((r) => setTimeout(r, ms)); +} + +function resolvePhase(explicit, datetime, eventDate) { + if (explicit === "pre" || explicit === "post") return explicit; + if (!datetime || !eventDate) return null; + return datetime.slice(0, 10) < eventDate ? "pre" : "post"; +} + +function bboxToGeometry(bbox) { + if (!bbox || bbox.length < 4) return null; + const [w, s, e, n] = bbox; + return { + type: "Polygon", + coordinates: [[[w, s], [e, s], [e, n], [w, n], [w, s]]], + }; +} + +function sceneSortKey(a, b) { + const phaseRank = (p) => (p === "pre" ? 0 : 1); + const sourceRank = (s) => (s === "Vantor" ? 0 : 1); + return ( + phaseRank(a.phase) - phaseRank(b.phase) || + sourceRank(a.source) - sourceRank(b.source) || + (a.datetime || "").localeCompare(b.datetime || "") + ); +} + +async function fetchJson(url) { + const res = await fetch(url, { mode: "cors" }); + if (!res.ok) throw new Error(`Fetch failed (${res.status}): ${url}`); + return res.json(); +} + +// Fetch JSON with small retry/backoff — STAC hosts occasionally 5xx. +async function fetchJsonRetry(url, attempts = 3) { + for (let i = 1; i <= attempts; i++) { + try { + const res = await fetch(url, { mode: "cors" }); + if (res.ok) return { doc: await res.json(), url: res.url || url }; + if (res.status < 500 || i === attempts) { + throw new Error(`Fetch failed (${res.status}): ${url}`); + } + } catch (err) { + if (i === attempts) throw err; + } + await delay(250 * i); + } + throw new Error(`Fetch failed: ${url}`); +} + +function stacLink(doc, rel, base) { + const link = (doc.links || []).find((l) => l.rel === rel && l.href); + return link ? absUrl(link.href, base) : null; +} + +function stacLinks(doc, rel) { + return (doc.links || []).filter((l) => l.rel === rel && l.href); +} + +function assetHref(asset, base) { + return asset && asset.href ? absUrl(asset.href, base) : null; +} + +function isGeotiffAsset(asset) { + return ( + ((asset && asset.type) || "").includes("geotiff") || + /\.tiff?($|\?)/i.test((asset && asset.href) || "") + ); +} + +function pickVisualAsset(assets = {}) { + return ( + assets.visual || + Object.values(assets).find( + (a) => isGeotiffAsset(a) && (a.roles || []).includes("visual") + ) || + Object.values(assets).find(isGeotiffAsset) || + null + ); +} + +function assetSize(asset) { + const raw = asset && (asset["file:size"] || asset.size || asset.file_size); + const n = Number(raw); + return Number.isFinite(n) ? n : null; +} + +// ── event discovery ───────────────────────────────────────────────────────── + +const MONTHS = { + jan: "01", feb: "02", mar: "03", apr: "04", may: "05", jun: "06", + jul: "07", aug: "08", sep: "09", oct: "10", nov: "11", dec: "12", +}; + +const HAZARD_WORDS = new Set([ + "earthquake", "typhoon", "hurricane", "flood", "flooding", "ebola", + "wildfire", "fire", "cyclone", "tornado", "landslide", "volcano", + "eruption", "tsunami", "storm", "drought", "outbreak", "mudslide", +]); + +// Turn a catalog id (e.g. "Venezuela-Earthquake-Jun-2026" or +// "venezuela-earthquake-2026-06-24") into a human label + best-effort date. +function prettifyEventName(id) { + const words = []; + const dateParts = []; + for (const tok of id.split(/[-_]/).filter(Boolean)) { + const lt = tok.toLowerCase(); + if (/^\d+$/.test(tok) || MONTHS[lt]) { + dateParts.push(MONTHS[lt] ? tok[0].toUpperCase() + lt.slice(1) : tok); + } else { + words.push(tok[0].toUpperCase() + tok.slice(1)); + } + } + const name = words.join(" ") || id; + return dateParts.length ? `${name} (${dateParts.join(" ")})` : name; +} + +function parseDateFromId(id) { + let m = id.match(/(\d{4})-(\d{2})-(\d{2})/); + if (m) return `${m[1]}-${m[2]}-${m[3]}`; + m = id.toLowerCase().match(/([a-z]{3,})-(\d{4})/); + if (m && MONTHS[m[1].slice(0, 3)]) return `${m[2]}-${MONTHS[m[1].slice(0, 3)]}-15`; + m = id.match(/(\d{4})/); + if (m) return `${m[1]}-01-01`; + return null; +} + +// Split an id into place vs hazard tokens (dates/numbers dropped) so events +// from different catalogs can be matched (e.g. Vantor "Venezuela-Earthquake- +// Jun-2026" ↔ Planet "venezuela-earthquake-2026-06-24"). +function eventTokens(id) { + const toks = id + .toLowerCase() + .split(/[-_]/) + .filter((t) => t.length > 1 && !/^\d+$/.test(t) && !(t in MONTHS)); + const place = new Set(toks.filter((t) => !HAZARD_WORDS.has(t))); + const hazard = new Set(toks.filter((t) => HAZARD_WORDS.has(t))); + return { place, hazard }; +} + +function eventsMatch(idA, idB) { + const a = eventTokens(idA); + const b = eventTokens(idB); + const placeOverlap = [...a.place].some((t) => b.place.has(t)); + if (!placeOverlap) return false; + const hazardOverlap = [...a.hazard].some((t) => b.hazard.has(t)); + return hazardOverlap || a.hazard.size === 0 || b.hazard.size === 0; +} + +async function discoverVantorEvents() { + const root = await fetchJson(VANTOR_ROOT_CATALOG); + return stacLinks(root, "child").map((l) => { + const href = absUrl(l.href, VANTOR_ROOT_CATALOG); + const m = href.match(/events\/([^/]+)\/collection\.json/); + const id = m ? m[1] : href; + return { id, collectionUrl: href, date: parseDateFromId(id) }; + }); +} + +async function discoverPlanetEvents() { + const { doc: root, url } = await fetchJsonRetry(PLANET_ROOT_CATALOG); + return stacLinks(root, "child").map((l) => { + const catalogUrl = absUrl(l.href, url); + const m = catalogUrl.match(/disasterdata\/([^/]+)\/catalog\.json/); + const id = m ? m[1] : catalogUrl; + return { id, catalogUrl, date: parseDateFromId(id) }; + }); +} + +/** + * Discover every disaster event available across Vantor and Planet, merging + * events that appear in both catalogs. Resilient: if one root catalog fails, + * the other's events are still returned. + * + * @returns {Promise<{ events: Array, errors: Array<{source, message}> }>} + * each event: { key, name, date, sources: { vantor?, planet? } } + */ +export async function discoverEvents() { + const errors = []; + const [vantor, planet] = await Promise.all([ + discoverVantorEvents().catch((err) => { + console.error("Vantor event discovery failed:", err); + errors.push({ source: "Vantor", message: err.message }); + return []; + }), + discoverPlanetEvents().catch((err) => { + console.error("Planet event discovery failed:", err); + errors.push({ source: "Planet", message: err.message }); + return []; + }), + ]); + + const events = []; + const planetUsed = new Set(); + for (const v of vantor) { + const match = planet.find( + (p) => !planetUsed.has(p.id) && eventsMatch(v.id, p.id) + ); + if (match) planetUsed.add(match.id); + events.push({ + key: v.id, + name: prettifyEventName(v.id), + date: v.date || match?.date || null, + sources: match ? { vantor: v, planet: match } : { vantor: v }, + }); + } + for (const p of planet) { + if (planetUsed.has(p.id)) continue; + events.push({ + key: p.id, + name: prettifyEventName(p.id), + date: p.date, + sources: { planet: p }, + }); + } + + // Most recent first; unknown dates last. + events.sort((a, b) => (b.date || "").localeCompare(a.date || "")); + return { events, errors }; +} + +// ── Vantor Open Data (STAC item links) ────────────────────────────────────── + +function normalizeVantorItem(item, eventDate) { + const props = item.properties || {}; + const visual = pickVisualAsset(item.assets); + const cogUrl = visual?.href || null; + const thumbUrl = + item.assets?.thumbnail?.href || + (cogUrl ? cogUrl.replace(/\.tif$/i, ".jpg") : null); + return { + id: item.id, + source: "Vantor", + phase: resolvePhase(props.phase, props.datetime, eventDate), + cogUrl, + thumbUrl, + bbox: item.bbox, + geometry: item.geometry || bboxToGeometry(item.bbox), + datetime: props.datetime || null, + title: props.title || item.id, + place: props.location || props.location_slug || null, + sensor: props.vehicle_name || props.platform || null, + constellation: props.constellation || null, + gsd: props.pan_gsd ?? props.gsd ?? null, + cloud: props["eo:cloud_cover"] ?? null, + offNadir: props["view:off_nadir"] ?? null, + sunElev: props["view:sun_elevation"] ?? null, + cogSize: assetSize(visual), + sourceUrl: cogUrl, + sourceTypeKey: SOURCE_TYPE_KEYS.vantor, + }; +} + +async function fetchVantorScenes(src) { + const collection = await fetchJson(src.collectionUrl); + const eventDate = (collection["odp:event_date"] || "").slice(0, 10) || null; + const itemLinks = stacLinks(collection, "item"); + const scenes = await Promise.all( + itemLinks.map(async (l) => { + try { + const item = await fetchJson(absUrl(l.href, src.collectionUrl)); + return normalizeVantorItem(item, eventDate); + } catch { + return null; + } + }) + ); + return scenes.filter(Boolean); +} + +// ── Planet Open Data (Source Cooperative STAC) ────────────────────────────── + +function planetPhaseFromCollectionId(id) { + if (id === "pre-event") return "pre"; + if (id === "post-event") return "post"; + return null; +} + +function planetSourceTypeKey(item, collection) { + const itemType = + item?.properties?.["pl:item_type"] || collection?.title || ""; + return /skysat/i.test(itemType) + ? SOURCE_TYPE_KEYS.planetSkysat + : SOURCE_TYPE_KEYS.planet; +} + +function normalizePlanetItem({ item, itemUrl, collection, aboutUrl, eventDate }) { + const props = item.properties || {}; + const visual = pickVisualAsset(item.assets); + const cogUrl = visual ? absUrl(visual.href, itemUrl) : null; + const thumbUrl = + assetHref(item.assets?.thumbnail, itemUrl) || + assetHref(collection?.assets?.thumbnail, itemUrl); + const phase = + planetPhaseFromCollectionId(item.collection || collection?.id) || + resolvePhase(null, props.datetime, eventDate); + return { + id: item.id, + source: "Planet", + phase, + cogUrl, + thumbUrl, + bbox: item.bbox, + geometry: item.geometry || bboxToGeometry(item.bbox), + datetime: props.datetime || props.start_datetime || props.end_datetime || null, + title: props.title || item.title || item.id, + place: props.location || props.location_slug || collection?.title || null, + sensor: props.platform || props["pl:item_type"] || props.constellation || null, + constellation: props.constellation || "planet", + gsd: props.gsd ?? collection?.summaries?.gsd?.[0] ?? null, + cloud: props["eo:cloud_cover"] ?? null, + offNadir: props["view:off_nadir"] ?? null, + sunElev: props["view:sun_elevation"] ?? null, + cogSize: assetSize(visual), + sourceUrl: cogUrl || aboutUrl, + sourceTypeKey: planetSourceTypeKey(item, collection), + }; +} + +// Some Planet collections expose a single pre-event mosaic asset instead of +// per-scene items. +function normalizePlanetMosaic({ collection, collectionUrl, asset }) { + const cogUrl = absUrl(asset.href, collectionUrl); + const [start] = collection.extent?.temporal?.interval?.[0] || []; + return { + id: `planet-${collection.id}-mosaic`, + source: "Planet", + phase: planetPhaseFromCollectionId(collection.id) || "pre", + cogUrl, + thumbUrl: assetHref(collection.assets?.thumbnail, collectionUrl) || null, + bbox: collection.extent?.spatial?.bbox?.[0] || null, + geometry: bboxToGeometry(collection.extent?.spatial?.bbox?.[0]), + datetime: start || null, + title: asset.title || collection.title || collection.id, + place: collection.title || null, + sensor: "Planet Basemap", + constellation: "planet", + gsd: collection.summaries?.gsd?.[0] ?? null, + cloud: null, + offNadir: null, + sunElev: null, + cogSize: assetSize(asset), + sourceUrl: cogUrl, + sourceTypeKey: SOURCE_TYPE_KEYS.planet, + }; +} + +async function fetchPlanetCollectionScenes({ collectionDoc, collectionUrl, aboutUrl, eventDate }) { + const phase = planetPhaseFromCollectionId(collectionDoc.id); + const mosaic = phase === "pre" ? collectionDoc.assets?.mosaic : null; + if (mosaic) { + return [ + normalizePlanetMosaic({ + collection: collectionDoc, + collectionUrl, + asset: mosaic, + }), + ]; + } + const itemLinks = stacLinks(collectionDoc, "item"); + const items = await Promise.all( + itemLinks.map(async (link) => { + const itemUrl = absUrl(link.href, collectionUrl); + try { + const { doc, url } = await fetchJsonRetry(itemUrl); + return normalizePlanetItem({ + item: doc, + itemUrl: url, + collection: collectionDoc, + aboutUrl, + eventDate, + }); + } catch (err) { + console.warn(`Skipping Planet STAC item ${itemUrl}: ${err.message}`); + return null; + } + }) + ); + return items.filter(Boolean); +} + +async function fetchPlanetScenes(src) { + const { doc: root, url: rootUrl } = await fetchJsonRetry(src.catalogUrl); + const childLinks = stacLinks(root, "child"); + const collections = await Promise.all( + childLinks.map((link) => fetchJsonRetry(absUrl(link.href, rootUrl))) + ); + const nested = await Promise.all( + collections.map(({ doc, url }) => + fetchPlanetCollectionScenes({ + collectionDoc: doc, + collectionUrl: url, + aboutUrl: + stacLink(doc, "about", url) || stacLink(root, "about", rootUrl), + eventDate: src.date, + }) + ) + ); + return nested.flat().filter(Boolean); +} + +// ── public API ────────────────────────────────────────────────────────────── + +/** + * Fetch and normalize the full catalog for a discovered event, pulling from + * whichever sources the event has. Resilient per-source: a failing source is + * dropped with a console warning and reported in `errors` rather than failing + * the whole catalog. + * + * @param {object} event a value from discoverEvents().events + * @returns {Promise<{ scenes: Array, errors: Array<{source, message}> }>} + */ +export async function fetchEventCatalog(event) { + const errors = []; + const vantorTask = event.sources?.vantor + ? fetchVantorScenes(event.sources.vantor).catch((err) => { + console.error("Vantor load failed:", err); + errors.push({ source: "Vantor", message: err.message }); + return []; + }) + : Promise.resolve([]); + const planetTask = event.sources?.planet + ? fetchPlanetScenes(event.sources.planet).catch((err) => { + console.error("Planet load failed:", err); + errors.push({ source: "Planet", message: err.message }); + return []; + }) + : Promise.resolve([]); + + const [vantor, planet] = await Promise.all([vantorTask, planetTask]); + const scenes = dedupeScenes([...vantor, ...planet]).sort(sceneSortKey); + // Guaranteed-unique identity for React keys / map feature ids, even if two + // scenes still share a STAC id after dedupe. + scenes.forEach((s, i) => { + s.uid = `${s.source}:${s.id}:${i}`; + }); + return { scenes, errors }; +} + +// Some source catalogs contain duplicate STAC ids (same scene listed twice). +// Collapse them by source + COG URL (falling back to id) so downstream React +// keys and map feature ids stay unique. +function dedupeScenes(scenes) { + const seen = new Set(); + const out = []; + for (const s of scenes) { + const key = `${s.source}|${s.cogUrl || s.id}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(s); + } + return out; +} diff --git a/ui/src/util/validation.js b/ui/src/util/validation.js index 87cc9d9..b8fdbe9 100644 --- a/ui/src/util/validation.js +++ b/ui/src/util/validation.js @@ -118,7 +118,7 @@ export function validateURL(url) { // Keep in sync with hastelib/src/hastegeo/core/utils/url_allowlist.py. const IMAGERY_URL_ALLOWED_HOST_DESCRIPTION = - "Azure Blob Storage (*.blob.core.windows.net) or AWS S3 (*.amazonaws.com)"; + "Azure Blob Storage (*.blob.core.windows.net), AWS S3 (*.amazonaws.com), or Source Cooperative (data.source.coop)"; const FOOTPRINT_URL_ALLOWED_HOST_DESCRIPTION = "Azure Blob Storage (*.blob.core.windows.net), AWS S3 (*.amazonaws.com), or the local upload host (in development)"; @@ -155,6 +155,12 @@ export function validateImageryUrlHost(url) { return [true, ""]; } + // Source Cooperative — the Planet Open Data STAC catalogs and COGs + // surfaced by the Open Data Catalog explorer are served from here. + if (host === "data.source.coop" || host.endsWith(".source.coop")) { + return [true, ""]; + } + return [ false, `URL host "${host}" is not on the allowlist. Allowed: ${IMAGERY_URL_ALLOWED_HOST_DESCRIPTION}.`,