feat: Open Data Catalog explorer with server-side clip

QUICKSTART.md

@@ -94,15 +94,23 @@ Compute the values:
 #          the VM's PUBLIC IP if you'll reach it from another machine.
 HOST_IP=localhost
 
-# DOCKER_GID: GID of the docker socket, so hastefuncqueues can spawn siblings.
-# Linux:
+# DOCKER_GID: GID that owns the docker socket *inside the container*, so
+# hastefuncqueues can spawn sibling containers (imageryprep/training).
+# Linux: the mounted socket keeps its host GID, so read it from the host:
 DOCKER_GID=$(stat -c '%g' /var/run/docker.sock)
-# macOS (BSD stat — the -c form does NOT work). Docker Desktop puts the socket
-# under $HOME, NOT /var/run, so resolve the real path from the docker context:
-# SOCK=$(docker context inspect --format '{{.Endpoints.docker.Host}}' | sed 's|unix://||')
-# DOCKER_GID=$(stat -f '%g' "$SOCK")   # typically 20 (staff) on macOS
+# macOS (Docker Desktop): the socket is bind-mounted into the container as
+# root:root, so the container-visible GID is 0 — NOT the host `stat` value
+# (which is typically 20/staff). Use 0 or the worker gets
+# `PermissionError(13, 'Permission denied')` and every job spawn fails:
+# DOCKER_GID=0
 ```
 
+> **⚠️ macOS gotcha:** set `DOCKER_GID=0`. The host socket may show GID 20, but
+> Docker Desktop presents it to the container as `root:root`, so `group_add`
+> must be 0. A non-zero value here is the usual cause of jobs that submit,
+> immediately fail, and show
+> `Error while fetching server API version: ... PermissionError(13)`.
+
 Write the file (adjust `HASTE_ENABLE_GPU` per your Gate-1 profile):
 
 ```bash

api/hastefuncapi/function_app.py

@@ -48,6 +48,7 @@
 from hastegeo.core.utils.logs import Logger
 from hastegeo.core.utils.metadata import MetadataUtils
 from hastegeo.core.utils.url_allowlist import (
+    validate_clip_bbox,
     validate_image_layer_imagery_urls,
     validate_image_layer_user_footprints_url,
 )
@@ -819,6 +820,10 @@ async def PutLayer(req: func.HttpRequest) -> func.HttpResponse:
         if footprint_url_error:
             return func.HttpResponse(footprint_url_error, status_code=400)
 
+        clip_bbox_error = validate_clip_bbox(image_data)
+        if clip_bbox_error:
+            return func.HttpResponse(clip_bbox_error, status_code=400)
+
         if image_data.imageLayerId is None:
             image_data.imageLayerId = MetadataUtils.generate_id()
         if image_data.creationDate is None:

docker/docker-compose.yml

@@ -173,6 +173,8 @@ services:
       # Memory settings for training containers (increase for large VMs)
       HASTE_DOCKER_SHM_SIZE: "${HASTE_DOCKER_SHM_SIZE:-64g}"
       HASTE_DOCKER_MEM_LIMIT: "${HASTE_DOCKER_MEM_LIMIT:-256g}"
+      # Debug: keep failed spawned containers so their logs can be inspected.
+      CLEANUP_CONTAINERS: "${CLEANUP_CONTAINERS:-1}"
     volumes:
       - ../data:/app/data
       - azurite-data:/shared/azurite  # Mount azurite blob storage for LocalRunner access

docker/nginx.conf

@@ -7,6 +7,11 @@ events {
 }
 
 http {
+    # The image-layer file uploader (incl. Open Data Catalog clip export)
+    # POSTs 4 MB chunks to /api/UploadFileByChunk. nginx's default 1 MB body
+    # limit rejects those with 413, so raise it well above the chunk size.
+    client_max_body_size 64m;
+
     upstream functions_backend {
         server hastefuncapi:8080;
     }
@@ -42,6 +47,13 @@ http {
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto $scheme;
+
+            # Clip/crop requests read from remote COGs and can take longer than
+            # nginx's default 60s to generate + stream. Give them headroom so
+            # the Open Data Catalog "Clip to area" export doesn't 504.
+            proxy_connect_timeout 30s;
+            proxy_send_timeout 300s;
+            proxy_read_timeout 300s;
         }
 
         location /api/ {

hastelib/src/hastegeo/core/models/projects.py

@@ -752,6 +752,11 @@ class ImageLayer(BaseModel):
     labelsUrl: Optional[str] = Field(default=None)
     buildingFootprintsUrl: Optional[str] = Field(default=None)
     userBuildingFootprintsUrl: Optional[str] = Field(default=None)
+    # Optional server-side clip AOI, [west, south, east, north] in EPSG:4326.
+    # When set, imagery prep clips the pre/post mosaics to this box (gdalwarp
+    # -te) so only the drawn area is processed/labeled. Set by the Open Data
+    # Catalog "clip to area" flow.
+    clipBbox: Optional[list[float]] = Field(default=None)
     validAreaMaskUrl: Optional[str] = Field(default=None)
     dependsOn: Optional[tuple[str, str]] = Field(
         default=("Project", "projectId")

hastelib/src/hastegeo/core/processors/imagery.py

@@ -292,6 +292,9 @@ def _execute_image_preprocess(self):
             "user_building_footprints_url": (
                 self.image_data.userBuildingFootprintsUrl
             ),
+            # Optional server-side clip AOI ([w, s, e, n] EPSG:4326). When set,
+            # prepare-imagery clips the pre/post mosaics to this box.
+            "clip_bbox": self.image_data.clipBbox,
         }
         self.storage.save(
             identifier=self.image_data.imageLayerId,

hastelib/src/hastegeo/core/utils/downloader.py

@@ -45,6 +45,11 @@ def download_imagery(self, **kwargs):
             dst_directory = kwargs.get("dst_directory", "./downloads")
             azure_urls = []
             aws_urls = []
+            # Allowlisted hosts without a dedicated SDK path (e.g. Source
+            # Cooperative / data.source.coop, used by the Planet Open Data
+            # catalog) are fetched over plain HTTPS via
+            # download_imagery_from_urls.
+            http_urls = []
 
             for url in urls:
                 try:
@@ -53,6 +58,8 @@ def download_imagery(self, **kwargs):
                         azure_urls.append(url)
                     elif url_type == "awss3":
                         aws_urls.append(url)
+                    else:
+                        http_urls.append(url)
                 except Exception as e:
                     self.logger.warning(f"Skipping URL not in allowlist: {e}")
 
@@ -65,6 +72,10 @@ def download_imagery(self, **kwargs):
                 file_paths.extend(
                     self.download_aws_s3_files(aws_urls, dst_directory)
                 )
+            if http_urls:
+                file_paths.extend(
+                    self.download_imagery_from_urls(http_urls, dst_directory)
+                )
             return file_paths
 
     @retry(

hastelib/src/hastegeo/core/utils/imagery.py

@@ -456,12 +456,17 @@ def mosaic_imagery(
         tif_files: List[str],
         output_file_path: str = None,
         gdal_warp_params: str = "",
+        clip_bbox: List[float] = None,
     ) -> str:
         """
         Parameters:
             tif_files (List[str]): List of paths to the input TIFF files.
             output_file_path (str, optional): Path to the output COG file. If not provided, a default path is generated.
             gdal_warp_params (str): GDAL warp parameters as a string.
+            clip_bbox (List[float], optional): [west, south, east, north] in
+                EPSG:4326. When provided, the output mosaic is clipped to this
+                box via gdalwarp ``-te``/``-te_srs`` (GDAL reprojects the bounds
+                to the mosaic's CRS), so only the drawn AOI is produced.
 
         Returns:
             str: Path to the output COG file.
@@ -549,9 +554,22 @@ def mosaic_imagery(
                     raise RuntimeError("VRT file does not exist on disk")
 
                 # Use gdal.Warp to convert the VRT to a COG (Cloud Optimized GeoTIFF)
-                warp_options = gdal.WarpOptions(
-                    options=gdal_warp_params.split()
-                )
+                warp_option_list = gdal_warp_params.split()
+                if clip_bbox:
+                    west, south, east, north = (float(v) for v in clip_bbox)
+                    # -te bounds are given in EPSG:4326 (-te_srs); GDAL
+                    # reprojects them to the mosaic's CRS before clipping.
+                    warp_option_list += [
+                        "-te",
+                        str(west),
+                        str(south),
+                        str(east),
+                        str(north),
+                        "-te_srs",
+                        "EPSG:4326",
+                    ]
+                    logger.info(f"Clipping mosaic to AOI bbox {clip_bbox}")
+                warp_options = gdal.WarpOptions(options=warp_option_list)
                 result = gdal.Warp(
                     output_file_path, str(vrt_path), options=warp_options
                 )

hastelib/src/hastegeo/core/utils/url_allowlist.py

@@ -16,8 +16,9 @@
 # Human-readable description used in user-facing error messages. Keep in
 # sync with the JS allowlist in ui/src/util/validation.js.
 ALLOWED_HOST_DESCRIPTION = (
-    "Azure Blob Storage (*.blob.core.windows.net) "
-    "or AWS S3 (*.amazonaws.com)"
+    "Azure Blob Storage (*.blob.core.windows.net), "
+    "AWS S3 (*.amazonaws.com), "
+    "or Source Cooperative (data.source.coop)"
 )
 
 # Building-footprint URLs additionally permit the host of the configured
@@ -62,6 +63,13 @@ def validate_imagery_url(url: str) -> str:
     ):
         return "awss3"
 
+    # Source Cooperative — Planet Open Data STAC catalogs and COGs surfaced
+    # by the Open Data Catalog explorer are served from data.source.coop.
+    # This is a public, well-known open-data host (not user-controlled),
+    # so allowlisting it does not meaningfully widen SSRF exposure.
+    if host == "data.source.coop" or host.endswith(".source.coop"):
+        return "sourcecoop"
+
     raise ValueError(
         f"URL host {host!r} is not on the allowlist of permitted imagery sources"
     )
@@ -232,3 +240,34 @@ def validate_image_layer_user_footprints_url(image_layer) -> Optional[str]:
             f"Rejected host: {host}."
         )
     return None
+
+
+def validate_clip_bbox(image_layer) -> Optional[str]:
+    """Validate the optional server-side clip AOI on an ImageLayer.
+
+    Returns a user-facing error message if ``clipBbox`` is set but malformed,
+    or ``None`` when it is absent or valid. Expected shape:
+    ``[west, south, east, north]`` in EPSG:4326 with ``west < east``,
+    ``south < north`` and coordinates within valid lon/lat ranges. Callers
+    (PutLayer) treat a non-None return as a hard 400.
+
+    Lives here (not in utils/aoi.py) so the lightweight PutLayer validation
+    path doesn't pull rasterio/GDAL into the API process just to bounds-check
+    four numbers.
+    """
+    bbox = getattr(image_layer, "clipBbox", None)
+    if bbox is None:
+        return None
+    if not isinstance(bbox, (list, tuple)) or len(bbox) != 4:
+        return "clipBbox must be a [west, south, east, north] array."
+    try:
+        west, south, east, north = (float(v) for v in bbox)
+    except (TypeError, ValueError):
+        return "clipBbox values must be numbers."
+    if not (-180 <= west <= 180 and -180 <= east <= 180):
+        return "clipBbox longitudes must be within [-180, 180]."
+    if not (-90 <= south <= 90 and -90 <= north <= 90):
+        return "clipBbox latitudes must be within [-90, 90]."
+    if west >= east or south >= north:
+        return "clipBbox must have west < east and south < north."
+    return None

hastelib/src/hastegeo/workflows/prepare_imagery.py

@@ -82,6 +82,7 @@ def __init__(
         fine_tune=False,
         dst_directory=None,
         user_building_footprints_url=None,
+        clip_bbox=None,
     ):
         """Initialize the ImageryWorkflow class.
         Args:
@@ -107,6 +108,9 @@ def __init__(
         self.fine_tune = fine_tune
         self.dst_directory = dst_directory or os.path.join(".", "outputs")
         self.user_building_footprints_url = user_building_footprints_url
+        # Optional server-side clip AOI ([w, s, e, n] EPSG:4326). Applied to
+        # both pre and post mosaics so only the drawn area is processed.
+        self.clip_bbox = clip_bbox
 
         self.pre_event_raw_paths = []
         self.pre_event_preview_paths = []
@@ -205,6 +209,7 @@ def process_pre_event(self):
             self.pre_event_raw_paths,
             dst_directory=self.dst_directory,
             save_as_prefix=save_as_prefix,
+            clip_bbox=self.clip_bbox,
         )
 
         cog_file_name = self.generate_prefix(PRE_EVENT_PROCESSED_COG_PREFIX)
@@ -243,6 +248,7 @@ def process_post_event(self):
             self.post_event_raw_paths,
             dst_directory=self.dst_directory,
             save_as_prefix=save_as_prefix,
+            clip_bbox=self.clip_bbox,
         )
 
         # The RGB band and scale adjusted imagery serves as supporting input to training and inference
@@ -763,19 +769,23 @@ def _save_raw_imagery(
     return raw_image_paths, preview_image_paths
 
 
-def _create_mosaic_cog(tif_files, dst_directory=None, save_as_prefix=None):
+def _create_mosaic_cog(
+    tif_files, dst_directory=None, save_as_prefix=None, clip_bbox=None
+):
     """Create a mosaic of TIFF files and convert to COG.
     Args:
         tif_files (list): List of TIFF files to mosaic (combine).
         dst_directory (str): (Optional) Directory to save the mosaic file. Defaults to current directory
         save_as_prefix (str): (Optional) Prefix for the saved mosaic file.
+        clip_bbox (list): (Optional) [w, s, e, n] EPSG:4326 AOI to clip to.
     Returns:
         str: Path to the created mosaic file."""
     dst_directory = dst_directory or "."
     return ImageryUtils.mosaic_imagery(
         tif_files,
         os.path.join(dst_directory, f"{save_as_prefix}.tif"),
         GDAL_WARP_PARAMS,
+        clip_bbox=clip_bbox,
     )
 
 
@@ -891,6 +901,7 @@ def main():
             user_building_footprints_url=config.get(
                 "user_building_footprints_url"
             ),
+            clip_bbox=config.get("clip_bbox"),
         )
 
         log_progress("Downloading imagery")

hastelib/tests/core/utils/test_url_allowlist.py

@@ -41,6 +41,17 @@ def test_accepts_aws_s3_host(self):
             "awss3",
         )
 
+    def test_accepts_source_cooperative_host(self):
+        # Planet Open Data STAC catalogs/COGs surfaced by the Open Data
+        # Catalog explorer are served from data.source.coop.
+        self.assertEqual(
+            validate_imagery_url(
+                "https://data.source.coop/planet/venezuela-earthquake"
+                "-2026-06-24/post-event/scene.tif"
+            ),
+            "sourcecoop",
+        )
+
     def test_rejects_arbitrary_host(self):
         with self.assertRaises(ValueError):
             validate_imagery_url("https://evil.example.com/x.tif")
@@ -322,5 +333,57 @@ def test_bad_url_reports_rejected_host(self):
         self.assertIn("evil.example", msg)
 
 
+class TestValidateClipBbox(unittest.TestCase):
+    """Tests for validate_clip_bbox (server-side clip AOI on an ImageLayer)."""
+
+    def _layer(self, bbox):
+        from types import SimpleNamespace
+
+        return SimpleNamespace(clipBbox=bbox)
+
+    def test_none_returns_none(self):
+        from hastegeo.core.utils.url_allowlist import validate_clip_bbox
+
+        self.assertIsNone(validate_clip_bbox(self._layer(None)))
+
+    def test_valid_bbox_returns_none(self):
+        from hastegeo.core.utils.url_allowlist import validate_clip_bbox
+
+        self.assertIsNone(
+            validate_clip_bbox(self._layer([-67.03, 10.54, -66.97, 10.61]))
+        )
+
+    def test_wrong_length_rejected(self):
+        from hastegeo.core.utils.url_allowlist import validate_clip_bbox
+
+        self.assertIsNotNone(validate_clip_bbox(self._layer([1, 2, 3])))
+
+    def test_non_numeric_rejected(self):
+        from hastegeo.core.utils.url_allowlist import validate_clip_bbox
+
+        self.assertIsNotNone(
+            validate_clip_bbox(self._layer([-67, "x", -66, 11]))
+        )
+
+    def test_out_of_range_rejected(self):
+        from hastegeo.core.utils.url_allowlist import validate_clip_bbox
+
+        self.assertIsNotNone(
+            validate_clip_bbox(self._layer([-67, 10, -66, 99]))
+        )
+
+    def test_inverted_bounds_rejected(self):
+        from hastegeo.core.utils.url_allowlist import validate_clip_bbox
+
+        # west >= east
+        self.assertIsNotNone(
+            validate_clip_bbox(self._layer([-66, 10.5, -67, 10.6]))
+        )
+        # south >= north
+        self.assertIsNotNone(
+            validate_clip_bbox(self._layer([-67, 10.6, -66, 10.5]))
+        )
+
+
 if __name__ == "__main__":
     unittest.main()