[DM-35482] Add support for generating the HiPS list

Makefile

@@ -1,9 +1,15 @@
 .PHONY: update-deps
 update-deps:
 	pip install --upgrade pip-tools pip setuptools
-	# Don't generate hashes here since we're installing daf_butler
-	pip-compile --upgrade --build-isolation --allow-unsafe --output-file requirements/main.txt requirements/main.in
-	pip-compile --upgrade --build-isolation --allow-unsafe --output-file requirements/dev.txt requirements/dev.in
+	pip-compile --upgrade --build-isolation --generate-hashes --output-file requirements/main.txt requirements/main.in
+	pip-compile --upgrade --build-isolation --generate-hashes --output-file requirements/dev.txt requirements/dev.in
+
+# Useful for testing against a Git version of a dependency.
+.PHONY: update-deps-no-hashes
+update-deps-no-hashes:
+	pip install --upgrade pip-tools pip setuptools
+	pip-compile --upgrade --build-isolation --output-file requirements/main.txt requirements/main.in
+	pip-compile --upgrade --build-isolation --output-file requirements/dev.txt requirements/dev.in
 
 .PHONY: init
 init:

requirements/dev.in

@@ -15,3 +15,5 @@ pre-commit
 pytest
 pytest-asyncio
 pytest-cov
+pytest-sugar
+respx

requirements/main.in

@@ -12,10 +12,8 @@ uvicorn[standard]
 
 # Other dependencies.
 google-cloud-storage
+httpx
 jinja2
+lsst-daf-butler
 safir
-
-# lsst.daf.butler is not yet on PyPI
-git+https://github.com/lsst/daf_butler.git@main#lsst-daf-butler
-boto3
-psycopg2
+structlog

src/datalinker/config.py

@@ -18,6 +18,18 @@ class Configuration:
     Set with the ``DATALINKER_CUTOUT_SYNC_URL`` environment variable.
     """
 
+    hips_base_url: str = os.getenv("DATALINKER_HIPS_BASE_URL", "")
+    """The base URL for HiPS lists.
+
+    Set with the ``DATALINKER_HIPS_BASE_URL`` environment variable.
+    """
+
+    token: str = os.getenv("DATALINKER_TOKEN", "")
+    """Token to use to authenticate to the HiPS service.
+
+    Set with the ``DATALINKER_TOKEN`` environment variable.
+    """
+
     name: str = os.getenv("SAFIR_NAME", "datalinker")
     """The application's name, which doubles as the root HTTP endpoint path.
 

src/datalinker/constants.py

@@ -0,0 +1,12 @@
+"""Constants that probably should be configuration options."""
+
+HIPS_DATASETS = (
+    "images/color_gri",
+    "images/band_u",
+    "images/band_g",
+    "images/band_r",
+    "images/band_i",
+    "images/band_z",
+    "images/band_y",
+)
+"""HiPS data sets to include in the HiPS list."""

src/datalinker/dependencies/hips.py

@@ -0,0 +1,94 @@
+"""HiPS list cache."""
+
+import re
+from typing import Optional
+
+from fastapi import Depends
+from httpx import AsyncClient
+from safir.dependencies.http_client import http_client_dependency
+from safir.dependencies.logger import logger_dependency
+from structlog.stdlib import BoundLogger
+
+from ..config import config
+from ..constants import HIPS_DATASETS
+
+__all__ = [
+    "HiPSListDependency",
+    "hips_list_dependency",
+]
+
+
+class HiPSListDependency:
+    """Maintain a cache of HiPS properties files for this deployment.
+
+    A deployment of the Science Platform may have several trees of HiPS data
+    served out of GCS.  Those need to be gathered together and served as a
+    unified HiPS list of all available trees.  Rather than making multiple API
+    calls to GCS each time this list is requested, cache the HiPS list
+    constructed from the ``properties`` files and prefer to serve them from
+    the cache.
+    """
+
+    def __init__(self) -> None:
+        self._hips_list: Optional[str] = None
+
+    async def __call__(
+        self,
+        client: AsyncClient = Depends(http_client_dependency),
+        logger: BoundLogger = Depends(logger_dependency),
+    ) -> str:
+        if not self._hips_list:
+            self._hips_list = await self._build_hips_list(client, logger)
+        return self._hips_list
+
+    async def _build_hips_list(
+        self, client: AsyncClient, logger: BoundLogger
+    ) -> str:
+        """Retrieve and cache properties files for all HiPS data sets.
+
+        Currently, this hard-codes the available lists.  This will eventually
+        be moved to configuration.
+
+        Parameters
+        ----------
+        client : `httpx.AsyncClient`
+            Client to use to retrieve the HiPS lists.
+        logger : `structlog.stdlib.BoundLogger`
+            Logger for any error messages.
+        """
+        lists = []
+        for dataset in HIPS_DATASETS:
+            url = config.hips_base_url + f"/{dataset}"
+            r = await client.get(
+                url + "/properties",
+                headers={"Authorization": f"bearer {config.token}"},
+            )
+            if r.status_code != 200:
+                logger.warning(
+                    "Unable to get HiPS list",
+                    url=url,
+                    status=r.status_code,
+                    error=r.reason_phrase,
+                )
+                continue
+            data = r.text
+
+            # Our HiPS properties files don't contain the URL
+            # (hips_service_url), but this is mandatory in the entries in the
+            # HiPS list.  Add it before hips_status.
+            service_url = "{:25}= {}".format("hips_service_url", url)
+            data = re.sub(
+                "^hips_status",
+                f"{service_url}\nhips_status",
+                r.text,
+                flags=re.MULTILINE,
+            )
+            lists.append(data)
+
+        # The HiPS list is the concatenation of all the properties files
+        # separated by blank lines.
+        return "\n".join(lists)
+
+
+hips_list_dependency = HiPSListDependency()
+"""The dependency that caches the HiPS list."""

src/datalinker/handlers/external.py

@@ -29,7 +29,7 @@
 )
 """FastAPI renderer for templated responses."""
 
-__all__ = ["get_index", "external_router"]
+__all__ = ["external_router"]
 
 
 def _get_butler(label: str) -> butler.Butler:

src/datalinker/handlers/hips.py

@@ -0,0 +1,24 @@
+"""Routes for the HiPS list.
+
+This is not part of the DataLink standard, but it is part of the overall
+purpose of datalinker to provide links and registries of other services in
+the same Science Platform deployment.  This route is a separate router because
+it doesn't require authentication and is served with a different prefix.
+"""
+
+from fastapi import APIRouter, Depends
+from fastapi.responses import PlainTextResponse
+
+from ..dependencies.hips import hips_list_dependency
+
+hips_router = APIRouter()
+"""FastAPI router for HiPS handlers."""
+
+__all__ = ["hips_router"]
+
+
+@hips_router.get(
+    "/list", response_class=PlainTextResponse, include_in_schema=False
+)
+async def get_hips_list(hips_list: str = Depends(hips_list_dependency)) -> str:
+    return hips_list

src/datalinker/main.py

@@ -17,6 +17,7 @@
 
 from .config import config
 from .handlers.external import external_router
+from .handlers.hips import hips_router
 from .handlers.internal import internal_router
 
 __all__ = ["app", "config"]
@@ -41,6 +42,7 @@
 # Attach the routers.
 app.include_router(internal_router)
 app.include_router(external_router, prefix="/api/datalink")
+app.include_router(hips_router, prefix="/api/hips")
 
 # Add the middleware.
 app.add_middleware(CaseInsensitiveQueryMiddleware)

tests/data/hips-properties

@@ -0,0 +1,32 @@
+creator_did              = temp://lsst/dp02_dc2/hips/images/color_gri
+obs_title                = DP0.2 HiPS from DESC DC2 sim: gri color visualization
+obs_description          = Color visualization of the DESC DC2 simulation (red: band i, green: band r, blue: band g) with a hue-preserving stretch.  Processed by Rubin Observatory pipeline, release 22, for Data Preview 0.2, and rendered to HiPS at its native resolution of 0.2 arcsec.  Corresponds to up to five years of the projected LSST survey data.  See https://dp0-2.lsst.io/ for documentation.
+prov_progenitor          = DESC DC2 simulation (doi:10.3847/1538-4365/abd62c)
+prov_progenitor          = Coaddition: LSST Science Pipelines v23.0.1 (https://pipelines.lsst.io/v/v23_0_1/index.html)
+prov_progenitor          = HiPS generation: internal pre-release code (https://pipelines.lsst.io/v/w_2022_22/index.html)
+obs_ack                  = We gratefully acknowledge permission to use the DC2 simulated dataset from the LSST Dark Energy Science Collaboration and thank the collaboration for all the work and insight it represents.
+obs_regime               = Optical
+data_pixel_bitpix        = -32
+dataproduct_type         = image
+moc_sky_fraction         = 0.008446268253974171
+data_ucd                 = phot.flux
+hips_creation_date       = 2022-06-26T01:25:28Z
+hips_builder             = lsst.pipe.tasks.hips.GenerateHipsTask
+hips_creator             = Vera C. Rubin Observatory
+hips_version             = 1.4
+hips_release_date        = 2022-06-26T01:25:28Z
+hips_frame               = equatorial
+hips_order               = 11
+hips_tile_width          = 512
+hips_status              = private master clonableOnce
+hips_tile_format         = png
+dataproduct_subtype      = color
+hips_pixel_bitpix        = -32
+hips_pixel_scale         = 5.591611998400726e-05
+hips_initial_ra          = 61.863
+hips_initial_dec         = -35.79
+hips_initial_fov         = 20.0
+em_min                   = 4.02e-07
+em_max                   = 8.18e-07
+t_min                    = 59582.04
+t_max                    = 61406.04

tests/handlers/hips_test.py

@@ -0,0 +1,44 @@
+"""Tests for the HiPS list routes."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+import respx
+from httpx import AsyncClient, Response
+
+from datalinker.config import config
+from datalinker.constants import HIPS_DATASETS
+
+
+@pytest.mark.asyncio
+async def test_hips_list(
+    client: AsyncClient, respx_mock: respx.Router
+) -> None:
+    hips_list_template = (
+        Path(__file__).parent.parent / "data" / "hips-properties"
+    ).read_text()
+    hips_lists = []
+    for dataset in HIPS_DATASETS:
+        url = f"https://hips.example.com/{dataset}"
+        respx_mock.get(url + "/properties").mock(
+            return_value=Response(200, text=hips_list_template)
+        )
+        hips_list = re.sub(
+            "^hips_status",
+            f"hips_service_url         = {url}\nhips_status",
+            hips_list_template,
+            flags=re.MULTILINE,
+        )
+        hips_lists.append(hips_list)
+
+    try:
+        config.hips_base_url = "https://hips.example.com"
+        r = await client.get("/api/hips/list")
+        assert r.status_code == 200
+        assert r.headers["Content-Type"].startswith("text/plain")
+        assert r.text == "\n".join(hips_lists)
+    finally:
+        config.hips_base_url = ""