From 763dc31a0198678dd7c9d1d1b6550ba347e2621a Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Wed, 21 Jun 2023 11:45:07 +0200 Subject: [PATCH 1/2] titiler-pgstac and tipg version --- README.md | 23 +++--- docker-compose.yml | 2 +- runtime/eoapi/raster/eoapi/raster/app.py | 11 ++- runtime/eoapi/raster/pyproject.toml | 2 +- runtime/eoapi/vector/eoapi/vector/app.py | 75 ++++++++++++------- runtime/eoapi/vector/eoapi/vector/config.py | 2 + .../vector/eoapi/vector/templates/header.html | 2 +- runtime/eoapi/vector/pyproject.toml | 2 +- 8 files changed, 78 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 3c86af1..1ce95c2 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,19 @@ --- ## Why should you use `eoAPI` + - **Focus on your use case:** `eoAPI` is used for large-scale data processing, building geographic information systems (GIS), creating real-time data applications, climate research and environmental monitoring, machine learning model training, and more. + - **Unified Repository:** `eoAPI` provides a single, unified repository to several state-of-the-art Earth Observation (EO) data services, including Metadata search (STAC), Raster, and Vector services. This can simplify the process of accessing and working with these services. + - **Interoperability:** `eoAPI` is designed to enable interoperability among its included services. This can make building complex applications that leverage different types of EO data easier. + - **Open Source and Community Support:** As an open-source project, `eoAPI` allows developers to inspect its code, contribute to its development, and use it as a base for custom solutions. It also benefits from the support and innovation of a community of developers and EO data users. + - **Scalability and Flexibility:** Each service in `eoAPI` can be used or deployed independently, which provides a lot of flexibility. If a developer's application only requires one or two of eoAPI's services, they don't need to deploy the entire suite. + - **Facilitate Earth Observation Tasks:** `eoAPI` includes specialized tools for working with EO data, such as dynamic tiling, metadata searching, and features/vector tiles API. These can significantly facilitate EO data processing, analysis, and visualization. + - **Ease of Deployment:** `eoAPI` supports containerized deployment using Docker, making it easier to set up, scale, and maintain applications built on it. Spin up the demo locally and start experimenting in minutes. --- @@ -50,10 +57,12 @@ - **STAC Metadata**: Built with [stac-fastapi.pgstac](https://github.com/stac-utils/stac-fastapi) and extended with a custom extension to connect it to **`TiTiler`** and a **[Search Viewer](http://localhost:8081/index.html)**. See [docs](http://localhost:8081/docs) for API details. + - **Raster Tiles**: Built with [titiler-pgstac](https://github.com/stac-utils/titiler-pgstac) and [pgstac](https://github.com/stac-utils/pgstac) to enable large scale mosaic based on results of STAC searches queries. See [docs](http://localhost:8082/docs) for API details. + - **OGC Features & Vector Tiles**: Built with [tipg](https://github.com/developmentseed/tipg) to create a lightweight OGC Features and Tiles API with a PostGIS database. See [docs](http://localhost:8083/api.html) for API details. -See [service details](./docs/src/services-details.md) for more information. +See [service details](./docs/src/services-details.md) for more information. *Note: The documentation links referenced require lauching the application with `docker-compose` or another deployment*. @@ -74,19 +83,15 @@ virtualenv .venv source .venv/bin/activate python -m pip install "psycopg[binary,pool]" uvicorn -python -m pip install runtime/eoapi/raster runtime/eoapi/stac runtime/eoapi/vector +python -m pip install runtime/eoapi/{SERVICE} # SERVICE should be one of `raster, vector, stac` export DATABASE_URL=postgresql://username:password@0.0.0.0:5439/postgis # Connect to the database of your choice -# OGC Features/Tiles -.venv/bin/uvicorn eoapi.vector.app:app --port 8000 --reload +.venv/bin/uvicorn eoapi.{SERVICE}.app:app --port 8000 --reload +``` -# Raster -.venv/bin/uvicorn eoapi.raster.app:app --port 8000 --reload +Note: services might have incompatible dependencies which you can resolve by using virtual environement per service -# STAC -.venv/bin/uvicorn eoapi.stac.app:app --port 8000 --reload -``` --- ## Deployment diff --git a/docker-compose.yml b/docker-compose.yml index 8fce26d..6e46e6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -128,7 +128,7 @@ services: database: container_name: eoapi.db - image: ghcr.io/stac-utils/pgstac:v0.7.1 + image: ghcr.io/stac-utils/pgstac:v0.7.10 environment: - POSTGRES_USER=username - POSTGRES_PASSWORD=password diff --git a/runtime/eoapi/raster/eoapi/raster/app.py b/runtime/eoapi/raster/eoapi/raster/app.py index 06ba99b..e8f37a6 100644 --- a/runtime/eoapi/raster/eoapi/raster/app.py +++ b/runtime/eoapi/raster/eoapi/raster/app.py @@ -55,7 +55,9 @@ optional_headers=optional_headers, router_prefix="/mosaic", add_statistics=True, - add_map_viewer=True, + # add /map viewer + add_viewer=True, + # add /mosaic/list endpoint add_mosaic_list=True, ) app.include_router(mosaic.router, tags=["Mosaic"], prefix="/mosaic") @@ -67,12 +69,17 @@ path_dependency=ItemPathParams, optional_headers=optional_headers, router_prefix="/collections/{collection_id}/items/{item_id}", + # add /map viewer + add_viewer=True, ) @stac.router.get("/viewer", response_class=HTMLResponse) def viewer(request: Request, item: pystac.Item = Depends(stac.path_dependency)): - """STAC Viewer.""" + """STAC Viewer + + Simplified version of https://github.com/developmentseed/titiler/blob/main/src/titiler/extensions/titiler/extensions/templates/stac_viewer.html + """ return templates.TemplateResponse( name="stac-viewer.html", context={ diff --git a/runtime/eoapi/raster/pyproject.toml b/runtime/eoapi/raster/pyproject.toml index cbb1ee4..dd325e1 100644 --- a/runtime/eoapi/raster/pyproject.toml +++ b/runtime/eoapi/raster/pyproject.toml @@ -19,7 +19,7 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "titiler.pgstac==0.3.3", + "titiler.pgstac==0.4.1", "starlette-cramjam>=0.3,<0.4", "importlib_resources>=1.1.0;python_version<'3.9'", ] diff --git a/runtime/eoapi/vector/eoapi/vector/app.py b/runtime/eoapi/vector/eoapi/vector/app.py index e7f2fa0..ce31851 100644 --- a/runtime/eoapi/vector/eoapi/vector/app.py +++ b/runtime/eoapi/vector/eoapi/vector/app.py @@ -1,5 +1,6 @@ -"""tipg app.""" +"""eoapi.vector app.""" +from contextlib import asynccontextmanager import jinja2 from eoapi.vector import __version__ as eoapi_vector_version @@ -11,7 +12,7 @@ from tipg.db import close_db_connection, connect_to_db, register_collection_catalog from tipg.errors import DEFAULT_STATUS_CODES, add_exception_handlers from tipg.factory import Endpoints as TiPgEndpoints -from tipg.middleware import CacheControlMiddleware +from tipg.middleware import CacheControlMiddleware, CatalogUpdateMiddleware from tipg.settings import PostgresSettings try: @@ -26,11 +27,39 @@ settings = ApiSettings() postgres_settings = PostgresSettings() + +@asynccontextmanager +async def lifespan(app: FastAPI): + """FastAPI Lifespan.""" + await connect_to_db( + app, + settings=postgres_settings, + # We enable both pgstac and public schemas (pgstac will be used by custom functions) + schemas=["pgstac", "public"], + user_sql_files=list(CUSTOM_SQL_DIRECTORY.glob("*.sql")), # type: ignore + ) + await register_collection_catalog( + app, + # For the Tables' Catalog we only use the `public` schema + schemas=["public"], + # We exclude public functions + exclude_function_schemas=["public"], + # We allow non-spatial tables + spatial=False, + ) + + yield + + # Close the Connection Pool + await close_db_connection(app) + + app = FastAPI( title=settings.name, version=eoapi_vector_version, openapi_url="/api", docs_url="/api.html", + lifespan=lifespan, ) # add eoapi_vector templates and tipg templates @@ -46,7 +75,9 @@ # Register TiPg endpoints. endpoints = TiPgEndpoints( - title=settings.name, templates=templates, with_tiles_viewer=True + title=settings.name, + templates=templates, + with_tiles_viewer=True, ) app.include_router(endpoints.router) @@ -62,34 +93,17 @@ app.add_middleware(CacheControlMiddleware, cachecontrol=settings.cachecontrol) app.add_middleware(CompressionMiddleware) -add_exception_handlers(app, DEFAULT_STATUS_CODES) - -@app.on_event("startup") -async def startup_event() -> None: - """Connect to database on startup.""" - await connect_to_db( - app, - settings=postgres_settings, - # We enable both pgstac and public schemas (pgstac will be used by custom functions) - schemas=["pgstac", "public"], - user_sql_files=list(CUSTOM_SQL_DIRECTORY.glob("*.sql")), # type: ignore - ) - await register_collection_catalog( - app, - # For the Tables' Catalog we only use the `public` schema +if settings.catalog_ttl: + app.add_middleware( + CatalogUpdateMiddleware, + ttl=settings.catalog_ttl, schemas=["public"], - # We exclude public functions exclude_function_schemas=["public"], - # We allow non-spatial tables spatial=False, ) - -@app.on_event("shutdown") -async def shutdown_event() -> None: - """Close database connection.""" - await close_db_connection(app) +add_exception_handlers(app, DEFAULT_STATUS_CODES) @app.get( @@ -114,5 +128,14 @@ async def raw_catalog(request: Request): @app.get("/refresh", include_in_schema=False) async def refresh(request: Request): """Return parsed catalog data for testing.""" - await startup_event() + await register_collection_catalog( + request.app, + # For the Tables' Catalog we only use the `public` schema + schemas=["public"], + # We exclude public functions + exclude_function_schemas=["public"], + # We allow non-spatial tables + spatial=False, + ) + return request.app.state.collection_catalog diff --git a/runtime/eoapi/vector/eoapi/vector/config.py b/runtime/eoapi/vector/eoapi/vector/config.py index 36098bd..7d07e2a 100644 --- a/runtime/eoapi/vector/eoapi/vector/config.py +++ b/runtime/eoapi/vector/eoapi/vector/config.py @@ -12,6 +12,8 @@ class ApiSettings(pydantic.BaseSettings): cachecontrol: str = "public, max-age=3600" debug: bool = False + catalog_ttl: int = 300 + @pydantic.validator("cors_origins") def parse_cors_origin(cls, v): """Parse CORS origins.""" diff --git a/runtime/eoapi/vector/eoapi/vector/templates/header.html b/runtime/eoapi/vector/eoapi/vector/templates/header.html index 37e393a..33b609f 100644 --- a/runtime/eoapi/vector/eoapi/vector/templates/header.html +++ b/runtime/eoapi/vector/eoapi/vector/templates/header.html @@ -20,7 +20,7 @@