Logging improvements

deployment/README.md

@@ -32,5 +32,12 @@ to deploy to your subscription:
 - `servicePrincipalId`
 - `servicePrincipalKey`
 
+If you want to deploy local code changes, instead of the published images, be
+sure to set the following environment variables with the correct public Azure
+Container Registry repo where you published your local images:
 
-__Note:__ Remember to bring down your resources after testing with `terraform destroy`!
+- `ACR_STAC_REPO`
+- `ACR_TILER_REPO`
+- `IMAGE_TAG`
+
+__Note:__ Remember to bring down your resources after testing with `terraform destroy`!

deployment/docker-compose.yml

@@ -6,6 +6,8 @@ services:
       context: .
       dockerfile: Dockerfile
     environment:
+      - ACR_STAC_REPO=${ACR_STAC_REPO:-mcr.microsoft.com/planetary-computer-apis/stac}
+      - ACR_TILER_REPO=${ACR_TILER_REPO:-mcr.microsoft.com/planetary-computer-apis/tiler}
       - IMAGE_TAG
       - GIT_COMMIT
 

deployment/helm/deploy-values.template.yaml

@@ -4,7 +4,7 @@ namespace: "pc"
 stac:
   enabled: true
   image:
-    repository: "mcr.microsoft.com/planetary-computer-apis/stac"
+    repository: "{{ env.ACR_STAC_REPO }}"
     tag: "{{ env.IMAGE_TAG }}"
     pullPolicy: Always
 
@@ -26,7 +26,7 @@ stac:
 tiler:
   enabled: true
   image:
-    repository: "mcr.microsoft.com/planetary-computer-apis/tiler"
+    repository: "{{ env.ACR_TILER_REPO }}"
     tag: "{{ env.IMAGE_TAG }}"
     pullPolicy: Always
 

deployment/helm/published/planetary-computer-tiler/templates/deployment.yaml

@@ -82,11 +82,11 @@ spec:
 
           livenessProbe:
             httpGet:
-              path: /
+              path: "/_mgmt/ping"
               port: http
           readinessProbe:
             httpGet:
-              path: /
+              path: "/_mgmt/ping"
               port: http
           resources:
             {{- toYaml .Values.tiler.deploy.resources | nindent 12 }}

docker-compose.yml

@@ -48,7 +48,7 @@ services:
     volumes:
       - ./pctiler:/opt/src/pctiler
       - ./pctiler/templates:/opt/src/templates
-      - ./pccommon:/opt/src/common
+      - ./pccommon:/opt/src/pccommon
     environment:
       - APP_ROOT_PATH=/data
       - APP_HOST=0.0.0.0

pccommon/logging.py

@@ -13,6 +13,11 @@
 from pccommon.config import CommonConfig
 
 
+class ServiceName:
+    STAC = "stac"
+    TILER = "tiler"
+
+
 # Custom filter that outputs custom_dimensions, only if present
 class OptionalCustomDimensionsFilter(logging.Formatter):
     def __init__(

pccommon/middleware.py

@@ -1,16 +1,21 @@
 import logging
-from typing import Awaitable, Callable
+from typing import Any, Awaitable, Callable, Coroutine
 
 from fastapi import HTTPException, Request, Response
+from fastapi.applications import FastAPI
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.types import Message
 
 from pccommon.logging import request_to_path
-from pccommon.tracing import HTTP_METHOD, HTTP_PATH, HTTP_URL
+from pccommon.tracing import HTTP_METHOD, HTTP_PATH, HTTP_URL, trace_request
 
 logger = logging.getLogger(__name__)
 
 
 async def handle_exceptions(
-    request: Request, call_next: Callable[[Request], Awaitable[Response]]
+    service_name: str,
+    request: Request,
+    call_next: Callable[[Request], Awaitable[Response]],
 ) -> Response:
     try:
         return await call_next(request)
@@ -25,8 +30,43 @@ async def handle_exceptions(
                     HTTP_URL: str(request.url),
                     HTTP_METHOD: str(request.method),
                     HTTP_PATH: request_to_path(request),
-                    "service": "tiler",
+                    "service": service_name,
                 }
             },
         )
         raise
+
+
+class RequestTracingMiddleware(BaseHTTPMiddleware):
+    """Custom middleware to use opencensus request traces
+
+    Middleware implementations that access a Request object directly
+    will cause subsequent middleware or route handlers to hang. See
+
+    https://github.com/tiangolo/fastapi/issues/394
+
+    for more details on this implementation.
+
+    An alternative approach is to use dependencies on the APIRouter, but
+    the stac-fast api implementation makes that difficult without having
+    to override much of the app initialization.
+    """
+
+    def __init__(self, app: FastAPI, service_name: str):
+        super().__init__(app)
+        self.service_name = service_name
+
+    async def set_body(self, request: Request) -> None:
+        receive_ = await request._receive()
+
+        async def receive() -> Message:
+            return receive_
+
+        request._receive = receive
+
+    async def dispatch(
+        self, request: Request, call_next: Callable[[Request], Awaitable[Response]]
+    ) -> Response:
+        await self.set_body(request)
+        response = await trace_request(self.service_name, request, call_next)
+        return response

pccommon/tests/data/cql.py

@@ -0,0 +1,67 @@
+cql = {
+    "and": [
+        {"lte": [{"property": "eo:cloud_cover"}, "10"]},
+        {"gte": [{"property": "datetime"}, "2021-04-08T04:39:23Z"]},
+        {
+            "or": [
+                {"eq": [{"property": "collection"}, "landsat"]},
+                {"lte": [{"property": "gsd"}, "10"]},
+            ]
+        },
+        {"lte": [{"property": "id"}, "l8_12345"]},
+    ]
+}
+
+cql_multi = {
+    "and": [
+        {"lte": [{"property": "eo:cloud_cover"}, "10"]},
+        {"gte": [{"property": "datetime"}, "2021-04-08T04:39:23Z"]},
+        {
+            "or": [
+                {"eq": [{"property": "collection"}, ["landsat", "sentinel"]]},
+                {"lte": [{"property": "gsd"}, "10"]},
+            ]
+        },
+    ]
+}
+
+cql2 = {
+    "op": "or",
+    "args": [
+        {"op": ">=", "args": [{"property": "sentinel:data_coverage"}, 50]},
+        {"op": "=", "args": [{"property": "collection"}, "landsat"]},
+        {
+            "op": "and",
+            "args": [
+                {"op": "isNull", "args": {"property": "sentinel:data_coverage"}},
+                {"op": "isNull", "args": {"property": "landsat:coverage_percent"}},
+            ],
+        },
+    ],
+}
+
+cql2_nested = {
+    "op": "or",
+    "args": [
+        {"op": ">=", "args": [{"property": "sentinel:data_coverage"}, 50]},
+        {
+            "op": "and",
+            "args": [
+                {"op": "isNull", "args": {"property": "sentinel:data_coverage"}},
+                {
+                    "op": "=",
+                    "args": [
+                        {"property": "collection"},
+                        ["landsat", "sentinel"],
+                    ],
+                },
+                {"op": "in", "args": [{"property": "id"}, ["l8_12345", "s2_12345"]]},
+            ],
+        },
+    ],
+}
+
+cql2_no_collection = {
+    "op": "or",
+    "args": [{"op": ">=", "args": [{"property": "sentinel:data_coverage"}, 50]}],
+}

pccommon/tests/test_tracing.py

@@ -0,0 +1,39 @@
+import unittest
+
+from ..tracing import _parse_cqljson
+from .data.cql import cql, cql2, cql2_nested, cql2_no_collection, cql_multi
+
+
+class TestSearchTracing(unittest.TestCase):
+    def test_tracing(self) -> None:
+        pass
+
+    def test_cql_collection_parsing(self) -> None:
+        collection_id, item_id = _parse_cqljson(cql)
+
+        self.assertEqual(collection_id, "landsat")
+        self.assertEqual(item_id, "l8_12345")
+
+    def test_cql_multi_collection_parsing(self) -> None:
+        collection_id, item_id = _parse_cqljson(cql_multi)
+
+        self.assertEqual(collection_id, "landsat,sentinel")
+        self.assertEqual(item_id, None)
+
+    def test_cql2_collection_parsing(self) -> None:
+        collection_id, item_id = _parse_cqljson(cql2)
+
+        self.assertEqual(collection_id, "landsat")
+        self.assertEqual(item_id, None)
+
+    def test_cql2_nested_multi_collection_parsing(self) -> None:
+        collection_id, item_id = _parse_cqljson(cql2_nested)
+
+        self.assertEqual(collection_id, "landsat,sentinel")
+        self.assertEqual(item_id, "l8_12345,s2_12345")
+
+    def test_cql2_no_collection(self) -> None:
+        collection_id, item_id = _parse_cqljson(cql2_no_collection)
+
+        self.assertEqual(collection_id, None)
+        self.assertEqual(item_id, None)