Bump minimal supported Python version to py3.11

.github/workflows/deploy.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.11
       - uses: actions/cache@v4
         with:
           key: ${{ github.ref }}

README.md

@@ -34,7 +34,7 @@ See the [docs](https://cesnet.github.io/dp3/howto/get-started/) for more details
 
 ### Installing for application development
 
-Pre-requisites: Python 3.9 or higher, `pip` (with `virtualenv` installed), `git`, `Docker` and `Docker Compose`.
+Pre-requisites: Python 3.11 or higher, `pip` (with `virtualenv` installed), `git`, `Docker` and `Docker Compose`.
 
 Create a virtualenv and install the DP³ platform using:
 
@@ -117,7 +117,7 @@ You are now ready to start developing your application!
 
 ## Installing for platform development
 
-Pre-requisites: Python 3.9 or higher, `pip` (with `virtualenv` installed), `git`, `Docker` and `Docker Compose`.
+Pre-requisites: Python 3.11 or higher, `pip` (with `virtualenv` installed), `git`, `Docker` and `Docker Compose`.
 
 Pull the repository and install using:
 

docker/python/Dockerfile

@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile:1
 
 # Base interpreter with installed requirements
-FROM python:3.9-slim AS base
+FROM python:3.11-slim AS base
 RUN apt-get update; apt-get install -y  \
     gcc \
     git

docs/howto/develop-dp3.md

@@ -16,7 +16,7 @@ You will end up with:
 
 For platform development, you need:
 
-- Python 3.9 or higher
+- Python 3.11 or higher
 - `pip`
 - `git`
 - Docker

docs/howto/get-started.md

@@ -15,7 +15,7 @@ You will end up with:
 
 For local application development, you need:
 
-- Python 3.9 or higher
+- Python 3.11 or higher
 - `pip`
 - `git`
 - Docker

dp3/api/internal/config.py

@@ -10,12 +10,15 @@
 
 from dp3.api.internal.dp_logger import DPLogger
 from dp3.common.config import ModelSpec, read_config_dir
+from dp3.common.utils import suppress_dependency_loggers
 from dp3.database.database import EntityDatabase
 from dp3.history_management.telemetry import TelemetryReader
 from dp3.task_processing.task_queue import TaskQueueWriter
 
 DATAPOINTS_INGESTION_URL_PATH = "/datapoints"
 
+suppress_dependency_loggers()
+
 
 class ConfigEnv(BaseModel):
     """Configuration environment variables container"""
@@ -44,7 +47,7 @@ def validate(cls, v):
 
 try:
     # Validate and parse environmental variables
-    conf_env = ConfigEnv.parse_obj(os.environ)
+    conf_env = ConfigEnv.model_validate(os.environ)
 except ValidationError as e:
     config_error = any("CONF_DIR" in x["loc"] and len(x["loc"]) > 1 for x in e.errors())
     env_error = any(len(x["loc"]) == 1 for x in e.errors())

dp3/api/internal/entity_response_models.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Annotated, Any, Optional, Union
+from typing import Annotated, Any
 
 from pydantic import BaseModel, Field, NonNegativeInt, PlainSerializer
 
@@ -25,11 +25,11 @@ class EntityState(BaseModel):
 JsonVal = Annotated[Any, PlainSerializer(to_json_friendly, when_used="json")]
 
 LinkVal = dict[str, JsonVal]
-PlainVal = Union[LinkVal, JsonVal]
+PlainVal = LinkVal | JsonVal
 MultiVal = list[PlainVal]
 HistoryVal = list[dict[str, PlainVal]]
 
-Dp3Val = Union[HistoryVal, MultiVal, PlainVal]
+Dp3Val = HistoryVal | MultiVal | PlainVal
 
 EntityEidMasterRecord = dict[str, Dp3Val]
 
@@ -45,7 +45,7 @@ class EntityEidList(BaseModel):
     Data does not include history of observations attributes and timeseries.
     """
 
-    time_created: Optional[datetime] = None
+    time_created: datetime | None = None
     count: int
     data: EntityEidSnapshots
 

dp3/api/internal/models.py

@@ -1,4 +1,6 @@
-from typing import Annotated, Any, Literal, Optional, Union
+from functools import reduce
+from operator import or_
+from typing import Annotated, Any, Literal
 
 from pydantic import BaseModel, Field, TypeAdapter, create_model, model_validator
 
@@ -26,10 +28,10 @@ class DataPoint(BaseModel):
     id: Any
     attr: str
     v: Any
-    t1: Optional[AwareDatetime] = None
-    t2: Optional[T2Datetime] = Field(None, validate_default=True)
+    t1: AwareDatetime | None = None
+    t2: T2Datetime | None = Field(None, validate_default=True)
     c: Annotated[float, Field(ge=0.0, le=1.0)] = 1.0
-    src: Optional[str] = None
+    src: str | None = None
 
     @model_validator(mode="after")
     def validate_against_attribute(self):
@@ -43,14 +45,14 @@ def validate_against_attribute(self):
 
 
 class EntityId(BaseModel):
-    """Dummy model for entity id
+    """Common interface for validated entity identifiers.
 
     Attributes:
         type: Entity type
         id: Entity ID
     """
 
-    type: Literal["entity_type"]
+    type: str
     id: Any
 
 
@@ -60,11 +62,14 @@ class EntityId(BaseModel):
     entity_id_models.append(
         create_model(
             f"EntityId{{{entity_type}}}",
-            __base__=BaseModel,
+            __base__=EntityId,
             type=(Literal[entity_type], Field(..., alias="etype")),
             id=(dtype, Field(..., alias="eid")),
         )
     )
 
-EntityId = Annotated[Union[tuple(entity_id_models)], Field(discriminator="type")]  # noqa: F811
-EntityIdAdapter = TypeAdapter(EntityId)
+if not entity_id_models:
+    raise RuntimeError("At least one entity type must be configured to run the API.")
+
+EntityIdType = Annotated[reduce(or_, entity_id_models), Field(discriminator="type")]
+EntityIdAdapter = TypeAdapter(EntityIdType)

dp3/api/routers/entity.py

@@ -1,5 +1,5 @@
-from datetime import datetime
-from typing import Annotated, Any, Optional
+from datetime import UTC, datetime
+from typing import Annotated, Any, cast
 
 from fastapi import APIRouter, Depends, HTTPException, Request
 from pydantic import Json, NonNegativeInt, ValidationError
@@ -22,7 +22,7 @@
 from dp3.common.attrspec import AttrType
 from dp3.common.datapoint import to_json_friendly
 from dp3.common.task import DataPointTask, task_context
-from dp3.common.types import UTC, AwareDatetime
+from dp3.common.types import AwareDatetime
 from dp3.database.database import DatabaseError
 
 
@@ -33,23 +33,23 @@ async def check_etype(etype: str):
     return etype
 
 
-async def parse_eid(etype: str, eid: str):
+async def parse_eid(etype: str, eid: str) -> EntityId:
     """Middleware to parse EID"""
     try:
-        return EntityIdAdapter.validate_python({"etype": etype, "eid": eid})
+        return cast(EntityId, EntityIdAdapter.validate_python({"etype": etype, "eid": eid}))
     except ValidationError as e:
         raise RequestValidationError(["path", "eid"], e.errors()[0]["msg"]) from e
 
 
 ParsedEid = Annotated[EntityId, Depends(parse_eid)]
 
 
-def _parse_optional_eid(etype: str, eid: Optional[str]) -> Any:
+def _parse_optional_eid(etype: str, eid: str | None) -> Any:
     """Parse optional entity id query parameter for entity-scoped endpoints."""
     if eid is None:
         return None
     try:
-        return EntityIdAdapter.validate_python({"etype": etype, "eid": eid}).id
+        return cast(EntityId, EntityIdAdapter.validate_python({"etype": etype, "eid": eid})).id
     except ValidationError as e:
         raise RequestValidationError(["query", "eid"], e.errors()[0]["msg"]) from e
 
@@ -72,7 +72,7 @@ def _raw_datapoint_to_response(raw_datapoint: dict[str, Any]) -> dict[str, Any]:
 
 
 def get_eid_master_record_handler(
-    e: EntityId, date_from: Optional[AwareDatetime] = None, date_to: Optional[AwareDatetime] = None
+    e: EntityId, date_from: AwareDatetime | None = None, date_to: AwareDatetime | None = None
 ):
     """Handler for getting master record of EID"""
     # TODO: This is probably not the most efficient way. Maybe gather only
@@ -97,8 +97,8 @@ def get_eid_master_record_handler(
 
 def get_eid_snapshots_handler(
     e: EntityId,
-    date_from: Optional[AwareDatetime] = None,
-    date_to: Optional[AwareDatetime] = None,
+    date_from: AwareDatetime | None = None,
+    date_to: AwareDatetime | None = None,
     skip: int = 0,
     limit: int = 0,
 ) -> list[dict[str, Any]]:
@@ -271,9 +271,9 @@ async def count_entity_type_eids(
 )
 async def get_entity_type_raw_datapoints(
     etype: str,
-    eid: Optional[str] = None,
-    attr: Optional[str] = None,
-    src: Optional[str] = None,
+    eid: str | None = None,
+    attr: str | None = None,
+    src: str | None = None,
     skip: NonNegativeInt = 0,
     limit: NonNegativeInt = 20,
 ) -> EntityRawDataPage:
@@ -305,7 +305,7 @@ async def get_entity_type_raw_datapoints(
 
 @router.get("/{etype}/{eid}")
 async def get_eid_data(
-    e: ParsedEid, date_from: Optional[AwareDatetime] = None, date_to: Optional[AwareDatetime] = None
+    e: ParsedEid, date_from: AwareDatetime | None = None, date_to: AwareDatetime | None = None
 ) -> EntityEidData:
     """Get data of the entity identified by `etype` and `eid`.
 
@@ -325,7 +325,7 @@ async def get_eid_data(
 
 @router.get("/{etype}/{eid}/master")
 async def get_eid_master_record(
-    e: ParsedEid, date_from: Optional[AwareDatetime] = None, date_to: Optional[AwareDatetime] = None
+    e: ParsedEid, date_from: AwareDatetime | None = None, date_to: AwareDatetime | None = None
 ) -> EntityEidMasterRecord:
     """Get the master record of the entity identified by `etype` and `eid`."""
     return get_eid_master_record_handler(e, date_from, date_to)
@@ -334,8 +334,8 @@ async def get_eid_master_record(
 @router.get("/{etype}/{eid}/snapshots")
 async def get_eid_snapshots(
     e: ParsedEid,
-    date_from: Optional[AwareDatetime] = None,
-    date_to: Optional[AwareDatetime] = None,
+    date_from: AwareDatetime | None = None,
+    date_to: AwareDatetime | None = None,
     skip: NonNegativeInt = 0,
     limit: NonNegativeInt = 0,
 ) -> EntityEidSnapshots:
@@ -351,8 +351,8 @@ async def get_eid_snapshots(
 async def get_eid_attr_value(
     e: ParsedEid,
     attr: str,
-    date_from: Optional[AwareDatetime] = None,
-    date_to: Optional[AwareDatetime] = None,
+    date_from: AwareDatetime | None = None,
+    date_to: AwareDatetime | None = None,
 ) -> EntityEidAttrValueOrHistory:
     """Get attribute value
 
@@ -373,9 +373,7 @@ async def get_eid_attr_value(
 
 
 @router.post("/{etype}/{eid}/set/{attr}")
-async def set_eid_attr_value(
-    etype: str, eid: str, attr: str, body: EntityEidAttrValue, request: Request
-) -> SuccessResponse:
+async def set_eid_attr_value(etype: str, eid: str, attr: str, request: Request) -> SuccessResponse:
     """Set current value of attribute
 
     Internally just creates datapoint for specified attribute and value.
@@ -386,6 +384,11 @@ async def set_eid_attr_value(
     if attr not in MODEL_SPEC.attribs(etype):
         raise RequestValidationError(["path", "attr"], f"Attribute '{attr}' doesn't exist")
 
+    try:
+        body = EntityEidAttrValue.model_validate(await request.json())
+    except ValueError as e:
+        raise RequestValidationError(["body"], str(e)) from e
+
     # Construct datapoint
     try:
         dp = DataPoint(
@@ -396,7 +399,7 @@ async def set_eid_attr_value(
             t1=datetime.now(UTC),
             src=f"{request.client.host} via API",
         )
-        dp3_dp = api_to_dp3_datapoint(dp.dict())
+        dp3_dp = api_to_dp3_datapoint(dp.model_dump())
     except ValidationError as e:
         raise RequestValidationError(["body", "value"], e.errors()[0]["msg"]) from e
 

dp3/api/routers/telemetry.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Literal, Optional
+from typing import Literal
 
 import requests
 from fastapi import APIRouter, HTTPException
@@ -41,9 +41,9 @@ async def get_snapshot_summary() -> dict:
 
 @router.get("/metadata")
 async def get_metadata(
-    module: Optional[str] = None,
-    date_from: Optional[AwareDatetime] = None,
-    date_to: Optional[AwareDatetime] = None,
+    module: str | None = None,
+    date_from: AwareDatetime | None = None,
+    date_to: AwareDatetime | None = None,
     skip: NonNegativeInt = 0,
     limit: NonNegativeInt = 0,
     sort: Literal["newest", "oldest"] = "newest",

dp3/bin/check.py

@@ -218,15 +218,15 @@ def main(args):
         unique_sources = []
         source_paths_and_errors = []
 
-        for path, source, err in zip(paths, sources, errors):
+        for path, source, err in zip(paths, sources, errors, strict=False):
             if source in unique_sources:
                 i = unique_sources.index(source)
                 source_paths_and_errors[i].add((path, err))
             else:
                 unique_sources.append(source)
                 source_paths_and_errors.append({(path, err)})
 
-        for source, paths_and_errors in zip(unique_sources, source_paths_and_errors):
+        for source, paths_and_errors in zip(unique_sources, source_paths_and_errors, strict=False):
             for path, err in paths_and_errors:
                 print(" -> ".join(path))
                 print("  ", err)

dp3/bin/schema_update.py

@@ -7,6 +7,7 @@
 import logging
 
 from dp3.common.config import ModelSpec, read_config_dir
+from dp3.common.utils import suppress_dependency_loggers
 from dp3.database.database import EntityDatabase
 
 
@@ -41,6 +42,7 @@ def main(args):
     LOGFORMAT = "%(asctime)-15s,%(name)s,[%(levelname)s] %(message)s"
     LOGDATEFORMAT = "%Y-%m-%dT%H:%M:%S"
     logging.basicConfig(level=logging.DEBUG, format=LOGFORMAT, datefmt=LOGDATEFORMAT)
+    suppress_dependency_loggers()
     log = logging.getLogger("SchemaUpdate")
 
     # Connect to database