From da779cd20d75c413adaccbb3dc5b80aa9e56c33b Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Sat, 15 Nov 2025 18:25:20 +0100 Subject: [PATCH 01/15] refactor ifaddr to optional dependency --- doc | 2 +- hololinked/utils.py | 73 ++++++++++++++++++++++----------------------- pyproject.toml | 7 ++++- 3 files changed, 42 insertions(+), 40 deletions(-) diff --git a/doc b/doc index e8ec16a0..d4e965b5 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit e8ec16a0f5ff9a74c38a8a13fab26baf615e73cb +Subproject commit d4e965b5ad5b8c0b88f807d031b72e76acf9cde9 diff --git a/hololinked/utils.py b/hololinked/utils.py index 0f24b892..751625d3 100644 --- a/hololinked/utils.py +++ b/hololinked/utils.py @@ -12,7 +12,6 @@ from functools import wraps from inspect import Parameter, signature -import ifaddr from pydantic import BaseModel, ConfigDict, Field, RootModel, create_model @@ -33,6 +32,8 @@ def get_IP_from_interface(interface_name: str = "Ethernet", adapter_name=None) - str: IP address of the interface """ + import ifaddr + adapters = ifaddr.get_adapters(include_unconfigured=True) for adapter in adapters: if not adapter_name: @@ -121,6 +122,19 @@ def get_default_logger( return logger +def get_current_async_loop(): + """ + get or automatically create an asnyc loop for the current thread. + """ + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + # set_event_loop_policy() - why not? + asyncio.set_event_loop(loop) + return loop + + def run_coro_sync(coro: typing.Coroutine): """ run coroutine synchronously @@ -139,9 +153,7 @@ def run_coro_sync(coro: typing.Coroutine): return eventloop.run_until_complete(coro) -def run_callable_somehow( - method: typing.Union[typing.Callable, typing.Coroutine], -) -> typing.Any: +def run_callable_somehow(method: typing.Union[typing.Callable, typing.Coroutine]) -> typing.Any: """ run method if synchronous, or when async, either schedule a coroutine or run it until its complete """ @@ -198,9 +210,23 @@ def print_pending_tasks_in_current_loop(): print(f"Task: {task}, Status: {task._state}") -def get_signature( - callable: typing.Callable, -) -> typing.Tuple[typing.List[str], typing.List[type]]: +def set_global_event_loop_policy(): + if sys.platform.lower().startswith("win"): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + + from .config import global_config + + if global_config.USE_UVLOOP and sys.platform.lower() in [ + "linux", + "darwin", + "linux2", + ]: + import uvloop + + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + + +def get_signature(callable: typing.Callable) -> typing.Tuple[typing.List[str], typing.List[type]]: """ Retrieve the names and types of arguments based on annotations for the given callable. @@ -336,19 +362,6 @@ def get_a_filename_from_instance(thing: type, extension: str = "json") -> str: return filename -def get_current_async_loop(): - """ - get or automatically create an asnyc loop for the current thread. - """ - try: - loop = asyncio.get_event_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - # set_event_loop_policy() - why not? - asyncio.set_event_loop(loop) - return loop - - class SerializableDataclass: """ Presents uniform serialization for serializers using getstate and setstate and json @@ -634,22 +647,6 @@ def wrapper(*args, **kwargs): return wrapper -def set_global_event_loop_policy(): - if sys.platform.lower().startswith("win"): - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - - from .config import global_config - - if global_config.USE_UVLOOP and sys.platform.lower() in [ - "linux", - "darwin", - "linux2", - ]: - import uvloop - - asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) - - __all__ = [ get_IP_from_interface.__name__, format_exception_as_json.__name__, @@ -658,14 +655,14 @@ def set_global_event_loop_policy(): run_coro_sync.__name__, run_callable_somehow.__name__, complete_pending_tasks_in_current_loop.__name__, + get_current_async_loop.__name__, + set_global_event_loop_policy.__name__, get_signature.__name__, isclassmethod.__name__, issubklass.__name__, - get_current_async_loop.__name__, get_input_model_from_signature.__name__, pydantic_validate_args_kwargs.__name__, get_return_type_from_signature.__name__, getattr_without_descriptor_read.__name__, forkable.__name__, - set_global_event_loop_policy.__name__, ] diff --git a/pyproject.toml b/pyproject.toml index eb0c6297..f6918563 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,6 @@ classifiers = [ keywords = ["data-acquisition", "zmq-rpc", "SCADA", "IoT", "Web of Things", "remote data logging"] dependencies = [ "argon2-cffi>=23.1.0", - "ifaddr>=0.2.0,<0.3", "msgspec>=0.18.6", "pyzmq>=25.1.0,<26.2", "SQLAlchemy>2.0.21", @@ -48,6 +47,12 @@ dependencies = [ "structlog>=25.5.0", ] +[project.optional-dependencies] +dev = [ + "ifaddr>=0.2.0,<0.3", +] + + [project.urls] Documentation = "https://hololinked.readthedocs.io/en/latest/index.html" Repository = "https://github.com/hololinked-dev/hololinked" From 70d56c0ee8e1cb9449a11348dc1129684701700d Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Sat, 15 Nov 2025 18:48:09 +0100 Subject: [PATCH 02/15] move exceptions to core logic as all of them belong to core --- hololinked/client/zmq/consumed_interactions.py | 3 +-- hololinked/core/actions.py | 2 +- hololinked/{ => core}/exceptions.py | 0 hololinked/core/property.py | 2 +- hololinked/core/state_machine.py | 2 +- hololinked/core/thing.py | 2 +- hololinked/core/zmq/brokers.py | 3 +-- hololinked/core/zmq/rpc_server.py | 3 +-- 8 files changed, 7 insertions(+), 10 deletions(-) rename hololinked/{ => core}/exceptions.py (100%) diff --git a/hololinked/client/zmq/consumed_interactions.py b/hololinked/client/zmq/consumed_interactions.py index eb9b39d7..dba824d3 100644 --- a/hololinked/client/zmq/consumed_interactions.py +++ b/hololinked/client/zmq/consumed_interactions.py @@ -26,8 +26,7 @@ raise_local_exception, SSE, ) -from ..exceptions import ReplyNotArrivedError -from ...exceptions import BreakLoop +from ..exceptions import ReplyNotArrivedError, BreakLoop __error_message_types__ = [TIMEOUT, ERROR, INVALID_MESSAGE] diff --git a/hololinked/core/actions.py b/hololinked/core/actions.py index df90abb5..35e30cc6 100644 --- a/hololinked/core/actions.py +++ b/hololinked/core/actions.py @@ -16,7 +16,7 @@ issubklass, isclassmethod, ) -from ..exceptions import StateMachineError +from .exceptions import StateMachineError from ..schema_validators.validators import JSONSchemaValidator, PydanticSchemaValidator from .dataklasses import ActionInfoValidator diff --git a/hololinked/exceptions.py b/hololinked/core/exceptions.py similarity index 100% rename from hololinked/exceptions.py rename to hololinked/core/exceptions.py diff --git a/hololinked/core/property.py b/hololinked/core/property.py index 4ef80ccf..d8f190b2 100644 --- a/hololinked/core/property.py +++ b/hololinked/core/property.py @@ -1,12 +1,12 @@ import typing from enum import Enum -from ..exceptions import StateMachineError from ..param.parameterized import Parameter, Parameterized, ParameterizedMetaclass from ..schema_validators import JSONSchemaValidator from ..utils import issubklass from .dataklasses import RemoteResourceInfoValidator from .events import Event, EventDispatcher # noqa: F401 +from .exceptions import StateMachineError class Property(Parameter): diff --git a/hololinked/core/state_machine.py b/hololinked/core/state_machine.py index 40037482..91249ceb 100644 --- a/hololinked/core/state_machine.py +++ b/hololinked/core/state_machine.py @@ -2,9 +2,9 @@ from enum import Enum, EnumMeta, StrEnum from types import FunctionType, MethodType -from ..exceptions import StateMachineError from ..param import edit_constant from .actions import Action +from .exceptions import StateMachineError from .meta import ThingMeta from .properties import Boolean, ClassSelector, TypedDict from .property import Property diff --git a/hololinked/core/thing.py b/hololinked/core/thing.py index 0921540f..6dccec88 100644 --- a/hololinked/core/thing.py +++ b/hololinked/core/thing.py @@ -6,7 +6,7 @@ from ..constants import ZMQ_TRANSPORTS from ..utils import * # noqa: F403 -from ..exceptions import * # noqa: F403 +from .exceptions import * # noqa: F403 from ..serializers import Serializers, BaseSerializer, JSONSerializer from .properties import String, ClassSelector from .property import Property diff --git a/hololinked/core/zmq/brokers.py b/hololinked/core/zmq/brokers.py index 688516a9..7f28a09c 100644 --- a/hololinked/core/zmq/brokers.py +++ b/hololinked/core/zmq/brokers.py @@ -13,9 +13,8 @@ from ...config import global_config from ...constants import ZMQ_EVENT_MAP, ZMQ_TRANSPORTS, get_socket_type_name -from ...exceptions import BreakLoop from ...serializers.serializers import Serializers -from ...utils import format_exception_as_json, get_current_async_loop, run_callable_somehow, uuid_hex +from ..exceptions import BreakLoop from .message import ( ERROR, EXIT, diff --git a/hololinked/core/zmq/rpc_server.py b/hololinked/core/zmq/rpc_server.py index 0a47453c..fdac7091 100644 --- a/hololinked/core/zmq/rpc_server.py +++ b/hololinked/core/zmq/rpc_server.py @@ -11,9 +11,7 @@ import zmq import zmq.asyncio -from ...config import global_config from ...constants import ZMQ_TRANSPORTS, Operations -from ...exceptions import BreakInnerLoop, BreakLoop from ...serializers import BaseSerializer, Serializers from ...utils import ( format_exception_as_json, @@ -22,6 +20,7 @@ set_global_event_loop_policy, ) from ..actions import BoundAction # noqa: F401 +from ..exceptions import BreakInnerLoop, BreakLoop from ..logger import LogHistoryHandler from ..properties import TypedList from ..property import Property # noqa: F401 From a826e2618a363e5118c31ef35574ca528678a860 Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Sat, 15 Nov 2025 21:14:06 +0100 Subject: [PATCH 03/15] add ruff isort settings --- pyproject.toml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f6918563..a0ef812b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "hololinked" -version = "0.3.7" +version = "0.3.8" authors = [ {name = "Vignesh Vaidyanathan", email = "info@hololinked.dev"}, ] @@ -142,4 +142,11 @@ exclude_dirs = [ "examples", "licenses", "tests" -] \ No newline at end of file +] + +[tool.ruff.lint] +extend-select = ["I"] + +[tool.ruff.lint.isort] +lines-between-types = 1 +lines-after-imports = 2 From d9e2b94ac1d96147578daa3f814907ce15d50694 Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Sat, 15 Nov 2025 21:14:19 +0100 Subject: [PATCH 04/15] do isort top level --- hololinked/__init__.py | 4 ++-- hololinked/config.py | 7 ++++--- hololinked/constants.py | 13 ++++--------- hololinked/logger.py | 13 ++++++++----- hololinked/utils.py | 1 + 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/hololinked/__init__.py b/hololinked/__init__.py index 31b44e41..59c4baf1 100644 --- a/hololinked/__init__.py +++ b/hololinked/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.3.7" +__version__ = "0.3.8" -from .config import global_config # noqa: F401 +from .config import global_config # noqa import hololinked.core # noqa: F401 diff --git a/hololinked/config.py b/hololinked/config.py index 862281e8..3b70498f 100644 --- a/hololinked/config.py +++ b/hololinked/config.py @@ -24,13 +24,14 @@ SOFTWARE. """ -import tempfile +import json +import logging import os +import tempfile import typing import warnings + import zmq.asyncio -import json -import logging class Configuration: diff --git a/hololinked/constants.py b/hololinked/constants.py index d9b09085..61236a00 100644 --- a/hololinked/constants.py +++ b/hololinked/constants.py @@ -1,7 +1,9 @@ -import zmq import logging import typing -from enum import StrEnum, IntEnum + +from enum import IntEnum, StrEnum + +import zmq # types @@ -80,13 +82,6 @@ class ZMQ_TRANSPORTS(StrEnum): INPROC = "INPROC" -class HTTPServerTypes(StrEnum): - "types of HTTP server" - - SYSTEM_HOST = "SYSTEM_HOST" - THING_SERVER = "THING_SERVER" - - class ZMQSocketType(IntEnum): PAIR = zmq.PAIR PUB = zmq.PUB diff --git a/hololinked/logger.py b/hololinked/logger.py index 1d816688..fb9718e5 100644 --- a/hololinked/logger.py +++ b/hololinked/logger.py @@ -26,14 +26,17 @@ def normalize_component_name(_, __, event_dict: dict[str, Any]) -> dict[str, Any def setup_logging(log_level: int = logging.INFO, colored_logs: bool = False, log_file: str = None) -> None: """ - Setup structured logging for hololinked library - Not a flexible setup, override the entire function if you want a different logging configuration or monkey patch - this method. + Setup structured logging using structlog. Not a flexible setup, override the entire function + if you want a different logging configuration or monkey patch this method. Parameters ---------- - log_level : int - The logging level to use + log_level: int + logging level to use + colored_logs: bool + whether to use colored logs in console, usually harder to pick it up in fluentd + log_file: str + optional log file to log into """ logging.basicConfig(stream=sys.stdout, format="%(message)s", level=log_level) global default_label_formatter diff --git a/hololinked/utils.py b/hololinked/utils.py index 751625d3..c6d483d9 100644 --- a/hololinked/utils.py +++ b/hololinked/utils.py @@ -7,6 +7,7 @@ import traceback import types import typing + from collections import OrderedDict from dataclasses import asdict from functools import wraps From 1fdd788f06f46aba73401e5662e18f65eb7be9f1 Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Sat, 15 Nov 2025 21:25:29 +0100 Subject: [PATCH 05/15] schema validators isort & move fastjsonschema to optional dependency --- hololinked/schema_validators/__init__.py | 4 ++-- hololinked/schema_validators/json_schema.py | 1 + hololinked/schema_validators/validators.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hololinked/schema_validators/__init__.py b/hololinked/schema_validators/__init__.py index f0691d9d..307db4b4 100644 --- a/hololinked/schema_validators/__init__.py +++ b/hololinked/schema_validators/__init__.py @@ -1,2 +1,2 @@ -from .validators import BaseSchemaValidator, JSONSchemaValidator, PydanticSchemaValidator # noqa: F401 -from .json_schema import JSONSchema # noqa: F401 +from .validators import BaseSchemaValidator, JSONSchemaValidator, PydanticSchemaValidator # noqa +from .json_schema import JSONSchema # noqa diff --git a/hololinked/schema_validators/json_schema.py b/hololinked/schema_validators/json_schema.py index 0a5c68ac..acc8f0ad 100644 --- a/hololinked/schema_validators/json_schema.py +++ b/hololinked/schema_validators/json_schema.py @@ -1,4 +1,5 @@ import typing + from ..constants import JSON diff --git a/hololinked/schema_validators/validators.py b/hololinked/schema_validators/validators.py index 4eb57d42..ce7947de 100644 --- a/hololinked/schema_validators/validators.py +++ b/hololinked/schema_validators/validators.py @@ -1,8 +1,9 @@ import jsonschema + from pydantic import BaseModel -from ..utils import pydantic_validate_args_kwargs, json_schema_merge_args_to_kwargs from ..constants import JSON +from ..utils import json_schema_merge_args_to_kwargs, pydantic_validate_args_kwargs class BaseSchemaValidator: # type definition From fd69412c0f83e57b27f0bd1b08bb08ae5543dcf3 Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Sat, 15 Nov 2025 21:43:07 +0100 Subject: [PATCH 06/15] order import core --- hololinked/core/__init__.py | 8 +++--- hololinked/core/actions.py | 21 ++++++++------- hololinked/core/dataklasses.py | 5 ++-- hololinked/core/events.py | 6 +++-- hololinked/core/logger.py | 15 ++++++----- hololinked/core/meta.py | 19 ++++++------- hololinked/core/property.py | 46 +++++++++++++++----------------- hololinked/core/state_machine.py | 1 + hololinked/core/thing.py | 34 +++++++++++------------ pyproject.toml | 4 +-- 10 files changed, 80 insertions(+), 79 deletions(-) diff --git a/hololinked/core/__init__.py b/hololinked/core/__init__.py index a19a1090..aade0a5f 100644 --- a/hololinked/core/__init__.py +++ b/hololinked/core/__init__.py @@ -1,6 +1,6 @@ # Order of import is reflected in this file to avoid circular imports -from .thing import * # noqa: F403 -from .events import * # noqa: F403 -from .actions import * # noqa: F403 -from .property import * # noqa: F403 +from .thing import * # noqa +from .events import * # noqa +from .actions import * # noqa +from .property import * # noqa from .meta import ThingMeta as ThingMeta diff --git a/hololinked/core/actions.py b/hololinked/core/actions.py index 35e30cc6..9c2a91af 100644 --- a/hololinked/core/actions.py +++ b/hololinked/core/actions.py @@ -1,24 +1,27 @@ import typing import warnings -import jsonschema + from enum import Enum +from inspect import getfullargspec, iscoroutinefunction from types import FunctionType, MethodType -from inspect import iscoroutinefunction, getfullargspec + +import jsonschema + from pydantic import BaseModel, RootModel -from ..param.parameterized import ParameterizedFunction -from ..constants import JSON from ..config import global_config +from ..constants import JSON +from ..param.parameterized import ParameterizedFunction +from ..schema_validators.validators import JSONSchemaValidator, PydanticSchemaValidator from ..utils import ( + get_input_model_from_signature, get_return_type_from_signature, has_async_def, - get_input_model_from_signature, - issubklass, isclassmethod, + issubklass, ) -from .exceptions import StateMachineError -from ..schema_validators.validators import JSONSchemaValidator, PydanticSchemaValidator from .dataklasses import ActionInfoValidator +from .exceptions import StateMachineError class Action: @@ -128,7 +131,7 @@ def __init__(self, obj: FunctionType, descriptor: Action, owner_inst, owner) -> def __post_init__(self): # never called, neither possible to call, only type hinting - from .thing import ThingMeta, Thing + from .thing import Thing, ThingMeta # owner class and instance self.owner: ThingMeta diff --git a/hololinked/core/dataklasses.py b/hololinked/core/dataklasses.py index b3196256..62e113a4 100644 --- a/hololinked/core/dataklasses.py +++ b/hololinked/core/dataklasses.py @@ -4,14 +4,15 @@ """ import typing + from enum import Enum from types import FunctionType, MethodType from pydantic import BaseModel, RootModel -from ..param.parameters import String, Boolean, Tuple, ClassSelector, Parameter -from ..param.parameterized import ParameterizedMetaclass from ..constants import USE_OBJECT_NAME +from ..param.parameterized import ParameterizedMetaclass +from ..param.parameters import Boolean, ClassSelector, Parameter, String, Tuple from ..schema_validators import BaseSchemaValidator from ..utils import issubklass diff --git a/hololinked/core/events.py b/hololinked/core/events.py index 0a612517..55bc9964 100644 --- a/hololinked/core/events.py +++ b/hololinked/core/events.py @@ -1,9 +1,10 @@ import typing + import jsonschema -from ..param.parameterized import Parameterized, ParameterizedMetaclass -from ..constants import JSON from ..config import global_config +from ..constants import JSON +from ..param.parameterized import Parameterized, ParameterizedMetaclass class Event: @@ -148,6 +149,7 @@ def _set_acknowledgement(self, *args, **kwargs) -> None: from .zmq.brokers import EventPublisher # noqa: E402 + __all__ = [ Event.__name__, ] diff --git a/hololinked/core/logger.py b/hololinked/core/logger.py index 5ad4ae7b..175a020d 100644 --- a/hololinked/core/logger.py +++ b/hololinked/core/logger.py @@ -1,17 +1,18 @@ -import logging -import typing +import asyncio import datetime +import logging import threading -import asyncio import time -import structlog +import typing + from collections import deque +import structlog + +from .actions import action as remote_method from .events import Event -from .properties import List -from .properties import Integer, Number +from .properties import Integer, List, Number from .thing import Thing as RemoteObject -from .actions import action as remote_method log_message_schema = { diff --git a/hololinked/core/meta.py b/hololinked/core/meta.py index 0e2b1ad5..8b4aebb7 100644 --- a/hololinked/core/meta.py +++ b/hololinked/core/meta.py @@ -1,21 +1,18 @@ import copy import inspect -from types import FunctionType import typing -from ..param.parameterized import ( - EventResolver as ParamEventResolver, - EventDispatcher as ParamEventDispatcher, - Parameter, - Parameterized, - ParameterizedMetaclass, - edit_constant as edit_constant_parameters, -) -from ..utils import getattr_without_descriptor_read +from types import FunctionType + from ..constants import JSON, JSONSerializable +from ..param.parameterized import EventDispatcher as ParamEventDispatcher +from ..param.parameterized import EventResolver as ParamEventResolver +from ..param.parameterized import Parameter, Parameterized, ParameterizedMetaclass +from ..param.parameterized import edit_constant as edit_constant_parameters +from ..utils import getattr_without_descriptor_read from .actions import Action, BoundAction, action +from .events import Event, EventDispatcher, EventPublisher from .property import Property -from .events import Event, EventPublisher, EventDispatcher class ThingMeta(ParameterizedMetaclass): diff --git a/hololinked/core/property.py b/hololinked/core/property.py index d8f190b2..536fba09 100644 --- a/hololinked/core/property.py +++ b/hololinked/core/property.py @@ -1,6 +1,9 @@ import typing + from enum import Enum +from pydantic import BaseModel, ConfigDict, RootModel, create_model + from ..param.parameterized import Parameter, Parameterized, ParameterizedMetaclass from ..schema_validators import JSONSchemaValidator from ..utils import issubklass @@ -324,32 +327,25 @@ def to_affordance(self, owner_inst=None): return PropertyAffordance.generate(self, owner_inst or self.owner) -try: - from pydantic import BaseModel, ConfigDict, RootModel, create_model - - def wrap_plain_types_in_rootmodel(model: type) -> type[BaseModel] | type[RootModel]: - """ - Ensure a type is a subclass of BaseModel. +def wrap_plain_types_in_rootmodel(model: type) -> type[BaseModel] | type[RootModel]: + """ + Ensure a type is a subclass of BaseModel. - If a `BaseModel` subclass is passed to this function, we will pass it - through unchanged. Otherwise, we wrap the type in a RootModel. - In the future, we may explicitly check that the argument is a type - and not a model instance. - """ - if model is None: - return - if issubklass(model, BaseModel): - return model - return create_model( - f"{model!r}", - root=(model, ...), - __base__=RootModel, - __config__=ConfigDict(arbitrary_types_allowed=True), - ) # type: ignore[call-overload] -except ImportError: - - def wrap_plain_types_in_rootmodel(model: type) -> type: - raise ImportError("pydantic is not installed, please install it to use this feature") from None + If a `BaseModel` subclass is passed to this function, we will pass it + through unchanged. Otherwise, we wrap the type in a RootModel. + In the future, we may explicitly check that the argument is a type + and not a model instance. + """ + if model is None: + return + if issubklass(model, BaseModel): + return model + return create_model( + f"{model!r}", + root=(model, ...), + __base__=RootModel, + __config__=ConfigDict(arbitrary_types_allowed=True), + ) # type: ignore[call-overload] __all__ = [Property.__name__] diff --git a/hololinked/core/state_machine.py b/hololinked/core/state_machine.py index 91249ceb..7859b8ef 100644 --- a/hololinked/core/state_machine.py +++ b/hololinked/core/state_machine.py @@ -1,4 +1,5 @@ import typing + from enum import Enum, EnumMeta, StrEnum from types import FunctionType, MethodType diff --git a/hololinked/core/thing.py b/hololinked/core/thing.py index 6dccec88..18bf53c3 100644 --- a/hololinked/core/thing.py +++ b/hololinked/core/thing.py @@ -1,18 +1,18 @@ -import logging import inspect +import logging import ssl import typing + import structlog from ..constants import ZMQ_TRANSPORTS -from ..utils import * # noqa: F403 -from .exceptions import * # noqa: F403 -from ..serializers import Serializers, BaseSerializer, JSONSerializer -from .properties import String, ClassSelector -from .property import Property +from ..serializers import BaseSerializer, JSONSerializer, Serializers +from ..utils import forkable, getattr_without_descriptor_read from .actions import BoundAction, action from .events import EventDispatcher -from .meta import ThingMeta, Propertized, RemoteInvokable, EventSource +from .meta import EventSource, Propertized, RemoteInvokable, ThingMeta +from .properties import ClassSelector, String +from .property import Property class Thing(Propertized, RemoteInvokable, EventSource, metaclass=ThingMeta): @@ -115,9 +115,9 @@ class attribute. if serializer is not None: Serializers.register_for_thing_instance(self.id, serializer) - from .logger import prepare_object_logger - from .state_machine import prepare_object_FSM - from ..storage import prepare_object_storage + from .logger import prepare_object_logger # noqa + from .state_machine import prepare_object_FSM # noqa + from ..storage import prepare_object_storage # noqa prepare_object_logger( instance=self, @@ -137,13 +137,12 @@ class attribute. # thing._qualified_id = f'{self._qualified_id}/{thing.id}' def __post_init__(self): - from .zmq.rpc_server import RPCServer # noqa: F401 - from ..server.zmq import ZMQServer # noqa: F401 - from .logger import RemoteAccessHandler from ..storage.database import ThingDB + from .logger import RemoteAccessHandler + from .zmq.rpc_server import RPCServer # noqa: F401 # Type definitions - self.rpc_server = None # type: typing.Optional[RPCServer | ZMQServer] + self.rpc_server = None # type: typing.Optional[RPCServer] self.db_engine: typing.Optional[ThingDB] self._owners = None if not hasattr(self, "_owners") else self._owners # type: typing.Optional[typing.List[Thing]] self._remote_access_loghandler: typing.Optional[RemoteAccessHandler] @@ -254,7 +253,7 @@ def run_with_zmq_server( ZMQ context object to be used for creating sockets. If not supplied, a global shared context is used. For INPROC, either do not supply context or use the same context across all threads. """ - from ..server import run, parse_params + from ..server import parse_params, run servers = parse_params(self.id, [("ZMQ", dict(access_points=access_points, logger=self.logger, **kwargs))]) @@ -302,8 +301,8 @@ def run_with_http_server( - `event_handler`: `BaseHandler` | `EventHandler`, custom event handler for handling events """ - from ..server.http import HTTPServer from ..server import run + from ..server.http import HTTPServer http_server = HTTPServer( port=port, @@ -336,7 +335,7 @@ def run( - `servers`: list[BaseProtocolServer] list of instantiated servers to expose the object. """ - from ..server import run, parse_params + from ..server import parse_params, run from ..server.server import BaseProtocolServer # noqa: F401 access_points = kwargs.get("access_points", None) # type: dict[str, dict | int | str | list[str]] @@ -401,4 +400,5 @@ def __exit__(self, exc_type, exc_value, traceback) -> None: from .state_machine import StateMachine # noqa: F401, E402 + __all__ = [Thing.__name__] diff --git a/pyproject.toml b/pyproject.toml index a0ef812b..0370799b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,8 @@ dependencies = [ [project.optional-dependencies] dev = [ "ifaddr>=0.2.0,<0.3", + "fastjsonschema==2.20.0", + "serpent==1.41", ] @@ -73,9 +75,7 @@ packages = [ [dependency-groups] dev = [ "ConfigParser==7.1.0", - "fastjsonschema==2.20.0", "ipython==8.12.3", - "serpent==1.41", "numpy>=2.0.0", "pandas==2.2.3", "faker==37.5.0", From 548fab9be6070b9ae7d5f388eeb85589f03aac6d Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Sat, 15 Nov 2025 21:53:24 +0100 Subject: [PATCH 07/15] core.zmq import sort --- hololinked/core/zmq/__init__.py | 12 ++++++------ hololinked/core/zmq/brokers.py | 3 +++ hololinked/core/zmq/message.py | 9 ++++++--- hololinked/core/zmq/rpc_server.py | 3 +++ pyproject.toml | 19 +++++++++++-------- 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/hololinked/core/zmq/__init__.py b/hololinked/core/zmq/__init__.py index 6082b459..47f0bc4a 100644 --- a/hololinked/core/zmq/__init__.py +++ b/hololinked/core/zmq/__init__.py @@ -1,10 +1,10 @@ from .brokers import ( # noqa: F401 - AsyncZMQServer, - ZMQServerPool, - SyncZMQClient, - AsyncZMQClient, - MessageMappedZMQClientPool, - EventPublisher, AsyncEventConsumer, + AsyncZMQClient, + AsyncZMQServer, EventConsumer, + EventPublisher, + MessageMappedZMQClientPool, + SyncZMQClient, + ZMQServerPool, ) diff --git a/hololinked/core/zmq/brokers.py b/hololinked/core/zmq/brokers.py index 7f28a09c..6fe6e631 100644 --- a/hololinked/core/zmq/brokers.py +++ b/hololinked/core/zmq/brokers.py @@ -4,16 +4,19 @@ import time import typing import warnings + from enum import Enum import structlog import zmq import zmq.asyncio + from zmq.utils.monitor import parse_monitor_message from ...config import global_config from ...constants import ZMQ_EVENT_MAP, ZMQ_TRANSPORTS, get_socket_type_name from ...serializers.serializers import Serializers +from ...utils import format_exception_as_json, get_current_async_loop, run_callable_somehow, uuid_hex from ..exceptions import BreakLoop from .message import ( ERROR, diff --git a/hololinked/core/zmq/message.py b/hololinked/core/zmq/message.py index f1981fbd..365021ad 100644 --- a/hololinked/core/zmq/message.py +++ b/hololinked/core/zmq/message.py @@ -1,11 +1,14 @@ import typing -import msgspec + from uuid import uuid4 +import msgspec + from ...constants import JSON, byte_types -from ...serializers.serializers import Serializers -from ...serializers.payloads import SerializableData, PreserializedData from ...param.parameters import Integer +from ...serializers.payloads import PreserializedData, SerializableData +from ...serializers.serializers import Serializers + # message types # both directions diff --git a/hololinked/core/zmq/rpc_server.py b/hololinked/core/zmq/rpc_server.py index fdac7091..bb245b53 100644 --- a/hololinked/core/zmq/rpc_server.py +++ b/hololinked/core/zmq/rpc_server.py @@ -5,12 +5,14 @@ import threading import tracemalloc import typing + from collections import deque import structlog import zmq import zmq.asyncio +from ...config import global_config from ...constants import ZMQ_TRANSPORTS, Operations from ...serializers import BaseSerializer, Serializers from ...utils import ( @@ -35,6 +37,7 @@ SerializableData, ) + if global_config.TRACE_MALLOC: tracemalloc.start() diff --git a/pyproject.toml b/pyproject.toml index 0370799b..66dd09f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,28 +31,33 @@ classifiers = [ ] keywords = ["data-acquisition", "zmq-rpc", "SCADA", "IoT", "Web of Things", "remote data logging"] dependencies = [ - "argon2-cffi>=23.1.0", "msgspec>=0.18.6", "pyzmq>=25.1.0,<26.2", - "SQLAlchemy>2.0.21", - "SQLAlchemy_Utils>=0.41", "pydantic>=2.8.0,<3.0.0", "tornado>=6.3.3", "jsonschema>=4.22.0,<5.0", "httpx>=0.28.1,<29.0", "sniffio>=1.3.1,<2.0", - "pymongo>=4.15.2", "aiomqtt>=2.4.0", - "psycopg2-binary>=2.9.11", "structlog>=25.5.0", ] [project.optional-dependencies] -dev = [ +generic = [ "ifaddr>=0.2.0,<0.3", "fastjsonschema==2.20.0", "serpent==1.41", ] +security = [ + "bcrypt==4.3.0", + "argon2-cffi>=23.1.0", +] +db = [ + "sqlalchemy>2.0.21", + "sqlachemy-utils>=0.41", + "psycopg2-binary>=2.9.11", + "pymongo>=4.15.2", +] [project.urls] @@ -89,8 +94,6 @@ test = [ "coverage==7.8.0", "numpy>=2.0.0", "faker==37.5.0", - "bcrypt==4.3.0", - "fastjsonschema==2.20.0", "pytest>=8.0.0", "pytest-cov>=4.0.0", "pytest-order>=1.0.0", From 76f9ac07c77fca21e11585edfa3e1f7eb59b053f Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Sat, 22 Nov 2025 11:39:38 +0100 Subject: [PATCH 08/15] do ruff isort more --- CHANGELOG.md | 6 ++++++ hololinked/storage/__init__.py | 4 ++-- hololinked/storage/config_models.py | 3 ++- hololinked/storage/database.py | 24 ++++++++++++++---------- hololinked/storage/json_storage.py | 4 +++- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0386f8e2..388f83e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ✓ means ready to try +## [v0.3.8] - 2025-11-15 + +- supports structlog for logging, with colored logs and updated log statements +- moves dependencies and imports to a more hexagonal architecture +- uses pytest instead of unittests + ## [v0.3.7] - 2025-10-30 - supports MQTT diff --git a/hololinked/storage/__init__.py b/hololinked/storage/__init__.py index 52b2ee3f..9f0b26da 100644 --- a/hololinked/storage/__init__.py +++ b/hololinked/storage/__init__.py @@ -1,6 +1,6 @@ -from .database import ThingDB, MongoThingDB -from .json_storage import ThingJSONStorage from ..utils import get_a_filename_from_instance +from .database import MongoThingDB, ThingDB +from .json_storage import ThingJSONStorage def prepare_object_storage(instance, **kwargs): diff --git a/hololinked/storage/config_models.py b/hololinked/storage/config_models.py index 6274e678..3578c96e 100644 --- a/hololinked/storage/config_models.py +++ b/hololinked/storage/config_models.py @@ -1,6 +1,7 @@ from typing import Literal + from pydantic import BaseModel, ConfigDict, Field, model_validator -from pydantic.types import StrictStr, SecretStr, StrictInt +from pydantic.types import SecretStr, StrictInt, StrictStr class SQLDBConfig(BaseModel): diff --git a/hololinked/storage/database.py b/hololinked/storage/database.py index f8e55828..d3b1a89f 100644 --- a/hololinked/storage/database.py +++ b/hololinked/storage/database.py @@ -1,19 +1,23 @@ +import base64 import threading import typing -import base64 + +from dataclasses import asdict, dataclass from datetime import datetime -from sqlalchemy import create_engine, select, inspect as inspect_database -from sqlalchemy import String, JSON, LargeBinary, Integer -from sqlalchemy.ext import asyncio as asyncio_ext -from sqlalchemy.orm import sessionmaker, Mapped, mapped_column, DeclarativeBase, MappedAsDataclass from sqlite3 import DatabaseError -from pymongo import MongoClient, errors as mongo_errors -from dataclasses import dataclass, asdict -from ..param import Parameterized -from ..core.property import Property +from pymongo import MongoClient +from pymongo import errors as mongo_errors +from sqlalchemy import JSON, Integer, LargeBinary, String, create_engine, select +from sqlalchemy import inspect as inspect_database +from sqlalchemy.ext import asyncio as asyncio_ext +from sqlalchemy.orm import DeclarativeBase, Mapped, MappedAsDataclass, mapped_column, sessionmaker + from ..constants import JSONSerializable -from ..serializers.serializers import PythonBuiltinJSONSerializer as JSONSerializer, BaseSerializer, Serializers +from ..core.property import Property +from ..param import Parameterized +from ..serializers.serializers import BaseSerializer, Serializers +from ..serializers.serializers import PythonBuiltinJSONSerializer as JSONSerializer from .config_models import MongoDBConfig, SQLDBConfig, SQLiteConfig diff --git a/hololinked/storage/json_storage.py b/hololinked/storage/json_storage.py index 75ecb76b..d0966514 100644 --- a/hololinked/storage/json_storage.py +++ b/hololinked/storage/json_storage.py @@ -1,9 +1,11 @@ import os import threading + from typing import Any, Dict, List, Optional, Union -from ..serializers import JSONSerializer + from ..core.property import Property from ..param import Parameterized +from ..serializers import JSONSerializer class ThingJSONStorage: From b684acf63ec377c97493dd58e92e95054f2f43d9 Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Sat, 22 Nov 2025 12:54:27 +0100 Subject: [PATCH 09/15] isort all dependencies --- hololinked/client/__init__.py | 2 +- hololinked/client/abstractions.py | 9 ++--- hololinked/client/factory.py | 3 ++ .../client/http/consumed_interactions.py | 10 +++--- .../client/mqtt/consumed_interactions.py | 14 +++++--- hololinked/client/proxy.py | 5 +-- .../client/zmq/consumed_interactions.py | 33 +++++++++---------- hololinked/serializers/__init__.py | 6 ++-- hololinked/serializers/payloads.py | 3 +- hololinked/serializers/serializers.py | 4 +-- hololinked/server/__init__.py | 8 ++--- hololinked/server/mqtt.py | 18 +++++----- hololinked/server/server.py | 10 +++--- hololinked/server/utils.py | 1 + hololinked/server/zmq.py | 5 +-- hololinked/td/__init__.py | 4 +-- hololinked/td/base.py | 2 ++ hololinked/td/forms.py | 4 ++- hololinked/td/interaction_affordance.py | 1 + hololinked/td/metadata.py | 1 + hololinked/td/pydantic_extensions.py | 5 +-- hololinked/td/security_definitions.py | 1 + 22 files changed, 87 insertions(+), 62 deletions(-) diff --git a/hololinked/client/__init__.py b/hololinked/client/__init__.py index f8eca045..b311ed03 100644 --- a/hololinked/client/__init__.py +++ b/hololinked/client/__init__.py @@ -1,2 +1,2 @@ -from .proxy import ObjectProxy as ObjectProxy from .factory import ClientFactory as ClientFactory +from .proxy import ObjectProxy as ObjectProxy diff --git a/hololinked/client/abstractions.py b/hololinked/client/abstractions.py index 26436f42..818facc0 100644 --- a/hololinked/client/abstractions.py +++ b/hololinked/client/abstractions.py @@ -24,16 +24,17 @@ # copied from wotpy repository import asyncio -import threading -import typing import builtins import logging +import threading +import typing + from dataclasses import dataclass -from ..td import PropertyAffordance, ActionAffordance, EventAffordance +from ..constants import Operations +from ..td import ActionAffordance, EventAffordance, PropertyAffordance from ..td.forms import Form from ..utils import get_current_async_loop -from ..constants import Operations class ConsumedThingAction: diff --git a/hololinked/client/factory.py b/hololinked/client/factory.py index 31459d9f..48e78c34 100644 --- a/hololinked/client/factory.py +++ b/hololinked/client/factory.py @@ -3,11 +3,13 @@ import threading import uuid import warnings + from typing import Any import aiomqtt import httpx import structlog + from paho.mqtt.client import CallbackAPIVersion, MQTTMessage, MQTTProtocolVersion from paho.mqtt.client import Client as PahoMQTTClient @@ -33,6 +35,7 @@ ZMQProperty, ) + set_global_event_loop_policy() diff --git a/hololinked/client/http/consumed_interactions.py b/hololinked/client/http/consumed_interactions.py index 7933d8d7..52d09699 100644 --- a/hololinked/client/http/consumed_interactions.py +++ b/hololinked/client/http/consumed_interactions.py @@ -7,20 +7,22 @@ import logging import threading import typing + +from copy import deepcopy +from typing import Any, AsyncIterator, Callable, Iterator + import httpcore import httpx -from typing import Any, AsyncIterator, Callable, Iterator -from copy import deepcopy from ...constants import Operations from ...serializers import Serializers +from ...td.forms import Form from ...td.interaction_affordance import ( ActionAffordance, EventAffordance, PropertyAffordance, ) -from ...td.forms import Form -from ..abstractions import ConsumedThingAction, ConsumedThingEvent, ConsumedThingProperty, raise_local_exception, SSE +from ..abstractions import SSE, ConsumedThingAction, ConsumedThingEvent, ConsumedThingProperty, raise_local_exception class HTTPConsumedAffordanceMixin: diff --git a/hololinked/client/mqtt/consumed_interactions.py b/hololinked/client/mqtt/consumed_interactions.py index 48830937..31b73769 100644 --- a/hololinked/client/mqtt/consumed_interactions.py +++ b/hololinked/client/mqtt/consumed_interactions.py @@ -1,12 +1,16 @@ -import aiomqtt import logging + from typing import Any, Callable -from paho.mqtt.client import Client as PahoMQTTClient, MQTTMessage -from ..abstractions import SSE, ConsumedThingEvent -from ...td.interaction_affordance import EventAffordance +import aiomqtt + +from paho.mqtt.client import Client as PahoMQTTClient +from paho.mqtt.client import MQTTMessage + +from ...serializers import BaseSerializer, Serializers # noqa: F401 from ...td.forms import Form -from ...serializers import Serializers, BaseSerializer # noqa: F401 +from ...td.interaction_affordance import EventAffordance +from ..abstractions import SSE, ConsumedThingEvent class MQTTConsumer(ConsumedThingEvent): diff --git a/hololinked/client/proxy.py b/hololinked/client/proxy.py index d4f93394..d5db35b6 100644 --- a/hololinked/client/proxy.py +++ b/hololinked/client/proxy.py @@ -1,8 +1,9 @@ +import base64 import typing + import structlog -import base64 -from .abstractions import ConsumedThingAction, ConsumedThingProperty, ConsumedThingEvent +from .abstractions import ConsumedThingAction, ConsumedThingEvent, ConsumedThingProperty class ObjectProxy: diff --git a/hololinked/client/zmq/consumed_interactions.py b/hololinked/client/zmq/consumed_interactions.py index dba824d3..5ef5c53b 100644 --- a/hololinked/client/zmq/consumed_interactions.py +++ b/hololinked/client/zmq/consumed_interactions.py @@ -1,32 +1,31 @@ import asyncio import logging -import typing import threading -import warnings import traceback +import typing import uuid +import warnings -from ...constants import Operations -from ...serializers.payloads import SerializableData -from ...core import Thing, Action # noqa: F401 -from ...td import PropertyAffordance, ActionAffordance, EventAffordance -from ...td.forms import Form -from ...core.zmq.message import ResponseMessage -from ...core.zmq.message import EMPTY_BYTE, TIMEOUT, ERROR, INVALID_MESSAGE -from ...core.zmq.brokers import ( - SyncZMQClient, - AsyncZMQClient, - EventConsumer, - AsyncEventConsumer, -) from ...client.abstractions import ( + SSE, ConsumedThingAction, ConsumedThingEvent, ConsumedThingProperty, raise_local_exception, - SSE, ) -from ..exceptions import ReplyNotArrivedError, BreakLoop +from ...constants import Operations +from ...core import Action, Thing # noqa: F401 +from ...core.zmq.brokers import ( + AsyncEventConsumer, + AsyncZMQClient, + EventConsumer, + SyncZMQClient, +) +from ...core.zmq.message import EMPTY_BYTE, ERROR, INVALID_MESSAGE, TIMEOUT, ResponseMessage +from ...serializers.payloads import SerializableData +from ...td import ActionAffordance, EventAffordance, PropertyAffordance +from ...td.forms import Form +from ..exceptions import BreakLoop, ReplyNotArrivedError __error_message_types__ = [TIMEOUT, ERROR, INVALID_MESSAGE] diff --git a/hololinked/serializers/__init__.py b/hololinked/serializers/__init__.py index 6493903c..7b1aefe9 100644 --- a/hololinked/serializers/__init__.py +++ b/hololinked/serializers/__init__.py @@ -1,9 +1,9 @@ from .serializers import ( # noqa: F401 + BaseSerializer, JSONSerializer, - PickleSerializer, MsgpackSerializer, - TextSerializer, + PickleSerializer, PythonBuiltinJSONSerializer, - BaseSerializer, Serializers, + TextSerializer, ) diff --git a/hololinked/serializers/payloads.py b/hololinked/serializers/payloads.py index ca98851e..7eee3447 100644 --- a/hololinked/serializers/payloads.py +++ b/hololinked/serializers/payloads.py @@ -1,8 +1,9 @@ import typing + from dataclasses import dataclass from ..constants import byte_types -from .serializers import Serializers, BaseSerializer +from .serializers import BaseSerializer, Serializers @dataclass diff --git a/hololinked/serializers/serializers.py b/hololinked/serializers/serializers.py index ebe71cb6..881cff3a 100644 --- a/hololinked/serializers/serializers.py +++ b/hololinked/serializers/serializers.py @@ -30,18 +30,18 @@ import inspect import io import json as pythonjson - -# serializers: import pickle import typing import uuid import warnings + from collections import deque from enum import Enum from msgspec import Struct, msgpack from msgspec import json as msgspecjson + # default dytypes: try: import numpy diff --git a/hololinked/server/__init__.py b/hololinked/server/__init__.py index 959d7272..c4c9d9aa 100644 --- a/hololinked/server/__init__.py +++ b/hololinked/server/__init__.py @@ -1,11 +1,11 @@ import asyncio -import uuid import threading +import uuid import warnings from ..config import global_config -from ..utils import get_current_async_loop, cancel_pending_tasks_in_current_loop, forkable -from ..core.zmq.rpc_server import RPCServer, ZMQ_TRANSPORTS +from ..core.zmq.rpc_server import ZMQ_TRANSPORTS, RPCServer +from ..utils import cancel_pending_tasks_in_current_loop, forkable, get_current_async_loop from .server import BaseProtocolServer @@ -65,9 +65,9 @@ def stop(): def parse_params(id: str, access_points: list[tuple[str, str | int | dict | list[str]]]) -> list[BaseProtocolServer]: - from .zmq import ZMQServer from .http import HTTPServer from .mqtt import MQTTPublisher + from .zmq import ZMQServer if access_points is not None and not isinstance(access_points, list): raise TypeError("access_points must be provided as a list of tuples.") diff --git a/hololinked/server/mqtt.py b/hololinked/server/mqtt.py index 41372c8a..26694ac1 100644 --- a/hololinked/server/mqtt.py +++ b/hololinked/server/mqtt.py @@ -1,20 +1,22 @@ -import uuid -import aiomqtt import copy import ssl -import structlog +import uuid + from typing import Any, Optional -from ..utils import get_current_async_loop -from .utils import consume_broker_queue, consume_broker_pubsub_per_event +import aiomqtt +import structlog + from ..config import global_config from ..constants import Operations -from ..serializers import Serializers -from ..param.parameters import Selector, String, ClassSelector -from ..core.zmq.message import EventMessage # noqa: F401 from ..core import Thing +from ..core.zmq.message import EventMessage # noqa: F401 +from ..param.parameters import ClassSelector, Selector, String +from ..serializers import Serializers from ..td.interaction_affordance import EventAffordance, PropertyAffordance +from ..utils import get_current_async_loop from .server import BaseProtocolServer +from .utils import consume_broker_pubsub_per_event, consume_broker_queue class MQTTPublisher(BaseProtocolServer): diff --git a/hololinked/server/server.py b/hololinked/server/server.py index da7a054a..6e29478b 100644 --- a/hololinked/server/server.py +++ b/hololinked/server/server.py @@ -1,12 +1,14 @@ import logging + import structlog + from pydantic import BaseModel, model_validator -from ..utils import forkable +from ..core import Action, Event, Property, Thing +from ..core.properties import ClassSelector, Integer, Selector, TypedList from ..param import Parameterized -from ..core.properties import ClassSelector, Integer, TypedList, Selector -from ..core import Thing, Property, Action, Event -from ..td.interaction_affordance import PropertyAffordance, ActionAffordance, EventAffordance +from ..td.interaction_affordance import ActionAffordance, EventAffordance, PropertyAffordance +from ..utils import forkable class BrokerThing(BaseModel): diff --git a/hololinked/server/utils.py b/hololinked/server/utils.py index e9eb2ef3..b3708722 100644 --- a/hololinked/server/utils.py +++ b/hololinked/server/utils.py @@ -1,5 +1,6 @@ import logging import uuid + from typing import Any, Optional import zmq.asyncio diff --git a/hololinked/server/zmq.py b/hololinked/server/zmq.py index 806e1616..13171139 100644 --- a/hololinked/server/zmq.py +++ b/hololinked/server/zmq.py @@ -1,13 +1,14 @@ import typing + +import structlog import zmq import zmq.asyncio -import structlog from ..constants import ZMQ_TRANSPORTS -from ..utils import get_current_async_loop from ..core.thing import Thing from ..core.zmq.brokers import AsyncEventConsumer, AsyncZMQServer, EventPublisher from ..core.zmq.rpc_server import RPCServer +from ..utils import get_current_async_loop from .server import BaseProtocolServer diff --git a/hololinked/td/__init__.py b/hololinked/td/__init__.py index 2db11bbd..1c7ef12a 100644 --- a/hololinked/td/__init__.py +++ b/hololinked/td/__init__.py @@ -1,7 +1,7 @@ from .interaction_affordance import ( # noqa: F401 - InteractionAffordance, - PropertyAffordance, ActionAffordance, EventAffordance, + InteractionAffordance, + PropertyAffordance, ) from .tm import ThingModel # noqa: F401 diff --git a/hololinked/td/base.py b/hololinked/td/base.py index 61bc3c89..1165c1cb 100644 --- a/hololinked/td/base.py +++ b/hololinked/td/base.py @@ -1,6 +1,8 @@ import inspect import typing + from typing import ClassVar + from pydantic import BaseModel diff --git a/hololinked/td/forms.py b/hololinked/td/forms.py index 0d2ec91c..7769bbe3 100644 --- a/hololinked/td/forms.py +++ b/hololinked/td/forms.py @@ -1,7 +1,9 @@ import typing + from pydantic import Field -from .base import Schema + from ..constants import JSON +from .base import Schema class ExpectedResponse(Schema): diff --git a/hololinked/td/interaction_affordance.py b/hololinked/td/interaction_affordance.py index d8f2c211..fcea105c 100644 --- a/hololinked/td/interaction_affordance.py +++ b/hololinked/td/interaction_affordance.py @@ -1,5 +1,6 @@ import copy import typing + from enum import Enum from typing import ClassVar, Optional diff --git a/hololinked/td/metadata.py b/hololinked/td/metadata.py index 2ce291d8..3ef6e048 100644 --- a/hololinked/td/metadata.py +++ b/hololinked/td/metadata.py @@ -1,4 +1,5 @@ import typing + from pydantic import Field from .base import Schema diff --git a/hololinked/td/pydantic_extensions.py b/hololinked/td/pydantic_extensions.py index 11676694..2b73a079 100644 --- a/hololinked/td/pydantic_extensions.py +++ b/hololinked/td/pydantic_extensions.py @@ -1,9 +1,10 @@ from __future__ import annotations +from typing import Any, Dict, List, Mapping, Optional, Sequence, Union + from pydantic import BaseModel, TypeAdapter +from pydantic._internal._core_utils import CoreSchemaOrField, is_core_schema from pydantic.json_schema import GenerateJsonSchema -from pydantic._internal._core_utils import is_core_schema, CoreSchemaOrField -from typing import Optional, Sequence, Union, Any, Mapping, List, Dict JSONSchema = dict[str, Any] # A type to represent JSONSchema diff --git a/hololinked/td/security_definitions.py b/hololinked/td/security_definitions.py index 043b2b5e..52744db0 100644 --- a/hololinked/td/security_definitions.py +++ b/hololinked/td/security_definitions.py @@ -1,4 +1,5 @@ import typing + from .base import Schema From 10f920cfb39bd7aa46ead7213ec49d17381dde34 Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Sat, 22 Nov 2025 13:16:05 +0100 Subject: [PATCH 10/15] update pyproject and readme --- pyproject.toml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 66dd09f8..499ce739 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dependencies = [ generic = [ "ifaddr>=0.2.0,<0.3", "fastjsonschema==2.20.0", - "serpent==1.41", + "serpent>=1.41,<2.0", ] security = [ "bcrypt==4.3.0", @@ -54,14 +54,17 @@ security = [ ] db = [ "sqlalchemy>2.0.21", - "sqlachemy-utils>=0.41", + "sqlalchemy-utils>=0.41", "psycopg2-binary>=2.9.11", "pymongo>=4.15.2", ] +linux = [ + "uvloop==0.20.0" +] [project.urls] -Documentation = "https://hololinked.readthedocs.io/en/latest/index.html" +Documentation = "https://docs.hololinked.dev" Repository = "https://github.com/hololinked-dev/hololinked" [tool.setuptools] @@ -84,16 +87,14 @@ dev = [ "numpy>=2.0.0", "pandas==2.2.3", "faker==37.5.0", - "bcrypt==4.3.0", "pip>=25.2", "ruff>=0.12.10", "jupyter>=1.1.1", ] test = [ "requests==2.32.3", - "coverage==7.8.0", - "numpy>=2.0.0", "faker==37.5.0", + "numpy>=2.0.0", "pytest>=8.0.0", "pytest-cov>=4.0.0", "pytest-order>=1.0.0", @@ -102,18 +103,16 @@ test = [ scanning = [ "bandit>=1.9.1", ] -linux = [ - "uvloop==0.20.0" -] + [tool.pytest.ini_options] minversion = "8.0" addopts = "-ra --strict-markers --strict-config --ignore=lib64" testpaths = ["tests"] -norecursedirs = ["tests/yet-to-be-integrated*"] python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] +norecursedirs = ["tests/yet-to-be-integrated*"] asyncio_mode = "auto" markers = [ "order: mark test to run in a specific order", From cfaf2b7297205d85fe18401187e5229df102859a Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Sun, 23 Nov 2025 14:29:35 +0100 Subject: [PATCH 11/15] fix typing module --- hololinked/schema_validators/json_schema.py | 18 +++--- hololinked/serializers/payloads.py | 4 +- hololinked/serializers/serializers.py | 44 +++++++-------- hololinked/storage/database.py | 62 ++++++++++----------- hololinked/td/tm.py | 9 +-- uv.lock | 51 +++++++++-------- 6 files changed, 94 insertions(+), 94 deletions(-) diff --git a/hololinked/schema_validators/json_schema.py b/hololinked/schema_validators/json_schema.py index acc8f0ad..8989e6d5 100644 --- a/hololinked/schema_validators/json_schema.py +++ b/hololinked/schema_validators/json_schema.py @@ -1,4 +1,4 @@ -import typing +from typing import Any from ..constants import JSON @@ -33,21 +33,21 @@ class JSONSchema: _schemas = {} @classmethod - def is_allowed_type(cls, typ: typing.Any) -> bool: + def is_allowed_type(cls, typ: Any) -> bool: """check if a certain base type has a JSON schema base type""" if typ in JSONSchema._replacements.keys(): return True return False @classmethod - def has_additional_schema_definitions(cls, typ: typing.Any) -> bool: + def has_additional_schema_definitions(cls, typ: Any) -> bool: """Check, if in additional to the JSON schema base type, additional schema definitions exists.""" if typ in JSONSchema._schemas.keys(): return True return False @classmethod - def get_base_type(cls, typ: typing.Any) -> str: + def get_base_type(cls, typ: Any) -> str: if not JSONSchema.is_allowed_type(typ): raise TypeError( f"Object for wot-td has invalid type for JSON conversion. Given type - {type(typ)}. " @@ -56,9 +56,7 @@ def get_base_type(cls, typ: typing.Any) -> str: return JSONSchema._replacements[typ] @classmethod - def register_type_replacement( - self, type: typing.Any, json_schema_base_type: str, schema: typing.Optional[JSON] = None - ) -> None: + def register_type_replacement(self, type: Any, json_schema_base_type: str, schema: JSON | None = None) -> None: """ Specify a python type to map to a specific JSON type. @@ -70,12 +68,12 @@ def register_type_replacement( Parameters ---------- - type: typing.Any + type: Any The Python type to register. The python type must be hashable (can be stored as a key in a dictionary). json_schema_base_type: str The base JSON schema type to map the Python type to. One of ('string', 'number', 'integer', 'boolean', 'object', 'array', 'null'). - schema: typing.Optional[JSON] + schema: Optional[JSON] An optional JSON schema to use for the type. """ if json_schema_base_type in JSONSchema._allowed_types: @@ -93,7 +91,7 @@ def register_type_replacement( ) @classmethod - def get_additional_schema_definitions(cls, typ: typing.Any): + def get_additional_schema_definitions(cls, typ: Any): """schema for array and objects only supported""" if not JSONSchema.has_additional_schema_definitions(typ): raise ValueError(f"Schema for {typ} not provided. register one with JSONSchema.register_type_replacement()") diff --git a/hololinked/serializers/payloads.py b/hololinked/serializers/payloads.py index 7eee3447..1f9816cd 100644 --- a/hololinked/serializers/payloads.py +++ b/hololinked/serializers/payloads.py @@ -1,4 +1,4 @@ -import typing +from typing import Any from dataclasses import dataclass @@ -13,7 +13,7 @@ class SerializableData: The content type decides the serializer to be used. """ - value: typing.Any + value: Any serializer: BaseSerializer | None = None content_type: str = "application/json" _serialized: bytes | None = None diff --git a/hololinked/serializers/serializers.py b/hololinked/serializers/serializers.py index 881cff3a..97f73f79 100644 --- a/hololinked/serializers/serializers.py +++ b/hololinked/serializers/serializers.py @@ -31,12 +31,12 @@ import io import json as pythonjson import pickle -import typing import uuid import warnings from collections import deque from enum import Enum +from typing import Any from msgspec import Struct, msgpack from msgspec import json as msgspecjson @@ -73,7 +73,7 @@ def __init__(self) -> None: super().__init__() self.type = None - def loads(self, data) -> typing.Any: + def loads(self, data) -> Any: "method called by ZMQ message brokers to deserialize data" raise NotImplementedError("implement loads()/deserialization in subclass") @@ -109,7 +109,7 @@ def __init__(self) -> None: super().__init__() self.type = msgspecjson - def loads(self, data: typing.Union[bytearray, memoryview, bytes]) -> JSONSerializable: + def loads(self, data: bytearray | memoryview | bytes) -> JSONSerializable: "method called by ZMQ message brokers to deserialize data" return msgspecjson.decode(self.convert_to_bytes(data)) @@ -174,7 +174,7 @@ def __init__(self) -> None: super().__init__() self.type = pythonjson - def loads(self, data: typing.Union[bytearray, memoryview, bytes]) -> typing.Any: + def loads(self, data: bytearray | memoryview | bytes) -> Any: "method called by ZMQ message brokers to deserialize data" return pythonjson.loads(self.convert_to_bytes(data)) @@ -184,7 +184,7 @@ def dumps(self, data) -> bytes: return data.encode("utf-8") @classmethod - def dump(cls, data: typing.Dict[str, typing.Any], file_desc) -> None: + def dump(cls, data: dict[str, Any], file_desc) -> None: "write JSON to file" pythonjson.dump(data, file_desc, ensure_ascii=False, allow_nan=True, default=cls.default) @@ -209,7 +209,7 @@ def dumps(self, data) -> bytes: return pickle.dumps(data) raise RuntimeError("Pickle serialization is not allowed by the global configuration") - def loads(self, data) -> typing.Any: + def loads(self, data) -> Any: "method called by ZMQ message brokers to deserialize data" from ..config import global_config @@ -238,11 +238,11 @@ def __init__(self) -> None: def dumps(self, value) -> bytes: return msgpack.encode(value, enc_hook=self.default_encode) - def loads(self, value) -> typing.Any: + def loads(self, value) -> Any: return msgpack.decode(self.convert_to_bytes(value), ext_hook=self.ext_decode) @classmethod - def default_encode(cls, obj) -> typing.Any: + def default_encode(cls, obj) -> Any: if "numpy" in globals() and isinstance(obj, numpy.ndarray): buf = io.BytesIO() numpy.save(buf, obj, allow_pickle=False) # use .npy. which stores dtype, shape, order, endianness @@ -250,7 +250,7 @@ def default_encode(cls, obj) -> typing.Any: raise TypeError("Given type cannot be converted to MessagePack : {}".format(type(obj))) @classmethod - def ext_decode(cls, code: int, obj: memoryview) -> typing.Any: + def ext_decode(cls, code: int, obj: memoryview) -> Any: if code == MsgpackSerializer.codes["NDARRAY_EXT"]: if "numpy" in globals(): return numpy.load(io.BytesIO(obj), allow_pickle=False) @@ -273,7 +273,7 @@ def __init__(self) -> None: def dumps(self, data) -> bytes: return str(data).encode("utf-8") - def loads(self, data) -> typing.Any: + def loads(self, data) -> Any: return data.decode("utf-8") @property @@ -295,7 +295,7 @@ def dumps(self, data) -> bytes: "method called by ZMQ message brokers to serialize data" return serpent.dumps(data, module_in_classname=True) - def loads(self, data) -> typing.Any: + def loads(self, data) -> Any: "method called by ZMQ message brokers to deserialize data" return serpent.loads(self.convert_to_bytes(data)) @@ -380,32 +380,32 @@ class Serializers(metaclass=MappableSingleton): doc="A dictionary of content types and their serializers", readonly=True, class_member=True, - ) # type: typing.Dict[str, BaseSerializer] + ) # type: dict[str, BaseSerializer] allowed_content_types = Parameter( default=None, class_member=True, doc="A list of content types that are usually considered safe and will be supported by default without any configuration", readonly=True, - ) # type: typing.List[str] + ) # type: list[str] object_content_type_map = Parameter( default=dict(), class_member=True, doc="A dictionary of content types for specific properties, actions and events", readonly=True, - ) # type: typing.Dict[str, typing.Dict[str, str]] + ) # type: dict[str, dict[str, str]] object_serializer_map = Parameter( default=dict(), class_member=True, doc="A dictionary of serializer for specific properties, actions and events", readonly=True, - ) # type: typing.Dict[str, typing.Dict[str, BaseSerializer]] + ) # type: dict[str, dict[str, BaseSerializer]] protocol_serializer_map = Parameter( default=dict(), class_member=True, doc="A dictionary of serializer for a specific protocol", readonly=True - ) # type: typing.Dict[str, BaseSerializer] + ) # type: dict[str, BaseSerializer] @classmethod def register(cls, serializer: BaseSerializer, name: str | None = None, override: bool = False) -> None: @@ -444,9 +444,9 @@ def for_object(cls, thing_id: str, thing_cls: str, objekt: str) -> BaseSerialize Parameters ---------- - thing_id: str | typing.Any + thing_id: str | Any the id of the Thing or the Thing that owns the property, action or event - thing_cls: str | typing.Any + thing_cls: str | Any the class name of the Thing or the Thing that owns the property, action or event objekt: str | Property | Action | Event the name of the property, action or event @@ -472,7 +472,7 @@ def for_object(cls, thing_id: str, thing_cls: str, objekt: str) -> BaseSerialize # @validate_call @classmethod - def register_for_object(cls, objekt: typing.Any, serializer: BaseSerializer) -> None: + def register_for_object(cls, objekt: Any, serializer: BaseSerializer) -> None: """ Register (an existing) serializer for a property, action or event. Other option is to register a content type, the effects are similar. @@ -505,7 +505,7 @@ def register_for_object(cls, objekt: typing.Any, serializer: BaseSerializer) -> # @validate_call @classmethod - def register_content_type_for_object(cls, objekt: typing.Any, content_type: str) -> None: + def register_content_type_for_object(cls, objekt: Any, content_type: str) -> None: """ Register content type for a property, action, event, or a `Thing` class to use a specific serializer. @@ -544,7 +544,7 @@ def register_content_type_for_object(cls, objekt: typing.Any, content_type: str) # @validate_call @classmethod def register_content_type_for_object_per_thing_instance( - cls, thing_id: str, objekt: str | typing.Any, content_type: str + cls, thing_id: str, objekt: str | Any, content_type: str ) -> None: """ Register an existing content type for a property, action or event to use a specific serializer. Other option is @@ -633,7 +633,7 @@ def reset(cls) -> None: cls.default = cls.json @allowed_content_types.getter - def get_allowed_content_types(cls) -> typing.List[str]: + def get_allowed_content_types(cls) -> list[str]: """ Get a list of all allowed content types for serialization. """ diff --git a/hololinked/storage/database.py b/hololinked/storage/database.py index d3b1a89f..5968daed 100644 --- a/hololinked/storage/database.py +++ b/hololinked/storage/database.py @@ -1,10 +1,10 @@ import base64 import threading -import typing from dataclasses import asdict, dataclass from datetime import datetime from sqlite3 import DatabaseError +from typing import Any from pymongo import MongoClient from pymongo import errors as mongo_errors @@ -67,7 +67,7 @@ class DeserializedProperty: # not part of database thing_id: str name: str - value: typing.Any + value: Any created_at: str updated_at: str @@ -75,7 +75,7 @@ class DeserializedProperty: # not part of database class BaseDB: """Implements configuration file reader for all irrespective sync or async DB operation""" - def __init__(self, instance: Parameterized, config_file: typing.Union[str, None] = None) -> None: + def __init__(self, instance: Parameterized, config_file: str | None = None) -> None: self.thing_instance = instance self.conf = BaseDB.load_conf(config_file, default_file_path=f"{instance.id}.db") self.URL = self.conf.URL @@ -86,7 +86,7 @@ def load_conf( cls, config_file: str, default_file_path: str = "", - ) -> typing.Union[SQLDBConfig, SQLiteConfig, MongoDBConfig]: + ) -> SQLDBConfig | SQLiteConfig | MongoDBConfig: """ load configuration file using JSON serializer """ @@ -130,8 +130,8 @@ class BaseAsyncDB(BaseDB): def __init__( self, instance: Parameterized, - serializer: typing.Optional[BaseSerializer] = None, - config_file: typing.Union[str, None] = None, + serializer: BaseSerializer | None = None, + config_file: str | None = None, ) -> None: super().__init__(instance=instance, serializer=serializer, config_file=config_file) self.engine = asyncio_ext.create_async_engine(self.URL) @@ -157,7 +157,7 @@ class BaseSyncDB(BaseDB): absolute path to database server configuration file """ - def __init__(self, instance: Parameterized, config_file: typing.Union[str, None] = None) -> None: + def __init__(self, instance: Parameterized, config_file: str | None = None) -> None: super().__init__(instance=instance, config_file=config_file) self.engine = create_engine(self.URL) self.sync_session = sessionmaker(self.engine, expire_on_commit=True) @@ -203,7 +203,7 @@ def fetch_own_info(self): # -> ThingInformation: "Multiple things with same instance name found, either cleanup database/detach/make new" ) - def get_property(self, property: typing.Union[str, Property], deserialized: bool = True) -> typing.Any: + def get_property(self, property: str | Property, deserialized: bool = True) -> Any: """ fetch a single property. @@ -227,7 +227,7 @@ def get_property(self, property: typing.Union[str, Property], deserialized: bool name=name, ) data = session.execute(stmt) - prop = data.scalars().all() # type: typing.Sequence[SerializedProperty] + prop = data.scalars().all() # type: list[SerializedProperty] if len(prop) == 0: raise DatabaseError(f"property {name} not found in database") elif len(prop) > 1: @@ -239,7 +239,7 @@ def get_property(self, property: typing.Union[str, Property], deserialized: bool ) return serializer.loads(prop[0].serialized_value) - def set_property(self, property: typing.Union[str, Property], value: typing.Any) -> None: + def set_property(self, property: str | Property, value: Any) -> None: """ change the value of an already existing property. @@ -290,9 +290,7 @@ def set_property(self, property: typing.Union[str, Property], value: typing.Any) session.add(prop) session.commit() - def get_properties( - self, properties: typing.Dict[typing.Union[str, Property], typing.Any], deserialized: bool = True - ) -> typing.Dict[str, typing.Any]: + def get_properties(self, properties: dict[str | Property, Any], deserialized: bool = True) -> dict[str, Any]: """ get multiple properties at once. @@ -327,7 +325,7 @@ def get_properties( ) return props - def set_properties(self, properties: typing.Dict[typing.Union[str, Property], typing.Any]) -> None: + def set_properties(self, properties: dict[str | Property, Any]) -> None: """ change the values of already existing few properties at once @@ -354,7 +352,7 @@ def set_properties(self, properties: typing.Dict[typing.Union[str, Property], ty db_props = data.scalars().all() for obj, value in properties.items(): name = obj if isinstance(obj, str) else obj.name - db_prop = list(filter(lambda db_prop: db_prop.name == name, db_props)) # type: typing.List[SerializedProperty] + db_prop = list(filter(lambda db_prop: db_prop.name == name, db_props)) # type: list[SerializedProperty] if len(db_prop) > 1: raise DatabaseError("multiple properties with same name found") # Impossible actually if len(db_prop) == 1: @@ -383,7 +381,7 @@ def set_properties(self, properties: typing.Dict[typing.Union[str, Property], ty session.add(prop) session.commit() - def get_all_properties(self, deserialized: bool = True) -> typing.Dict[str, typing.Any]: + def get_all_properties(self, deserialized: bool = True) -> dict[str, Any]: """ read all properties of the `Thing` instance. @@ -397,7 +395,7 @@ def get_all_properties(self, deserialized: bool = True) -> typing.Dict[str, typi thing_id=self.thing_instance.id, thing_class=self.thing_instance.__class__.__name__ ) data = session.execute(stmt) - existing_props = data.scalars().all() # type: typing.Sequence[SerializedProperty] + existing_props = data.scalars().all() # type: list[SerializedProperty] if not deserialized: return existing_props props = dict() @@ -409,7 +407,7 @@ def get_all_properties(self, deserialized: bool = True) -> typing.Dict[str, typi return props def create_missing_properties( - self, properties: typing.Dict[str, Property], get_missing_property_names: bool = False + self, properties: dict[str, Property], get_missing_property_names: bool = False ) -> None: """ create any and all missing properties of `Thing` instance in database. @@ -449,9 +447,7 @@ def create_missing_properties( if get_missing_property_names: return missing_props - def create_db_init_properties( - self, thing_id: str = None, thing_class: str = None, **properties: typing.Any - ) -> None: + def create_db_init_properties(self, thing_id: str = None, thing_class: str = None, **properties: Any) -> None: """ Create properties that are initialized from database for a thing instance. Invoke this method once before running the thing instance to store its initial value in database. @@ -496,7 +492,7 @@ def __enter__(self) -> None: self.db_engine._context[threading.get_ident()] = dict() def __exit__(self, exc_type, exc_value, exc_tb) -> None: - data = self.db_engine._context.pop(threading.get_ident(), dict()) # typing.Dict[str, typing.Any] + data = self.db_engine._context.pop(threading.get_ident(), dict()) # dict[str, Any] if exc_type is None: self.db_engine.set_properties(data) return @@ -522,7 +518,7 @@ class MongoThingDB: Methods mirror the interface of ThingDB for compatibility. """ - def __init__(self, instance: Parameterized, config_file: typing.Union[str, None] = None) -> None: + def __init__(self, instance: Parameterized, config_file: str | None = None) -> None: """ Initialize MongoThingDB for a Thing instance. Connects to MongoDB and sets up collections. @@ -536,7 +532,7 @@ def __init__(self, instance: Parameterized, config_file: typing.Union[str, None] self.things = self.db["things"] @classmethod - def load_conf(cls, config_file: str) -> typing.Dict[str, typing.Any]: + def load_conf(cls, config_file: str | None) -> dict[str, Any]: """ Load configuration from JSON file if provided. """ @@ -555,7 +551,7 @@ def fetch_own_info(self): doc = self.things.find_one({"id": self.id}) return doc - def get_property(self, property: typing.Union[str, Property], deserialized: bool = True) -> typing.Any: + def get_property(self, property: str | Property, deserialized: bool = True) -> Any: """ Get a property value from MongoDB for this Thing. If deserialized=True, returns the Python value. @@ -569,7 +565,7 @@ def get_property(self, property: typing.Union[str, Property], deserialized: bool serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, name) return serializer.loads(base64.b64decode(doc["serialized_value"])) - def set_property(self, property: typing.Union[str, Property], value: typing.Any) -> None: + def set_property(self, property: str | Property, value: Any) -> None: """ Set a property value in MongoDB for this Thing. Value is serialized before storage. @@ -581,9 +577,7 @@ def set_property(self, property: typing.Union[str, Property], value: typing.Any) {"id": self.id, "name": name}, {"$set": {"serialized_value": serialized_value}}, upsert=True ) - def get_properties( - self, properties: typing.Dict[typing.Union[str, Property], typing.Any], deserialized: bool = True - ) -> typing.Dict[str, typing.Any]: + def get_properties(self, properties: dict[str | Property, Any], deserialized: bool = True) -> dict[str, Any]: """ Get multiple property values from MongoDB for this Thing. Returns a dict of property names to values. @@ -600,7 +594,7 @@ def get_properties( ) return result - def set_properties(self, properties: typing.Dict[typing.Union[str, Property], typing.Any]) -> None: + def set_properties(self, properties: dict[str | Property, Any]) -> None: """ Set multiple property values in MongoDB for this Thing. """ @@ -612,7 +606,7 @@ def set_properties(self, properties: typing.Dict[typing.Union[str, Property], ty {"id": self.id, "name": name}, {"$set": {"serialized_value": serialized_value}}, upsert=True ) - def get_all_properties(self, deserialized: bool = True) -> typing.Dict[str, typing.Any]: + def get_all_properties(self, deserialized: bool = True) -> dict[str, Any]: cursor = self.properties.find({"id": self.id}) result = {} for doc in cursor: @@ -625,8 +619,10 @@ def get_all_properties(self, deserialized: bool = True) -> typing.Dict[str, typi return result def create_missing_properties( - self, properties: typing.Dict[str, Property], get_missing_property_names: bool = False - ) -> typing.Any: + self, + properties: dict[str, Property], + get_missing_property_names: bool = False, + ) -> Any: missing_props = [] existing_props = self.get_all_properties() for name, new_prop in properties.items(): diff --git a/hololinked/td/tm.py b/hololinked/td/tm.py index 7e0ec47b..5c705abe 100644 --- a/hololinked/td/tm.py +++ b/hololinked/td/tm.py @@ -1,12 +1,13 @@ import typing -from pydantic import Field, ConfigDict +from pydantic import ConfigDict, Field + +from ..core import Thing +from ..core.state_machine import BoundFSM from .base import Schema from .data_schema import DataSchema +from .interaction_affordance import ActionAffordance, EventAffordance, PropertyAffordance from .metadata import VersionInfo -from .interaction_affordance import PropertyAffordance, ActionAffordance, EventAffordance -from ..core.state_machine import BoundFSM -from ..core import Thing class ThingModel(Schema): diff --git a/uv.lock b/uv.lock index a5d884aa..aaeeb7ca 100644 --- a/uv.lock +++ b/uv.lock @@ -637,40 +637,48 @@ wheels = [ [[package]] name = "hololinked" -version = "0.3.7" +version = "0.3.8" source = { editable = "." } dependencies = [ { name = "aiomqtt" }, - { name = "argon2-cffi" }, - { name = "bandit" }, { name = "httpx" }, - { name = "ifaddr" }, { name = "jsonschema" }, { name = "msgspec" }, - { name = "psycopg2-binary" }, { name = "pydantic" }, - { name = "pymongo" }, { name = "pyzmq" }, { name = "sniffio" }, - { name = "sqlalchemy" }, - { name = "sqlalchemy-utils" }, { name = "structlog" }, { name = "tornado" }, ] +[package.optional-dependencies] +db = [ + { name = "psycopg2-binary" }, + { name = "pymongo" }, + { name = "sqlalchemy" }, + { name = "sqlalchemy-utils" }, +] +generic = [ + { name = "fastjsonschema" }, + { name = "ifaddr" }, + { name = "serpent" }, +] +security = [ + { name = "argon2-cffi" }, + { name = "bcrypt" }, +] + [package.dev-dependencies] dev = [ { name = "bcrypt" }, { name = "configparser" }, { name = "faker" }, - { name = "fastjsonschema" }, { name = "ipython" }, { name = "jupyter" }, { name = "numpy" }, { name = "pandas" }, { name = "pip" }, { name = "ruff" }, - { name = "serpent" }, ] linux = [ { name = "uvloop" }, @@ -679,10 +687,8 @@ scanning = [ { name = "bandit" }, ] test = [ - { name = "bcrypt" }, { name = "coverage" }, { name = "faker" }, - { name = "fastjsonschema" }, { name = "numpy" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -694,44 +700,43 @@ test = [ [package.metadata] requires-dist = [ { name = "aiomqtt", specifier = ">=2.4.0" }, - { name = "argon2-cffi", specifier = ">=23.1.0" }, - { name = "bandit", specifier = ">=1.9.1" }, + { name = "argon2-cffi", marker = "extra == 'security'", specifier = ">=23.1.0" }, + { name = "bcrypt", marker = "extra == 'security'", specifier = "==4.3.0" }, + { name = "fastjsonschema", marker = "extra == 'generic'", specifier = "==2.20.0" }, { name = "httpx", specifier = ">=0.28.1,<29.0" }, - { name = "ifaddr", specifier = ">=0.2.0,<0.3" }, + { name = "ifaddr", marker = "extra == 'generic'", specifier = ">=0.2.0,<0.3" }, { name = "jsonschema", specifier = ">=4.22.0,<5.0" }, { name = "msgspec", specifier = ">=0.18.6" }, - { name = "psycopg2-binary", specifier = ">=2.9.11" }, + { name = "psycopg2-binary", marker = "extra == 'db'", specifier = ">=2.9.11" }, { name = "pydantic", specifier = ">=2.8.0,<3.0.0" }, - { name = "pymongo", specifier = ">=4.15.2" }, + { name = "pymongo", marker = "extra == 'db'", specifier = ">=4.15.2" }, { name = "pyzmq", specifier = ">=25.1.0,<26.2" }, + { name = "serpent", marker = "extra == 'generic'", specifier = "==1.41" }, { name = "sniffio", specifier = ">=1.3.1,<2.0" }, - { name = "sqlalchemy", specifier = ">2.0.21" }, - { name = "sqlalchemy-utils", specifier = ">=0.41" }, + { name = "sqlalchemy", marker = "extra == 'db'", specifier = ">2.0.21" }, + { name = "sqlalchemy-utils", marker = "extra == 'db'", specifier = ">=0.41" }, { name = "structlog", specifier = ">=25.5.0" }, { name = "tornado", specifier = ">=6.3.3" }, ] +provides-extras = ["generic", "security", "db"] [package.metadata.requires-dev] dev = [ { name = "bcrypt", specifier = "==4.3.0" }, { name = "configparser", specifier = "==7.1.0" }, { name = "faker", specifier = "==37.5.0" }, - { name = "fastjsonschema", specifier = "==2.20.0" }, { name = "ipython", specifier = "==8.12.3" }, { name = "jupyter", specifier = ">=1.1.1" }, { name = "numpy", specifier = ">=2.0.0" }, { name = "pandas", specifier = "==2.2.3" }, { name = "pip", specifier = ">=25.2" }, { name = "ruff", specifier = ">=0.12.10" }, - { name = "serpent", specifier = "==1.41" }, ] linux = [{ name = "uvloop", specifier = "==0.20.0" }] scanning = [{ name = "bandit" }] test = [ - { name = "bcrypt", specifier = "==4.3.0" }, { name = "coverage", specifier = "==7.8.0" }, { name = "faker", specifier = "==37.5.0" }, - { name = "fastjsonschema", specifier = "==2.20.0" }, { name = "numpy", specifier = ">=2.0.0" }, { name = "pytest", specifier = ">=8.0.0" }, { name = "pytest-asyncio", specifier = ">=1.3.0" }, From 09cf5e37536672fbe586cae521f11df9e408087e Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Mon, 24 Nov 2025 20:34:53 +0100 Subject: [PATCH 12/15] check diff --- .github/workflows/ci-pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 9c3a746b..460549f9 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -100,6 +100,7 @@ jobs: test: name: unit-integration tests needs: scan + if: false strategy: matrix: From 0c3f215be68e47188b491d529b4dc7361957d4e1 Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Tue, 25 Nov 2025 08:31:01 +0100 Subject: [PATCH 13/15] do ruff via terminal --- hololinked/client/http/consumed_interactions.py | 8 +++++++- hololinked/client/zmq/consumed_interactions.py | 8 +++++++- hololinked/core/zmq/brokers.py | 7 ++++++- hololinked/serializers/payloads.py | 3 +-- hololinked/server/__init__.py | 6 +++++- hololinked/server/http/__init__.py | 2 ++ hololinked/server/http/handlers.py | 2 ++ hololinked/server/server.py | 6 +++++- hololinked/server/utils.py | 2 +- hololinked/storage/database.py | 8 +++++++- hololinked/td/tm.py | 6 +++++- tests/helper-scripts/run_test_thing.py | 15 +++++++++------ tests/test_05_brokers.py | 2 +- tests/test_09_rpc_broker.py | 2 ++ tests/test_11_rpc_e2e.py | 2 ++ tests/test_12_protocols_zmq_ipc.py | 1 + tests/test_13_protocols_zmq_tcp.py | 1 + tests/test_14_protocols_http.py | 2 ++ tests/test_15_protocols_http_e2e.py | 1 + tests/things/spectrometer.py | 11 ++++++----- .../working/test_07_properties_mongodb.py | 17 ++++++++++++++--- 21 files changed, 87 insertions(+), 25 deletions(-) diff --git a/hololinked/client/http/consumed_interactions.py b/hololinked/client/http/consumed_interactions.py index 52d09699..6dc1a206 100644 --- a/hololinked/client/http/consumed_interactions.py +++ b/hololinked/client/http/consumed_interactions.py @@ -22,7 +22,13 @@ EventAffordance, PropertyAffordance, ) -from ..abstractions import SSE, ConsumedThingAction, ConsumedThingEvent, ConsumedThingProperty, raise_local_exception +from ..abstractions import ( + SSE, + ConsumedThingAction, + ConsumedThingEvent, + ConsumedThingProperty, + raise_local_exception, +) class HTTPConsumedAffordanceMixin: diff --git a/hololinked/client/zmq/consumed_interactions.py b/hololinked/client/zmq/consumed_interactions.py index 5ef5c53b..e61cc769 100644 --- a/hololinked/client/zmq/consumed_interactions.py +++ b/hololinked/client/zmq/consumed_interactions.py @@ -21,7 +21,13 @@ EventConsumer, SyncZMQClient, ) -from ...core.zmq.message import EMPTY_BYTE, ERROR, INVALID_MESSAGE, TIMEOUT, ResponseMessage +from ...core.zmq.message import ( + EMPTY_BYTE, + ERROR, + INVALID_MESSAGE, + TIMEOUT, + ResponseMessage, +) from ...serializers.payloads import SerializableData from ...td import ActionAffordance, EventAffordance, PropertyAffordance from ...td.forms import Form diff --git a/hololinked/core/zmq/brokers.py b/hololinked/core/zmq/brokers.py index 6fe6e631..61d1e68a 100644 --- a/hololinked/core/zmq/brokers.py +++ b/hololinked/core/zmq/brokers.py @@ -16,7 +16,12 @@ from ...config import global_config from ...constants import ZMQ_EVENT_MAP, ZMQ_TRANSPORTS, get_socket_type_name from ...serializers.serializers import Serializers -from ...utils import format_exception_as_json, get_current_async_loop, run_callable_somehow, uuid_hex +from ...utils import ( + format_exception_as_json, + get_current_async_loop, + run_callable_somehow, + uuid_hex, +) from ..exceptions import BreakLoop from .message import ( ERROR, diff --git a/hololinked/serializers/payloads.py b/hololinked/serializers/payloads.py index 1f9816cd..9ebc6df3 100644 --- a/hololinked/serializers/payloads.py +++ b/hololinked/serializers/payloads.py @@ -1,6 +1,5 @@ -from typing import Any - from dataclasses import dataclass +from typing import Any from ..constants import byte_types from .serializers import BaseSerializer, Serializers diff --git a/hololinked/server/__init__.py b/hololinked/server/__init__.py index c4c9d9aa..8d3cace5 100644 --- a/hololinked/server/__init__.py +++ b/hololinked/server/__init__.py @@ -5,7 +5,11 @@ from ..config import global_config from ..core.zmq.rpc_server import ZMQ_TRANSPORTS, RPCServer -from ..utils import cancel_pending_tasks_in_current_loop, forkable, get_current_async_loop +from ..utils import ( + cancel_pending_tasks_in_current_loop, + forkable, + get_current_async_loop, +) from .server import BaseProtocolServer diff --git a/hololinked/server/http/__init__.py b/hololinked/server/http/__init__.py index 9c55beb0..84d2ecca 100644 --- a/hololinked/server/http/__init__.py +++ b/hololinked/server/http/__init__.py @@ -3,9 +3,11 @@ import ssl import typing import warnings + from copy import deepcopy import structlog + from pydantic import BaseModel from tornado import ioloop from tornado.httpserver import HTTPServer as TornadoHTTP1Server diff --git a/hololinked/server/http/handlers.py b/hololinked/server/http/handlers.py index 4ab5f780..5132baf0 100644 --- a/hololinked/server/http/handlers.py +++ b/hololinked/server/http/handlers.py @@ -3,6 +3,7 @@ import uuid import msgspec + from msgspec import DecodeError as MsgspecJSONDecodeError from tornado.iostream import StreamClosedError from tornado.web import RequestHandler, StaticFileHandler @@ -34,6 +35,7 @@ from ...td.forms import Form from ...utils import format_exception_as_json, get_current_async_loop + try: from ..security import BcryptBasicSecurity except ImportError: diff --git a/hololinked/server/server.py b/hololinked/server/server.py index 6e29478b..8b0988e2 100644 --- a/hololinked/server/server.py +++ b/hololinked/server/server.py @@ -7,7 +7,11 @@ from ..core import Action, Event, Property, Thing from ..core.properties import ClassSelector, Integer, Selector, TypedList from ..param import Parameterized -from ..td.interaction_affordance import ActionAffordance, EventAffordance, PropertyAffordance +from ..td.interaction_affordance import ( + ActionAffordance, + EventAffordance, + PropertyAffordance, +) from ..utils import forkable diff --git a/hololinked/server/utils.py b/hololinked/server/utils.py index b3708722..db6e65b9 100644 --- a/hololinked/server/utils.py +++ b/hololinked/server/utils.py @@ -81,7 +81,7 @@ async def consume_broker_queue( return client, TD -def consumer_broker_pubsub(id: str = None, access_point: str = "INPROC") -> AsyncEventConsumer: +def consume_broker_pubsub(id: str = None, access_point: str = "INPROC") -> AsyncEventConsumer: """Consume all events from ZMQ (usually INPROC) pubsub""" return AsyncEventConsumer( id=id or f"event-proxy-{uuid.uuid4().hex[:8]}", diff --git a/hololinked/storage/database.py b/hololinked/storage/database.py index 5968daed..6504c5fd 100644 --- a/hololinked/storage/database.py +++ b/hololinked/storage/database.py @@ -11,7 +11,13 @@ from sqlalchemy import JSON, Integer, LargeBinary, String, create_engine, select from sqlalchemy import inspect as inspect_database from sqlalchemy.ext import asyncio as asyncio_ext -from sqlalchemy.orm import DeclarativeBase, Mapped, MappedAsDataclass, mapped_column, sessionmaker +from sqlalchemy.orm import ( + DeclarativeBase, + Mapped, + MappedAsDataclass, + mapped_column, + sessionmaker, +) from ..constants import JSONSerializable from ..core.property import Property diff --git a/hololinked/td/tm.py b/hololinked/td/tm.py index 5c705abe..96cc862c 100644 --- a/hololinked/td/tm.py +++ b/hololinked/td/tm.py @@ -6,7 +6,11 @@ from ..core.state_machine import BoundFSM from .base import Schema from .data_schema import DataSchema -from .interaction_affordance import ActionAffordance, EventAffordance, PropertyAffordance +from .interaction_affordance import ( + ActionAffordance, + EventAffordance, + PropertyAffordance, +) from .metadata import VersionInfo diff --git a/tests/helper-scripts/run_test_thing.py b/tests/helper-scripts/run_test_thing.py index 57a68e16..99c6d314 100644 --- a/tests/helper-scripts/run_test_thing.py +++ b/tests/helper-scripts/run_test_thing.py @@ -1,16 +1,19 @@ import os -import sys import ssl +import sys + sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))) -from hololinked.server.http import HTTPServer -from hololinked.server.zmq import ZMQServer -from hololinked.server.mqtt import MQTTPublisher -from hololinked.serializers import Serializers +from things import OceanOpticsSpectrometer, TestThing + from hololinked.config import global_config +from hololinked.serializers import Serializers from hololinked.server import run, stop -from things import TestThing, OceanOpticsSpectrometer +from hololinked.server.http import HTTPServer +from hololinked.server.mqtt import MQTTPublisher +from hololinked.server.zmq import ZMQServer + global_config.DEBUG = True diff --git a/tests/test_05_brokers.py b/tests/test_05_brokers.py index 32209feb..752a674d 100644 --- a/tests/test_05_brokers.py +++ b/tests/test_05_brokers.py @@ -7,6 +7,7 @@ import pytest +from hololinked.core.exceptions import BreakLoop from hololinked.core.zmq.brokers import ( AsyncZMQClient, AsyncZMQServer, @@ -24,7 +25,6 @@ ResponseMessage, SerializableData, ) -from hololinked.exceptions import BreakLoop from hololinked.utils import get_current_async_loop, uuid_hex diff --git a/tests/test_09_rpc_broker.py b/tests/test_09_rpc_broker.py index 8fea0770..cfd99b2b 100644 --- a/tests/test_09_rpc_broker.py +++ b/tests/test_09_rpc_broker.py @@ -2,6 +2,7 @@ import random import threading import time + from copy import deepcopy from types import SimpleNamespace from typing import Any, Generator @@ -20,6 +21,7 @@ from hololinked.td.forms import Form from hololinked.utils import get_all_sub_things_recusively, uuid_hex + try: from .test_06_actions import replace_methods_with_actions from .things import TestThing diff --git a/tests/test_11_rpc_e2e.py b/tests/test_11_rpc_e2e.py index cd576cd6..8c36455a 100644 --- a/tests/test_11_rpc_e2e.py +++ b/tests/test_11_rpc_e2e.py @@ -1,4 +1,5 @@ import time + from typing import Any, Generator import pytest @@ -8,6 +9,7 @@ from hololinked.client.proxy import ObjectProxy from hololinked.utils import uuid_hex + try: from .things import TestThing from .utils import fake diff --git a/tests/test_12_protocols_zmq_ipc.py b/tests/test_12_protocols_zmq_ipc.py index 46cef4bc..22814adc 100644 --- a/tests/test_12_protocols_zmq_ipc.py +++ b/tests/test_12_protocols_zmq_ipc.py @@ -1,5 +1,6 @@ import pytest + try: from .test_11_rpc_e2e import TestRPC_E2E as BaseRPC_E2E # noqa: F401 from .test_11_rpc_e2e import client, thing, thing_model # noqa: F401 diff --git a/tests/test_13_protocols_zmq_tcp.py b/tests/test_13_protocols_zmq_tcp.py index fa00462f..94b703a7 100644 --- a/tests/test_13_protocols_zmq_tcp.py +++ b/tests/test_13_protocols_zmq_tcp.py @@ -1,5 +1,6 @@ import pytest + try: from .test_11_rpc_e2e import TestRPC_E2E as BaseRPC_E2E # noqa: F401 from .test_11_rpc_e2e import client, thing, thing_model # noqa: F401 diff --git a/tests/test_14_protocols_http.py b/tests/test_14_protocols_http.py index 83ca4370..c6f7c47c 100644 --- a/tests/test_14_protocols_http.py +++ b/tests/test_14_protocols_http.py @@ -3,6 +3,7 @@ import random import sys import time + from contextlib import contextmanager from dataclasses import dataclass from typing import Any, Generator @@ -26,6 +27,7 @@ from hololinked.server.security import Argon2BasicSecurity, BcryptBasicSecurity, Security from hololinked.utils import uuid_hex + try: from .things import OceanOpticsSpectrometer except ImportError: diff --git a/tests/test_15_protocols_http_e2e.py b/tests/test_15_protocols_http_e2e.py index 6061eaf4..6e547e94 100644 --- a/tests/test_15_protocols_http_e2e.py +++ b/tests/test_15_protocols_http_e2e.py @@ -6,6 +6,7 @@ from hololinked.server import stop from hololinked.utils import uuid_hex + try: from .test_11_rpc_e2e import TestRPC_E2E as BaseRPC_E2E # noqa: F401 from .test_11_rpc_e2e import client, thing, thing_model # noqa: F401 diff --git a/tests/things/spectrometer.py b/tests/things/spectrometer.py index 735ab001..a962b073 100644 --- a/tests/things/spectrometer.py +++ b/tests/things/spectrometer.py @@ -1,17 +1,18 @@ import datetime -from enum import StrEnum import threading import time import typing -import numpy + from dataclasses import dataclass +from enum import StrEnum +import numpy -from hololinked.core import Thing, Property, action, Event -from hololinked.core.properties import String, Integer, Number, List, Boolean, Selector, ClassSelector, TypedList +from hololinked.core import Event, Thing, action +from hololinked.core.properties import Boolean, ClassSelector, Integer, List, Number, Selector, String, TypedList from hololinked.core.state_machine import StateMachine -from hololinked.serializers import JSONSerializer from hololinked.schema_validators import JSONSchema +from hololinked.serializers import JSONSerializer from hololinked.server.http import HTTPServer diff --git a/tests/yet-to-be-integrated/working/test_07_properties_mongodb.py b/tests/yet-to-be-integrated/working/test_07_properties_mongodb.py index 54def57b..56612771 100644 --- a/tests/yet-to-be-integrated/working/test_07_properties_mongodb.py +++ b/tests/yet-to-be-integrated/working/test_07_properties_mongodb.py @@ -1,9 +1,11 @@ import unittest -from hololinked.core.property import Property -from hololinked.core import Thing -from hololinked.storage.database import MongoThingDB + from pymongo import MongoClient +from hololinked.core import Thing +from hololinked.core.property import Property + + class TestMongoDBOperations(unittest.TestCase): @classmethod def setUpClass(cls): @@ -18,6 +20,7 @@ def setUpClass(cls): def test_mongo_string_property(self): class MongoTestThing(Thing): str_prop = Property(default="hello", db_persist=True) + instance = MongoTestThing(id="mongo_str", use_mongo_db=True) instance.str_prop = "world" value_from_db = instance.db_engine.get_property("str_prop") @@ -26,6 +29,7 @@ class MongoTestThing(Thing): def test_mongo_float_property(self): class MongoTestThing(Thing): float_prop = Property(default=1.23, db_persist=True) + instance = MongoTestThing(id="mongo_float", use_mongo_db=True) instance.float_prop = 4.56 value_from_db = instance.db_engine.get_property("float_prop") @@ -34,6 +38,7 @@ class MongoTestThing(Thing): def test_mongo_bool_property(self): class MongoTestThing(Thing): bool_prop = Property(default=False, db_persist=True) + instance = MongoTestThing(id="mongo_bool", use_mongo_db=True) instance.bool_prop = True value_from_db = instance.db_engine.get_property("bool_prop") @@ -42,6 +47,7 @@ class MongoTestThing(Thing): def test_mongo_dict_property(self): class MongoTestThing(Thing): dict_prop = Property(default={"a": 1}, db_persist=True) + instance = MongoTestThing(id="mongo_dict", use_mongo_db=True) instance.dict_prop = {"b": 2, "c": 3} value_from_db = instance.db_engine.get_property("dict_prop") @@ -50,6 +56,7 @@ class MongoTestThing(Thing): def test_mongo_list_property(self): class MongoTestThing(Thing): list_prop = Property(default=[1, 2], db_persist=True) + instance = MongoTestThing(id="mongo_list", use_mongo_db=True) instance.list_prop = [3, 4, 5] value_from_db = instance.db_engine.get_property("list_prop") @@ -58,6 +65,7 @@ class MongoTestThing(Thing): def test_mongo_none_property(self): class MongoTestThing(Thing): none_prop = Property(default=None, db_persist=True, allow_None=True) + instance = MongoTestThing(id="mongo_none", use_mongo_db=True) instance.none_prop = None value_from_db = instance.db_engine.get_property("none_prop") @@ -69,12 +77,15 @@ def test_mongo_property_persistence(self): client = MongoClient("mongodb://localhost:27017") db = client["hololinked"] db["properties"].delete_many({"id": thing_id, "name": prop_name}) + class MongoTestThing(Thing): test_prop_unique = Property(default=123, db_persist=True) + instance = MongoTestThing(id=thing_id, use_mongo_db=True) instance.test_prop_unique = 456 value_from_db = instance.db_engine.get_property(prop_name) self.assertEqual(value_from_db, 456) + if __name__ == "__main__": unittest.main() From e448fee9d204a7163fc2d0da56b7855d9e89df7b Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Tue, 25 Nov 2025 08:45:00 +0100 Subject: [PATCH 14/15] update changelog and fix exception import for client --- CHANGELOG.md | 3 +- hololinked/client/exceptions.py | 8 ++ .../client/zmq/consumed_interactions.py | 3 +- pyproject.toml | 32 +++---- uv.lock | 88 ++++--------------- 5 files changed, 42 insertions(+), 92 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 388f83e3..f835cfe7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [v0.3.8] - 2025-11-15 - supports structlog for logging, with colored logs and updated log statements -- moves dependencies and imports to a more hexagonal architecture +- SAST with bandit & gitleaks integrated into CI/CD pipelines - uses pytest instead of unittests +- code improvements with isort, dependency refactoring etc. ## [v0.3.7] - 2025-10-30 diff --git a/hololinked/client/exceptions.py b/hololinked/client/exceptions.py index 2bde853f..7540a95f 100644 --- a/hololinked/client/exceptions.py +++ b/hololinked/client/exceptions.py @@ -2,3 +2,11 @@ class ReplyNotArrivedError(Exception): """Exception raised when a reply is not received in time.""" pass + + +class BreakLoop(Exception): + """ + raise and catch to exit a loop from within another function or method + """ + + pass diff --git a/hololinked/client/zmq/consumed_interactions.py b/hololinked/client/zmq/consumed_interactions.py index e61cc769..4c69f413 100644 --- a/hololinked/client/zmq/consumed_interactions.py +++ b/hololinked/client/zmq/consumed_interactions.py @@ -18,6 +18,7 @@ from ...core.zmq.brokers import ( AsyncEventConsumer, AsyncZMQClient, + BreakLoop, EventConsumer, SyncZMQClient, ) @@ -31,7 +32,7 @@ from ...serializers.payloads import SerializableData from ...td import ActionAffordance, EventAffordance, PropertyAffordance from ...td.forms import Form -from ..exceptions import BreakLoop, ReplyNotArrivedError +from ..exceptions import ReplyNotArrivedError __error_message_types__ = [TIMEOUT, ERROR, INVALID_MESSAGE] diff --git a/pyproject.toml b/pyproject.toml index 499ce739..f1553724 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,28 +40,19 @@ dependencies = [ "sniffio>=1.3.1,<2.0", "aiomqtt>=2.4.0", "structlog>=25.5.0", -] - -[project.optional-dependencies] -generic = [ + # generic additional "ifaddr>=0.2.0,<0.3", "fastjsonschema==2.20.0", "serpent>=1.41,<2.0", -] -security = [ + # security "bcrypt==4.3.0", "argon2-cffi>=23.1.0", -] -db = [ + # DB "sqlalchemy>2.0.21", "sqlalchemy-utils>=0.41", "psycopg2-binary>=2.9.11", "pymongo>=4.15.2", ] -linux = [ - "uvloop==0.20.0" -] - [project.urls] Documentation = "https://docs.hololinked.dev" @@ -82,14 +73,12 @@ packages = [ [dependency-groups] dev = [ - "ConfigParser==7.1.0", "ipython==8.12.3", + "jupyter>=1.1.1", "numpy>=2.0.0", "pandas==2.2.3", - "faker==37.5.0", "pip>=25.2", "ruff>=0.12.10", - "jupyter>=1.1.1", ] test = [ "requests==2.32.3", @@ -133,6 +122,13 @@ exclude = [ "hololinked/param" ] +[tool.ruff.lint] +extend-select = ["I"] + +[tool.ruff.lint.isort] +lines-between-types = 1 +lines-after-imports = 2 + [tool.bandit] exclude_dirs = [ ".venv", @@ -146,9 +142,3 @@ exclude_dirs = [ "tests" ] -[tool.ruff.lint] -extend-select = ["I"] - -[tool.ruff.lint.isort] -lines-between-types = 1 -lines-after-imports = 2 diff --git a/uv.lock b/uv.lock index aaeeb7ca..7d1e210c 100644 --- a/uv.lock +++ b/uv.lock @@ -421,15 +421,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294 }, ] -[[package]] -name = "configparser" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a5/2e/a8d83652990ecb5df54680baa0c53d182051d9e164a25baa0582363841d1/configparser-7.1.0.tar.gz", hash = "sha256:eb82646c892dbdf773dae19c633044d163c3129971ae09b49410a303b8e0a5f7", size = 50122 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/df/1514580907b0bac0970415e5e24ef96a9c1fa71dcf2aa0139045b58fae9a/configparser-7.1.0-py3-none-any.whl", hash = "sha256:98e374573c4e10e92399651e3ba1c47a438526d633c44ee96143dec26dad4299", size = 17074 }, -] - [[package]] name = "coverage" version = "7.8.0" @@ -641,38 +632,27 @@ version = "0.3.8" source = { editable = "." } dependencies = [ { name = "aiomqtt" }, + { name = "argon2-cffi" }, + { name = "bcrypt" }, + { name = "fastjsonschema" }, { name = "httpx" }, + { name = "ifaddr" }, { name = "jsonschema" }, { name = "msgspec" }, + { name = "psycopg2-binary" }, { name = "pydantic" }, + { name = "pymongo" }, { name = "pyzmq" }, + { name = "serpent" }, { name = "sniffio" }, - { name = "structlog" }, - { name = "tornado" }, -] - -[package.optional-dependencies] -db = [ - { name = "psycopg2-binary" }, - { name = "pymongo" }, { name = "sqlalchemy" }, { name = "sqlalchemy-utils" }, -] -generic = [ - { name = "fastjsonschema" }, - { name = "ifaddr" }, - { name = "serpent" }, -] -security = [ - { name = "argon2-cffi" }, - { name = "bcrypt" }, + { name = "structlog" }, + { name = "tornado" }, ] [package.dev-dependencies] dev = [ - { name = "bcrypt" }, - { name = "configparser" }, - { name = "faker" }, { name = "ipython" }, { name = "jupyter" }, { name = "numpy" }, @@ -680,14 +660,10 @@ dev = [ { name = "pip" }, { name = "ruff" }, ] -linux = [ - { name = "uvloop" }, -] scanning = [ { name = "bandit" }, ] test = [ - { name = "coverage" }, { name = "faker" }, { name = "numpy" }, { name = "pytest" }, @@ -700,31 +676,27 @@ test = [ [package.metadata] requires-dist = [ { name = "aiomqtt", specifier = ">=2.4.0" }, - { name = "argon2-cffi", marker = "extra == 'security'", specifier = ">=23.1.0" }, - { name = "bcrypt", marker = "extra == 'security'", specifier = "==4.3.0" }, - { name = "fastjsonschema", marker = "extra == 'generic'", specifier = "==2.20.0" }, + { name = "argon2-cffi", specifier = ">=23.1.0" }, + { name = "bcrypt", specifier = "==4.3.0" }, + { name = "fastjsonschema", specifier = "==2.20.0" }, { name = "httpx", specifier = ">=0.28.1,<29.0" }, - { name = "ifaddr", marker = "extra == 'generic'", specifier = ">=0.2.0,<0.3" }, + { name = "ifaddr", specifier = ">=0.2.0,<0.3" }, { name = "jsonschema", specifier = ">=4.22.0,<5.0" }, { name = "msgspec", specifier = ">=0.18.6" }, - { name = "psycopg2-binary", marker = "extra == 'db'", specifier = ">=2.9.11" }, + { name = "psycopg2-binary", specifier = ">=2.9.11" }, { name = "pydantic", specifier = ">=2.8.0,<3.0.0" }, - { name = "pymongo", marker = "extra == 'db'", specifier = ">=4.15.2" }, + { name = "pymongo", specifier = ">=4.15.2" }, { name = "pyzmq", specifier = ">=25.1.0,<26.2" }, - { name = "serpent", marker = "extra == 'generic'", specifier = "==1.41" }, + { name = "serpent", specifier = ">=1.41,<2.0" }, { name = "sniffio", specifier = ">=1.3.1,<2.0" }, - { name = "sqlalchemy", marker = "extra == 'db'", specifier = ">2.0.21" }, - { name = "sqlalchemy-utils", marker = "extra == 'db'", specifier = ">=0.41" }, + { name = "sqlalchemy", specifier = ">2.0.21" }, + { name = "sqlalchemy-utils", specifier = ">=0.41" }, { name = "structlog", specifier = ">=25.5.0" }, { name = "tornado", specifier = ">=6.3.3" }, ] -provides-extras = ["generic", "security", "db"] [package.metadata.requires-dev] dev = [ - { name = "bcrypt", specifier = "==4.3.0" }, - { name = "configparser", specifier = "==7.1.0" }, - { name = "faker", specifier = "==37.5.0" }, { name = "ipython", specifier = "==8.12.3" }, { name = "jupyter", specifier = ">=1.1.1" }, { name = "numpy", specifier = ">=2.0.0" }, @@ -732,10 +704,8 @@ dev = [ { name = "pip", specifier = ">=25.2" }, { name = "ruff", specifier = ">=0.12.10" }, ] -linux = [{ name = "uvloop", specifier = "==0.20.0" }] -scanning = [{ name = "bandit" }] +scanning = [{ name = "bandit", specifier = ">=1.9.1" }] test = [ - { name = "coverage", specifier = "==7.8.0" }, { name = "faker", specifier = "==37.5.0" }, { name = "numpy", specifier = ">=2.0.0" }, { name = "pytest", specifier = ">=8.0.0" }, @@ -2626,26 +2596,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795 }, ] -[[package]] -name = "uvloop" -version = "0.20.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bc/f1/dc9577455e011ad43d9379e836ee73f40b4f99c02946849a44f7ae64835e/uvloop-0.20.0.tar.gz", hash = "sha256:4603ca714a754fc8d9b197e325db25b2ea045385e8a3ad05d3463de725fdf469", size = 2329938 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/bf/45828beccf685b7ed9638d9b77ef382b470c6ca3b5bff78067e02ffd5663/uvloop-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e50289c101495e0d1bb0bfcb4a60adde56e32f4449a67216a1ab2750aa84f037", size = 1320593 }, - { url = "https://files.pythonhosted.org/packages/27/c0/3c24e50bee7802a2add96ca9f0d5eb0ebab07e0a5615539d38aeb89499b9/uvloop-0.20.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e237f9c1e8a00e7d9ddaa288e535dc337a39bcbf679f290aee9d26df9e72bce9", size = 736676 }, - { url = "https://files.pythonhosted.org/packages/83/ce/ffa3c72954eae36825acfafd2b6a9221d79abd2670c0d25e04d6ef4a2007/uvloop-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:746242cd703dc2b37f9d8b9f173749c15e9a918ddb021575a0205ec29a38d31e", size = 3494573 }, - { url = "https://files.pythonhosted.org/packages/46/6d/4caab3a36199ba52b98d519feccfcf48921d7a6649daf14a93c7e77497e9/uvloop-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82edbfd3df39fb3d108fc079ebc461330f7c2e33dbd002d146bf7c445ba6e756", size = 3489932 }, - { url = "https://files.pythonhosted.org/packages/e4/4f/49c51595bd794945c88613df88922c38076eae2d7653f4624aa6f4980b07/uvloop-0.20.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:80dc1b139516be2077b3e57ce1cb65bfed09149e1d175e0478e7a987863b68f0", size = 4185596 }, - { url = "https://files.pythonhosted.org/packages/b8/94/7e256731260d313f5049717d1c4582d52a3b132424c95e16954a50ab95d3/uvloop-0.20.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4f44af67bf39af25db4c1ac27e82e9665717f9c26af2369c404be865c8818dcf", size = 4185746 }, - { url = "https://files.pythonhosted.org/packages/2d/64/31cbd379d6e260ac8de3f672f904e924f09715c3f192b09f26cc8e9f574c/uvloop-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4b75f2950ddb6feed85336412b9a0c310a2edbcf4cf931aa5cfe29034829676d", size = 1324302 }, - { url = "https://files.pythonhosted.org/packages/1e/6b/9207e7177ff30f78299401f2e1163ea41130d4fd29bcdc6d12572c06b728/uvloop-0.20.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:77fbc69c287596880ecec2d4c7a62346bef08b6209749bf6ce8c22bbaca0239e", size = 738105 }, - { url = "https://files.pythonhosted.org/packages/c1/ba/b64b10f577519d875992dc07e2365899a1a4c0d28327059ce1e1bdfb6854/uvloop-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6462c95f48e2d8d4c993a2950cd3d31ab061864d1c226bbf0ee2f1a8f36674b9", size = 4090658 }, - { url = "https://files.pythonhosted.org/packages/0a/f8/5ceea6876154d926604f10c1dd896adf9bce6d55a55911364337b8a5ed8d/uvloop-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:649c33034979273fa71aa25d0fe120ad1777c551d8c4cd2c0c9851d88fcb13ab", size = 4173357 }, - { url = "https://files.pythonhosted.org/packages/18/b2/117ab6bfb18274753fbc319607bf06e216bd7eea8be81d5bac22c912d6a7/uvloop-0.20.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3a609780e942d43a275a617c0839d85f95c334bad29c4c0918252085113285b5", size = 4029868 }, - { url = "https://files.pythonhosted.org/packages/6f/52/deb4be09060637ef4752adaa0b75bf770c20c823e8108705792f99cd4a6f/uvloop-0.20.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aea15c78e0d9ad6555ed201344ae36db5c63d428818b4b2a42842b3870127c00", size = 4115980 }, -] - [[package]] name = "wcwidth" version = "0.2.14" From f65e06881abbb0e6ccd911b623805e9910a1e3c3 Mon Sep 17 00:00:00 2001 From: Vignesh Venkatasubramanian Vaidyanathan <62492557+VigneshVSV@users.noreply.github.com> Date: Tue, 25 Nov 2025 08:52:15 +0100 Subject: [PATCH 15/15] cleanup --- .github/workflows/ci-pipeline.yml | 1 - pyproject.toml | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 460549f9..9c3a746b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -100,7 +100,6 @@ jobs: test: name: unit-integration tests needs: scan - if: false strategy: matrix: diff --git a/pyproject.toml b/pyproject.toml index f1553724..66809b44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ version = "0.3.8" authors = [ {name = "Vignesh Vaidyanathan", email = "info@hololinked.dev"}, ] -description = "A ZMQ-based protocol-agnostic object oriented RPC toolkit primarily focussed for instrumentation control, data acquisition or IoT, however can be used for controlling generic python objects." +description = "A ZMQ-based protocol-agnostic object oriented RPC toolkit primarily focussed for instrumentation control, data acquisition or IoT." readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" @@ -93,7 +93,6 @@ scanning = [ "bandit>=1.9.1", ] - [tool.pytest.ini_options] minversion = "8.0" addopts = "-ra --strict-markers --strict-config --ignore=lib64"