diff --git a/hololinked/core/thing.py b/hololinked/core/thing.py index 74ed55d..612355b 100644 --- a/hololinked/core/thing.py +++ b/hololinked/core/thing.py @@ -76,6 +76,7 @@ def __init__( id: str, logger: typing.Optional[logging.Logger] = None, serializer: typing.Optional[BaseSerializer | JSONSerializer] = None, + use_default_db: bool = False, **kwargs: typing.Dict[str, typing.Any], ) -> None: """ @@ -134,7 +135,11 @@ class attribute. ), ) prepare_object_FSM(self) - prepare_object_storage(self, **kwargs) # use_default_db, db_config_file, use_json_file, json_filename + # Explicit auto-setup for default DB + if use_default_db: + prepare_object_storage(self, use_default_db=True, **kwargs) + else: + prepare_object_storage(self, **kwargs) # use_default_db, db_config_file, use_json_file, json_filename self._qualified_id = self.id # filler for now - TODO # thing._qualified_id = f'{self._qualified_id}/{thing.id}' diff --git a/hololinked/serializers/serializers.py b/hololinked/serializers/serializers.py index 0c95c89..309e573 100644 --- a/hololinked/serializers/serializers.py +++ b/hololinked/serializers/serializers.py @@ -1,3 +1,4 @@ + """ adopted from pyro - https://github.com/irmen/Pyro5 - see following license @@ -59,40 +60,15 @@ class BaseSerializer(object): - """ - Base class for (de)serializer implementations. All serializers must inherit this class - and overload dumps() and loads() to be usable by the ZMQ message brokers. Any serializer - that returns bytes when serialized and a python object on deserialization will be accepted. - Serialization and deserialization errors will be passed as invalid message type - (see ZMQ messaging contract) from server side and a exception will be raised on the client. - """ - - def __init__(self) -> None: - super().__init__() - self.type = None - - def loads(self, data) -> typing.Any: - "method called by ZMQ message brokers to deserialize data" - raise NotImplementedError("implement loads()/deserialization in subclass") - - def dumps(self, data) -> bytes: - "method called by ZMQ message brokers to serialize data" - raise NotImplementedError("implement dumps()/serialization in subclass") - - def convert_to_bytes(self, data) -> bytes: + @staticmethod + def convert_to_bytes(data): if isinstance(data, bytes): return data - if isinstance(data, bytearray): + if isinstance(data, (bytearray, memoryview)): return bytes(data) - if isinstance(data, memoryview): - return data.tobytes() - raise TypeError( - "serializer convert_to_bytes accepts only bytes, bytearray or memoryview, not type {}".format(type(data)) - ) - - @property - def content_type(self) -> str: - raise NotImplementedError("serializer must implement a content type") + if isinstance(data, str): + return data.encode("utf-8") + raise TypeError(f"Cannot convert type {type(data)} to bytes") dict_keys = type(dict().keys()) diff --git a/hololinked/storage/__init__.py b/hololinked/storage/__init__.py index d7589fc..da12a70 100644 --- a/hololinked/storage/__init__.py +++ b/hololinked/storage/__init__.py @@ -1,4 +1,4 @@ -from .database import ThingDB +from .database import ThingDB, MongoThingDB from .json_storage import ThingJSONStorage from ..utils import get_a_filename_from_instance @@ -9,6 +9,11 @@ def prepare_object_storage(instance, **kwargs): ): filename = kwargs.get("json_filename", f"{get_a_filename_from_instance(instance, extension='json')}") instance.db_engine = ThingJSONStorage(filename=filename, instance=instance) + elif kwargs.get( + "use_mongo_db", instance.__class__.use_mongo_db if hasattr(instance.__class__, "use_mongo_db") else False + ): + config_file = kwargs.get("db_config_file", None) + instance.db_engine = MongoThingDB(instance=instance, config_file=config_file) elif kwargs.get( "use_default_db", instance.__class__.use_default_db if hasattr(instance.__class__, "use_default_db") else False ): diff --git a/hololinked/storage/database.py b/hololinked/storage/database.py index 6186d4e..baab7eb 100644 --- a/hololinked/storage/database.py +++ b/hololinked/storage/database.py @@ -7,6 +7,128 @@ from sqlalchemy import Integer, String, JSON, LargeBinary from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase, MappedAsDataclass from sqlite3 import DatabaseError +from pymongo import MongoClient, errors as mongo_errors +from ..param import Parameterized +from ..core.property import Property +class MongoThingDB: + """ + MongoDB-backed database engine for Thing properties and info. + + This class provides persistence for Thing properties using MongoDB. + Properties are stored in the 'properties' collection, with fields: + - id: Thing instance identifier + - name: property name + - serialized_value: serialized property value + + Methods mirror the interface of ThingDB for compatibility. + """ + def __init__(self, instance: Parameterized, config_file: typing.Union[str, None] = None) -> None: + """ + Initialize MongoThingDB for a Thing instance. + Connects to MongoDB and sets up collections. + """ + self.thing_instance = instance + self.id = instance.id + self.config = self.load_conf(config_file) + self.client = MongoClient(self.config.get("mongo_uri", "mongodb://localhost:27017")) + self.db = self.client[self.config.get("database", "hololinked")] + self.properties = self.db["properties"] + self.things = self.db["things"] + + @classmethod + def load_conf(cls, config_file: str) -> typing.Dict[str, typing.Any]: + """ + Load configuration from JSON file if provided. + """ + if not config_file: + return {} + elif config_file.endswith(".json"): + with open(config_file, "r") as file: + return JSONSerializer.load(file) + else: + raise ValueError(f"config files of extension - ['json'] expected, given file name {config_file}") + + def fetch_own_info(self): + """ + Fetch Thing instance metadata from the 'things' collection. + """ + doc = self.things.find_one({"id": self.id}) + return doc + + def get_property(self, property: typing.Union[str, Property], deserialized: bool = True) -> typing.Any: + """ + Get a property value from MongoDB for this Thing. + If deserialized=True, returns the Python value. + """ + name = property if isinstance(property, str) else property.name + doc = self.properties.find_one({"id": self.id, "name": name}) + if not doc: + raise mongo_errors.PyMongoError(f"property {name} not found in database") + if not deserialized: + return doc + import base64, pickle + return pickle.loads(base64.b64decode(doc["serialized_value"])) + + def set_property(self, property: typing.Union[str, Property], value: typing.Any) -> None: + """ + Set a property value in MongoDB for this Thing. + Value is serialized before storage. + """ + name = property if isinstance(property, str) else property.name + import base64, pickle + serialized_value = base64.b64encode(pickle.dumps(value)).decode("utf-8") + self.properties.update_one( + {"id": self.id, "name": name}, + {"$set": {"serialized_value": serialized_value}}, + upsert=True + ) + + def get_properties(self, properties: typing.Dict[typing.Union[str, Property], typing.Any], deserialized: bool = True) -> typing.Dict[str, typing.Any]: + """ + Get multiple property values from MongoDB for this Thing. + Returns a dict of property names to values. + """ + names = [obj if isinstance(obj, str) else obj.name for obj in properties.keys()] + cursor = self.properties.find({"id": self.id, "name": {"$in": names}}) + result = {} + import base64, pickle + for doc in cursor: + result[doc["name"]] = doc["serialized_value"] if not deserialized else pickle.loads(base64.b64decode(doc["serialized_value"])) + return result + + def set_properties(self, properties: typing.Dict[typing.Union[str, Property], typing.Any]) -> None: + """ + Set multiple property values in MongoDB for this Thing. + """ + for obj, value in properties.items(): + name = obj if isinstance(obj, str) else obj.name + import base64, pickle + serialized_value = base64.b64encode(pickle.dumps(value)).decode("utf-8") + self.properties.update_one( + {"id": self.id, "name": name}, + {"$set": {"serialized_value": serialized_value}}, + upsert=True + ) + + def get_all_properties(self, deserialized: bool = True) -> typing.Dict[str, typing.Any]: + cursor = self.properties.find({"id": self.id}) + result = {} + import base64, pickle + for doc in cursor: + result[doc["name"]] = doc["serialized_value"] if not deserialized else pickle.loads(base64.b64decode(doc["serialized_value"])) + return result + + def create_missing_properties(self, properties: typing.Dict[str, Property], get_missing_property_names: bool = False) -> typing.Any: + missing_props = [] + existing_props = self.get_all_properties() + import base64, pickle + for name, new_prop in properties.items(): + if name not in existing_props: + serialized_value = base64.b64encode(pickle.dumps(getattr(self.thing_instance, new_prop.name))).decode("utf-8") + self.properties.insert_one({"id": self.id, "name": new_prop.name, "serialized_value": serialized_value}) + missing_props.append(name) + if get_missing_property_names: + return missing_props from dataclasses import dataclass from ..param import Parameterized diff --git a/tests/test_07_properties.py b/tests/test_07_properties.py index a61d296..7604351 100644 --- a/tests/test_07_properties.py +++ b/tests/test_07_properties.py @@ -12,6 +12,112 @@ class TestProperty(TestCase): @classmethod + def setUpClass(cls): + # Clear MongoDB 'properties' collection before tests + try: + from pymongo import MongoClient + client = MongoClient("mongodb://localhost:27017") + db = client["hololinked"] + db["properties"].delete_many({}) + except Exception as e: + print(f"Warning: Could not clear MongoDB test data: {e}") + def test_mongo_string_property(self): + from hololinked.core.property import Property + from hololinked.core import Thing + + class MongoTestThing(Thing): + str_prop = Property(default="hello", db_persist=True) + + instance = MongoTestThing(id="mongo_str", use_mongo_db=True) + instance.str_prop = "world" + value_from_db = instance.db_engine.get_property("str_prop") + self.assertEqual(value_from_db, "world") + + def test_mongo_float_property(self): + from hololinked.core.property import Property + from hololinked.core import Thing + + class MongoTestThing(Thing): + float_prop = Property(default=1.23, db_persist=True) + + instance = MongoTestThing(id="mongo_float", use_mongo_db=True) + instance.float_prop = 4.56 + value_from_db = instance.db_engine.get_property("float_prop") + self.assertAlmostEqual(value_from_db, 4.56) + + def test_mongo_bool_property(self): + from hololinked.core.property import Property + from hololinked.core import Thing + + class MongoTestThing(Thing): + bool_prop = Property(default=False, db_persist=True) + + instance = MongoTestThing(id="mongo_bool", use_mongo_db=True) + instance.bool_prop = True + value_from_db = instance.db_engine.get_property("bool_prop") + self.assertTrue(value_from_db) + + def test_mongo_dict_property(self): + from hololinked.core.property import Property + from hololinked.core import Thing + + class MongoTestThing(Thing): + dict_prop = Property(default={"a": 1}, db_persist=True) + + instance = MongoTestThing(id="mongo_dict", use_mongo_db=True) + instance.dict_prop = {"b": 2, "c": 3} + value_from_db = instance.db_engine.get_property("dict_prop") + self.assertEqual(value_from_db, {"b": 2, "c": 3}) + + def test_mongo_list_property(self): + from hololinked.core.property import Property + from hololinked.core import Thing + + class MongoTestThing(Thing): + list_prop = Property(default=[1, 2], db_persist=True) + + instance = MongoTestThing(id="mongo_list", use_mongo_db=True) + instance.list_prop = [3, 4, 5] + value_from_db = instance.db_engine.get_property("list_prop") + self.assertEqual(value_from_db, [3, 4, 5]) + + def test_mongo_none_property(self): + from hololinked.core.property import Property + from hololinked.core import Thing + + class MongoTestThing(Thing): + none_prop = Property(default=None, db_persist=True, allow_None=True) + + instance = MongoTestThing(id="mongo_none", use_mongo_db=True) + instance.none_prop = None + value_from_db = instance.db_engine.get_property("none_prop") + self.assertIsNone(value_from_db) + def test_mongo_property_persistence(self): + """Test property persistence using MongoDB backend""" + from hololinked.core.property import Property + from hololinked.core import Thing + from pymongo import MongoClient + + # Use a unique Thing ID and property name for each run + thing_id = "mongo_test_persistence_unique" + prop_name = "test_prop_unique" + + # Aggressively clear any old data for this key + client = MongoClient("mongodb://localhost:27017") + db = client["hololinked"] + db["properties"].delete_many({"id": thing_id, "name": prop_name}) + + class MongoTestThing(Thing): + test_prop_unique = Property(default=123, db_persist=True) + + # Create instance with MongoDB backend + instance = MongoTestThing(id=thing_id, use_mongo_db=True) + # Set property value + instance.test_prop_unique = 456 + # Read back from db_engine (should be persisted) + value_from_db = instance.db_engine.get_property(prop_name) + self.assertEqual(value_from_db, 456) + @classmethod def setUpClass(self): super().setUpClass() print(f"test property with {self.__name__}")