diff --git a/packages/core/minos-microservice-aggregate/tests/test_aggregate/test_events/test_repositories/test_pg.py b/packages/core/minos-microservice-aggregate/tests/test_aggregate/test_events/test_repositories/test_pg.py index 37c6be8bf..bb04300c0 100644 --- a/packages/core/minos-microservice-aggregate/tests/test_aggregate/test_events/test_repositories/test_pg.py +++ b/packages/core/minos-microservice-aggregate/tests/test_aggregate/test_events/test_repositories/test_pg.py @@ -44,7 +44,7 @@ def build_event_repository(self) -> EventRepository: return PostgreSqlEventRepository(**self.repository_db) def test_constructor(self): - repository = PostgreSqlEventRepository("host", 1234, "database", "user", "password") + repository = PostgreSqlEventRepository("database", "host", 1234, "user", "password") self.assertIsInstance(repository, EventRepository) self.assertEqual("host", repository.host) self.assertEqual(1234, repository.port) diff --git a/packages/core/minos-microservice-aggregate/tests/test_aggregate/test_transactions/test_repositories/test_pg.py b/packages/core/minos-microservice-aggregate/tests/test_aggregate/test_transactions/test_repositories/test_pg.py index febd8fba0..2bcace111 100644 --- a/packages/core/minos-microservice-aggregate/tests/test_aggregate/test_transactions/test_repositories/test_pg.py +++ b/packages/core/minos-microservice-aggregate/tests/test_aggregate/test_transactions/test_repositories/test_pg.py @@ -43,7 +43,7 @@ async def test_subclass(self) -> None: self.assertTrue(issubclass(PostgreSqlTransactionRepository, TransactionRepository)) def test_constructor(self): - repository = PostgreSqlTransactionRepository("host", 1234, "database", "user", "password") + repository = PostgreSqlTransactionRepository("database", "host", 1234, "user", "password") self.assertIsInstance(repository, PostgreSqlTransactionRepository) self.assertEqual("host", repository.host) self.assertEqual(1234, repository.port) diff --git a/packages/core/minos-microservice-common/minos/common/config/abc.py b/packages/core/minos-microservice-common/minos/common/config/abc.py index cedfa0587..9c4714094 100644 --- a/packages/core/minos-microservice-common/minos/common/config/abc.py +++ b/packages/core/minos-microservice-common/minos/common/config/abc.py @@ -23,6 +23,7 @@ from typing import ( TYPE_CHECKING, Any, + Optional, Union, ) @@ -43,6 +44,8 @@ InjectableMixin, ) +sentinel = object() + @Injectable("config") class Config(ABC): @@ -50,6 +53,8 @@ class Config(ABC): __slots__ = ("_file_path", "_data", "_with_environment", "_parameterized") + DEFAULT_VALUES: dict[str, Any] = dict() + def __init__(self, path: Union[Path, str], with_environment: bool = True, **kwargs): super().__init__() if isinstance(path, str): @@ -282,7 +287,7 @@ def get_by_key(self, key: str) -> Any: :return: A value instance. """ - def _fn(k: str, data: dict[str, Any], previous: str) -> Any: + def _fn(k: str, data: dict[str, Any], previous: str = "", default: Optional[Any] = sentinel) -> Any: current, _sep, following = k.partition(".") full = f"{previous}.{current}".lstrip(".") @@ -293,20 +298,33 @@ def _fn(k: str, data: dict[str, Any], previous: str) -> Any: with suppress(KeyError): return os.environ[self._to_environment_variable(full)] - part = data[current] - if not following: - if not isinstance(part, dict): - return part + if default is not sentinel and current in default: + default_part = default[current] + else: + default_part = sentinel + + if current not in data and default_part is not sentinel: + part = default_part + else: + part = data[current] + + if following: + return _fn(following, part, full, default_part) + + if not isinstance(part, dict): + return part - result = dict() - for subpart in part: - result[subpart] = _fn(subpart, part, full) - return result + keys = part.keys() + if isinstance(default_part, dict): + keys |= default_part.keys() - return _fn(following, part, full) + result = dict() + for subpart in keys: + result[subpart] = _fn(subpart, part, full, default_part) + return result try: - return _fn(key, self._data, str()) + return _fn(key, self._data, default=self.DEFAULT_VALUES) except Exception: raise MinosConfigException(f"{key!r} field is not defined on the configuration!") diff --git a/packages/core/minos-microservice-common/minos/common/config/v1.py b/packages/core/minos-microservice-common/minos/common/config/v1.py index fcf578d60..3825b72d2 100644 --- a/packages/core/minos-microservice-common/minos/common/config/v1.py +++ b/packages/core/minos-microservice-common/minos/common/config/v1.py @@ -5,9 +5,6 @@ from contextlib import ( suppress, ) -from pathlib import ( - Path, -) from typing import ( TYPE_CHECKING, Any, @@ -156,12 +153,14 @@ def _get_interface_http(self) -> dict[str, Any]: except Exception as exc: raise MinosConfigException(f"The 'http' interface is not available: {exc!r}") + try: + connector = self.get_by_key("rest") + except MinosConfigException: + connector = dict() + return { "port": import_module(port), - "connector": { - "host": self.get_by_key("rest.host"), - "port": int(self.get_by_key("rest.port")), - }, + "connector": connector, } def _get_interface_broker(self) -> dict[str, Any]: @@ -170,18 +169,27 @@ def _get_interface_broker(self) -> dict[str, Any]: except Exception as exc: raise MinosConfigException(f"The 'broker' interface is not available: {exc!r}") + try: + common = self.get_by_key("broker") + except MinosConfigException: + common = dict() + + try: + common["queue"] = self.get_by_key("broker.queue") + common["queue"].pop("database", None) + common["queue"].pop("port", None) + common["queue"].pop("host", None) + common["queue"].pop("port", None) + common["queue"].pop("user", None) + common["queue"].pop("password", None) + except MinosConfigException: + common["queue"] = dict() + return { "port": import_module(port), "publisher": dict(), "subscriber": dict(), - "common": { - "host": self.get_by_key("broker.host"), - "port": int(self.get_by_key("broker.port")), - "queue": { - "records": int(self.get_by_key("broker.queue.records")), - "retry": int(self.get_by_key("broker.queue.retry")), - }, - }, + "common": common, } def _get_interface_periodic(self): @@ -254,10 +262,8 @@ def _get_database_broker(self): return self._get_database_by_name("broker.queue") def _get_database_saga(self) -> dict[str, Any]: - raw = self.get_by_key("saga.storage.path") - return { - "path": Path(raw) if raw.startswith("/") else self.file_path.parent / raw, - } + raw = self.get_by_key("saga.storage") + return raw def _get_database_event(self) -> dict[str, Any]: return self._get_database_by_name("repository") @@ -269,20 +275,15 @@ def _get_database_snapshot(self) -> dict[str, Any]: return self._get_database_by_name("snapshot") def _get_database_by_name(self, prefix: str): - return { - "database": self.get_by_key(f"{prefix}.database"), - "user": self.get_by_key(f"{prefix}.user"), - "password": self.get_by_key(f"{prefix}.password"), - "host": self.get_by_key(f"{prefix}.host"), - "port": int(self.get_by_key(f"{prefix}.port")), - } + data = self.get_by_key(prefix) + data.pop("records", None) + data.pop("retry", None) + return data def _get_discovery(self) -> dict[str, Any]: - return { - "client": self.get_type_by_key("discovery.client"), - "host": self.get_by_key("discovery.host"), - "port": self.get_by_key("discovery.port"), - } + data = self.get_by_key("discovery") + data["client"] = self.get_type_by_key("discovery.client") + return data def _to_parameterized_variable(self, key: str) -> str: return self._PARAMETERIZED_MAPPER[key] diff --git a/packages/core/minos-microservice-common/minos/common/config/v2.py b/packages/core/minos-microservice-common/minos/common/config/v2.py index 0456c1abf..246feee6e 100644 --- a/packages/core/minos-microservice-common/minos/common/config/v2.py +++ b/packages/core/minos-microservice-common/minos/common/config/v2.py @@ -8,9 +8,6 @@ from copy import ( deepcopy, ) -from pathlib import ( - Path, -) from typing import ( TYPE_CHECKING, Any, @@ -98,15 +95,8 @@ def _get_injections(self) -> list[type[InjectableMixin]]: def _get_databases(self) -> dict[str, dict[str, Any]]: data = deepcopy(self.get_by_key("databases")) - - if "saga" in data: - if "path" in data["saga"]: - data["saga"]["path"] = self._str_to_path(data["saga"]["path"]) return data - def _str_to_path(self, raw: str) -> Path: - return Path(raw) if raw.startswith("/") else self.file_path.parent / raw - def _get_interfaces(self) -> dict[str, dict[str, Any]]: data = deepcopy(self.get_by_key("interfaces")) diff --git a/packages/core/minos-microservice-common/minos/common/database/abc.py b/packages/core/minos-microservice-common/minos/common/database/abc.py index 1a9cc7439..942c863ac 100644 --- a/packages/core/minos-microservice-common/minos/common/database/abc.py +++ b/packages/core/minos-microservice-common/minos/common/database/abc.py @@ -32,17 +32,66 @@ class PostgreSqlMinosDatabase(SetupMixin): """PostgreSql Minos Database base class.""" - def __init__(self, host: str, port: int, database: str, user: str, password: str, *args, **kwargs): + def __init__( + self, + database: str, + host: Optional[str] = None, + port: Optional[int] = None, + user: Optional[str] = None, + password: Optional[str] = None, + *args, + **kwargs, + ): super().__init__(*args, **kwargs) - self.host = host - self.port = port - self.database = database - self.user = user - self.password = password + self._database = database + self._host = host + self._port = port + self._user = user + self._password = password self._pool = None self._owned_pool = False + @property + def database(self) -> str: + """Get the database's database. + + :return: A ``str`` value. + """ + return self.pool.database + + @property + def host(self) -> str: + """Get the database's host. + + :return: A ``str`` value. + """ + return self.pool.host + + @property + def port(self) -> int: + """Get the database's port. + + :return: An ``int`` value. + """ + return self.pool.port + + @property + def user(self) -> str: + """Get the database's user. + + :return: A ``str`` value. + """ + return self.pool.user + + @property + def password(self) -> str: + """Get the database's password. + + :return: A ``str`` value. + """ + return self.pool.password + async def _destroy(self) -> None: if self._owned_pool: await self._pool.destroy() @@ -178,6 +227,6 @@ def _build_pool(self, pool: PostgreSqlPool = None) -> tuple[PostgreSqlPool, bool return pool, False pool = PostgreSqlPool( - host=self.host, port=self.port, database=self.database, user=self.user, password=self.password + host=self._host, port=self._port, database=self._database, user=self._user, password=self._password ) return pool, True diff --git a/packages/core/minos-microservice-common/minos/common/database/pools.py b/packages/core/minos-microservice-common/minos/common/database/pools.py index 13a8c11d8..7dfcc5138 100644 --- a/packages/core/minos-microservice-common/minos/common/database/pools.py +++ b/packages/core/minos-microservice-common/minos/common/database/pools.py @@ -40,11 +40,30 @@ class PostgreSqlPool(Pool[ContextManager]): """Postgres Pool class.""" - def __init__(self, host: str, port: int, database: str, user: str, password: str, *args, **kwargs): + def __init__( + self, + database: str, + host: Optional[str] = None, + port: Optional[int] = None, + user: Optional[str] = None, + password: Optional[str] = None, + *args, + **kwargs, + ): super().__init__(*args, **kwargs) + + if host is None: + host = "localhost" + if port is None: + port = 5432 + if user is None: + user = "postgres" + if password is None: + password = "" + + self.database = database self.host = host self.port = port - self.database = database self.user = user self.password = password diff --git a/packages/core/minos-microservice-common/minos/common/storage/lmdb.py b/packages/core/minos-microservice-common/minos/common/storage/lmdb.py index eebad0395..3e1a1ed26 100644 --- a/packages/core/minos-microservice-common/minos/common/storage/lmdb.py +++ b/packages/core/minos-microservice-common/minos/common/storage/lmdb.py @@ -97,7 +97,9 @@ def _get_table(self, table: str): return self._tables[table] @classmethod - def build(cls, path: Union[str, Path], max_db: int = 10, map_size: int = int(1e9), **kwargs) -> MinosStorageLmdb: + def build( + cls, path: Optional[Union[str, Path]] = None, max_db: int = 10, map_size: int = int(1e9), **kwargs + ) -> MinosStorageLmdb: """Build a new instance. :param path: Path in which the database is stored. @@ -106,6 +108,8 @@ def build(cls, path: Union[str, Path], max_db: int = 10, map_size: int = int(1e9 :param kwargs: Additional named arguments. :return: A ``MinosStorageLmdb`` instance. """ + if path is None: + path = ".lmdb" env: lmdb.Environment = lmdb.open(str(path), max_dbs=max_db, map_size=map_size) return cls(env, **kwargs) diff --git a/packages/core/minos-microservice-common/tests/test_common/test_config/test_abc.py b/packages/core/minos-microservice-common/tests/test_common/test_config/test_abc.py index f794ffcf4..df62c39e0 100644 --- a/packages/core/minos-microservice-common/tests/test_common/test_config/test_abc.py +++ b/packages/core/minos-microservice-common/tests/test_common/test_config/test_abc.py @@ -25,6 +25,8 @@ class _Config(Config): """For testing purposes.""" + DEFAULT_VALUES = {"foo": {"bar": 56}, "saga": {"name": "foobar"}} + # noinspection PyPropertyDefinition @property def _version(self) -> int: @@ -88,6 +90,13 @@ def test_file_raises(self): def test_get_by_key(self): self.assertEqual("Order", self.config.get_by_key("service.name")) + def test_get_by_key_with_default_without_overlap(self): + self.assertEqual(56, self.config.get_by_key("foo.bar")) + + def test_get_by_key_with_default_with_overlap(self): + expected = {"storage": {"path": "./order.lmdb"}, "name": "foobar"} + self.assertEqual(expected, self.config.get_by_key("saga")) + def test_get_by_key_raises(self): with self.assertRaises(MinosConfigException): self.assertEqual("Order", self.config.get_by_key("something")) diff --git a/packages/core/minos-microservice-common/tests/test_common/test_config/test_v1/test_base.py b/packages/core/minos-microservice-common/tests/test_common/test_config/test_v1/test_base.py index 3a7ae925f..f73eb24dc 100644 --- a/packages/core/minos-microservice-common/tests/test_common/test_config/test_v1/test_base.py +++ b/packages/core/minos-microservice-common/tests/test_common/test_config/test_v1/test_base.py @@ -82,6 +82,23 @@ def test_interface_http(self): } self.assertEqual(expected, observed) + def test_interface_http_connector_not_defined(self): + base = self.config.get_by_key + + def _fn(label): + if label == "rest": + raise MinosConfigException("") + return base(label) + + with patch.object(ConfigV1, "get_by_key", side_effect=_fn): + observed = self.config.get_interface_by_name("http") + + expected = { + "port": FakeHttpPort, + "connector": {}, + } + self.assertEqual(expected, observed) + def test_interface_http_not_defined(self): with patch.object(ConfigV1, "get_by_key", side_effect=MinosConfigException("")): with self.assertRaises(MinosConfigException): @@ -104,6 +121,46 @@ def test_interface_broker(self): self.assertEqual(expected, broker) + def test_interface_broker_section_not_defined(self): + base = self.config.get_by_key + + def _fn(label): + if label == "broker": + raise MinosConfigException("") + return base(label) + + with patch.object(ConfigV1, "get_by_key", side_effect=_fn): + observed = self.config.get_interface_by_name("broker") + + expected = { + "port": FakeBrokerPort, + "common": { + "queue": {"records": 10, "retry": 2}, + }, + "publisher": {}, + "subscriber": {}, + } + self.assertEqual(expected, observed) + + def test_interface_broker_queue_not_defined(self): + base = self.config.get_by_key + + def _fn(label): + if label == "broker.queue": + raise MinosConfigException("") + return base(label) + + with patch.object(ConfigV1, "get_by_key", side_effect=_fn): + observed = self.config.get_interface_by_name("broker") + + expected = { + "port": FakeBrokerPort, + "common": {"host": "localhost", "port": 9092, "queue": {}}, + "publisher": {}, + "subscriber": {}, + } + self.assertEqual(expected, observed) + def test_interface_broker_not_defined(self): with patch.object(ConfigV1, "get_by_key", side_effect=MinosConfigException("")): with self.assertRaises(MinosConfigException): @@ -210,7 +267,7 @@ def test_database_broker(self): def test_database_saga(self): config = ConfigV1(self.file_path, with_environment=False) saga = config.get_database_by_name("saga") - self.assertEqual(self.file_path.parent / "order.lmdb", saga["path"]) + self.assertEqual("./order.lmdb", saga["path"]) def test_discovery(self): config = ConfigV1(self.file_path, with_environment=False) diff --git a/packages/core/minos-microservice-common/tests/test_common/test_config/test_v2/test_base.py b/packages/core/minos-microservice-common/tests/test_common/test_config/test_v2/test_base.py index d9207bcca..5a583bedc 100644 --- a/packages/core/minos-microservice-common/tests/test_common/test_config/test_v2/test_base.py +++ b/packages/core/minos-microservice-common/tests/test_common/test_config/test_v2/test_base.py @@ -233,7 +233,7 @@ def test_database_broker(self): def test_database_saga(self): config = ConfigV2(self.file_path, with_environment=False) saga = config.get_database_by_name("saga") - self.assertEqual(self.file_path.parent / "order.lmdb", saga["path"]) + self.assertEqual("./order.lmdb", saga["path"]) def test_discovery(self): config = ConfigV2(self.file_path, with_environment=False) diff --git a/packages/core/minos-microservice-common/tests/test_common/test_database/test_abc.py b/packages/core/minos-microservice-common/tests/test_common/test_database/test_abc.py index 5da636f6f..ff3629df0 100644 --- a/packages/core/minos-microservice-common/tests/test_common/test_database/test_abc.py +++ b/packages/core/minos-microservice-common/tests/test_common/test_database/test_abc.py @@ -20,6 +20,14 @@ class TestPostgreSqlMinosDatabase(PostgresAsyncTestCase): CONFIG_FILE_PATH = CONFIG_FILE_PATH def test_constructor(self): + pool = PostgreSqlMinosDatabase("foo") + self.assertEqual("foo", pool.database) + self.assertEqual("postgres", pool.user) + self.assertEqual("", pool.password) + self.assertEqual("localhost", pool.host) + self.assertEqual(5432, pool.port) + + def test_constructor_extended(self): database = PostgreSqlMinosDatabase(**self.repository_db) self.assertEqual(self.repository_db["host"], database.host) self.assertEqual(self.repository_db["port"], database.port) diff --git a/packages/core/minos-microservice-common/tests/test_common/test_database/test_pools.py b/packages/core/minos-microservice-common/tests/test_common/test_database/test_pools.py index d55ab8f09..15e26fb64 100644 --- a/packages/core/minos-microservice-common/tests/test_common/test_database/test_pools.py +++ b/packages/core/minos-microservice-common/tests/test_common/test_database/test_pools.py @@ -32,6 +32,14 @@ def setUp(self) -> None: super().setUp() self.pool = PostgreSqlPool.from_config(self.config) + def test_constructor(self): + pool = PostgreSqlPool("foo") + self.assertEqual("foo", pool.database) + self.assertEqual("postgres", pool.user) + self.assertEqual("", pool.password) + self.assertEqual("localhost", pool.host) + self.assertEqual(5432, pool.port) + async def asyncSetUp(self): await super().asyncSetUp() await self.pool.setup() diff --git a/packages/core/minos-microservice-common/tests/test_common/test_storage/test_lmdb.py b/packages/core/minos-microservice-common/tests/test_common/test_storage/test_lmdb.py index 21e9ee7ea..7245670b5 100644 --- a/packages/core/minos-microservice-common/tests/test_common/test_storage/test_lmdb.py +++ b/packages/core/minos-microservice-common/tests/test_common/test_storage/test_lmdb.py @@ -1,5 +1,8 @@ import shutil import unittest +from pathlib import ( + Path, +) from minos.common import ( MinosStorageLmdb, @@ -15,6 +18,11 @@ def setUp(self) -> None: def tearDown(self) -> None: shutil.rmtree(self.path, ignore_errors=True) + shutil.rmtree(".lmdb", ignore_errors=True) + + def test_constructor_default_path(self): + MinosStorageLmdb.build() + self.assertTrue(Path(".lmdb").exists()) def test_storage_add_text(self): storage = MinosStorageLmdb.build(self.path) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py index dc393a397..d04a70973 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/collections/queues/pg.py @@ -57,9 +57,21 @@ class PostgreSqlBrokerQueue(BrokerQueue, PostgreSqlMinosDatabase): _queue: PriorityQueue[_Entry] - def __init__(self, *args, query_factory: PostgreSqlBrokerQueueQueryFactory, retry: int, records: int, **kwargs): + def __init__( + self, + *args, + query_factory: PostgreSqlBrokerQueueQueryFactory, + retry: Optional[int] = None, + records: Optional[int] = None, + **kwargs, + ): super().__init__(*args, **kwargs) + if retry is None: + retry = 2 + if records is None: + records = 1000 + self._query_factory = query_factory self._retry = retry self._records = records @@ -68,6 +80,22 @@ def __init__(self, *args, query_factory: PostgreSqlBrokerQueueQueryFactory, retr self._run_task = None + @property + def retry(self) -> int: + """Get the retry value. + + :return: A ``int`` value. + """ + return self._retry + + @property + def records(self) -> int: + """Get the records value. + + :return: A ``int`` value. + """ + return self._records + @property def query_factory(self) -> PostgreSqlBrokerQueueQueryFactory: """Get the query factory. @@ -79,7 +107,7 @@ def query_factory(self) -> PostgreSqlBrokerQueueQueryFactory: @classmethod def _from_config(cls, config: Config, **kwargs) -> PostgreSqlBrokerQueue: broker_interface = config.get_interface_by_name("broker") - queue_config = broker_interface["common"]["queue"] + queue_config = broker_interface.get("common", dict()).get("queue", dict()) database_config = config.get_database_by_name("broker") return cls(**(kwargs | database_config | queue_config)) diff --git a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py index 0253fcbaf..239b6522e 100644 --- a/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py +++ b/packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py @@ -137,7 +137,7 @@ def with_config(self, config: Config): :return: This method return the builder instance. """ self.kwargs |= config.get_database_by_name("broker") - self.kwargs |= config.get_interface_by_name("broker")["common"]["queue"] + self.kwargs |= config.get_interface_by_name("broker").get("common", dict()).get("queue", dict()) return super().with_config(config) def build(self) -> PostgreSqlBrokerSubscriberQueue: diff --git a/packages/core/minos-microservice-networks/minos/networks/discovery/connectors.py b/packages/core/minos-microservice-networks/minos/networks/discovery/connectors.py index 4b47ee50e..960639e09 100644 --- a/packages/core/minos-microservice-networks/minos/networks/discovery/connectors.py +++ b/packages/core/minos-microservice-networks/minos/networks/discovery/connectors.py @@ -14,6 +14,7 @@ ) from typing import ( Any, + Optional, ) from minos.common import ( @@ -42,8 +43,19 @@ class DiscoveryConnector(SetupMixin): """Discovery Connector class.""" - def __init__(self, client, name: str, host: str, port: int, endpoints: list[dict[str, Any]], *args, **kwargs): + def __init__( + self, + client: DiscoveryClient, + name: str, + endpoints: list[dict[str, Any]], + host: str, + port: Optional[int] = None, + *args, + **kwargs, + ): super().__init__(*args, **kwargs) + if port is None: + port = 8080 self.client = client @@ -60,15 +72,15 @@ def _from_config(cls, *args, config: Config, **kwargs) -> DiscoveryConnector: host = get_host_ip() endpoints = cls._endpoints_from_config(config) - return cls(client, name, host, port, endpoints, *args, **kwargs) + return cls(client, name, endpoints, host, port, *args, **kwargs) @classmethod def _client_from_config(cls, config: Config) -> DiscoveryClient: discovery_config = config.get_discovery() client_cls = cls._client_cls_from_config(discovery_config) - client_host = discovery_config["host"] - client_port = discovery_config["port"] + client_host = discovery_config.get("host") + client_port = discovery_config.get("port") return client_cls(host=client_host, port=client_port) @@ -80,10 +92,10 @@ def _client_cls_from_config(discovery_config: dict[str, Any]) -> type[DiscoveryC return client_cls @staticmethod - def _port_from_config(config: Config) -> int: + def _port_from_config(config: Config) -> Optional[int]: http_config = config.get_interface_by_name("http") connector_config = http_config["connector"] - port = connector_config["port"] + port = connector_config.get("port") return port @staticmethod diff --git a/packages/core/minos-microservice-networks/minos/networks/http/connectors.py b/packages/core/minos-microservice-networks/minos/networks/http/connectors.py index 0385fe572..cd2bbfe4c 100644 --- a/packages/core/minos-microservice-networks/minos/networks/http/connectors.py +++ b/packages/core/minos-microservice-networks/minos/networks/http/connectors.py @@ -56,23 +56,27 @@ class HttpConnector(ABC, SetupMixin, Generic[RawRequest, RawResponse]): """Http Application base class.""" - def __init__(self, host: str, port: int, adapter: HttpAdapter, **kwargs): + def __init__(self, adapter: HttpAdapter, host: Optional[str] = None, port: Optional[int] = None, **kwargs): super().__init__(**kwargs) + if host is None: + host = "0.0.0.0" + if port is None: + port = 8080 + self._adapter = adapter self._host = host self._port = port - self._adapter = adapter @classmethod def _from_config(cls, config: Config, **kwargs) -> HttpConnector: http_config = config.get_interface_by_name("http") connector_config = http_config["connector"] - host = connector_config["host"] - port = connector_config["port"] + host = connector_config.get("host") + port = connector_config.get("port") adapter = HttpAdapter.from_config(config) - return cls(host, port, adapter) + return cls(adapter, host, port) async def start(self) -> None: """Start the connector. diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_collections/test_queues/test_pg.py b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_collections/test_queues/test_pg.py index b5cc4309d..8a87f1f6a 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_collections/test_queues/test_pg.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_brokers/test_collections/test_queues/test_pg.py @@ -43,6 +43,13 @@ def setUp(self) -> None: def test_is_subclass(self): self.assertTrue(issubclass(PostgreSqlBrokerQueue, (BrokerQueue, PostgreSqlMinosDatabase))) + def test_constructor(self): + queue = PostgreSqlBrokerQueue("foo_db", query_factory=self.query_factory) + self.assertEqual("foo_db", queue.database) + self.assertEqual(self.query_factory, queue.query_factory) + self.assertEqual(2, queue.retry) + self.assertEqual(1000, queue.records) + async def test_query_factory(self): queue = PostgreSqlBrokerQueue.from_config(self.config, query_factory=self.query_factory) diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_discovery/test_connectors.py b/packages/core/minos-microservice-networks/tests/test_networks/test_discovery/test_connectors.py index 2daaaa147..304955766 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_discovery/test_connectors.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_discovery/test_connectors.py @@ -27,6 +27,15 @@ def setUp(self) -> None: self.ip = get_host_ip() self.discovery = DiscoveryConnector.from_config(config=self.config) + def test_constructor(self): + connector = DiscoveryConnector(self.discovery.client, "foo", [], "192.168.1.32") + + self.assertEqual(self.discovery.client, connector.client) + self.assertEqual("foo", connector.name) + self.assertEqual([], connector.endpoints) + self.assertEqual("192.168.1.32", connector.host) + self.assertEqual(8080, connector.port) + def test_config_minos_client_does_not_exist(self): config = Config(self.CONFIG_FILE_PATH, minos_discovery_client="wrong-client") with self.assertRaises(MinosImportException): diff --git a/packages/core/minos-microservice-networks/tests/test_networks/test_http/test_connectors.py b/packages/core/minos-microservice-networks/tests/test_networks/test_http/test_connectors.py index 1f4bcf490..573e445f2 100644 --- a/packages/core/minos-microservice-networks/tests/test_networks/test_http/test_connectors.py +++ b/packages/core/minos-microservice-networks/tests/test_networks/test_http/test_connectors.py @@ -63,6 +63,12 @@ def test_abstract(self): HttpConnector.__abstractmethods__, ) + def test_constructor(self): + connector = FakeHttpConnector(self.adapter) + self.assertEqual(self.adapter, connector.adapter) + self.assertEqual("0.0.0.0", connector.host) + self.assertEqual(8080, connector.port) + def test_host(self): self.assertEqual("localhost", self.connector.host) diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py index 50a28f841..44857db05 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py @@ -13,6 +13,9 @@ from functools import ( partial, ) +from typing import ( + Optional, +) from aiokafka import ( AIOKafkaProducer, @@ -59,21 +62,42 @@ def _from_config(cls, config: Config, **kwargs) -> InMemoryQueuedKafkaBrokerPubl class KafkaBrokerPublisher(BrokerPublisher, KafkaCircuitBreakerMixin): """Kafka Broker Publisher class.""" - def __init__(self, *args, broker_host: str, broker_port: int, **kwargs): + def __init__(self, *args, host: Optional[str] = None, port: Optional[int] = None, **kwargs): super().__init__(*args, **kwargs) - self.broker_host = broker_host - self.broker_port = broker_port + if host is None: + host = "localhost" + + if port is None: + port = 9092 + + self._host = host + self._port = port self._client = None + @property + def host(self) -> str: + """The host of kafka. + + :return: A ``str`` value. + """ + return self._host + + @property + def port(self) -> int: + """The port of kafka. + + :return: A ``int`` value. + """ + return self._port + @classmethod def _from_config(cls, config: Config, **kwargs) -> KafkaBrokerPublisher: broker_config = config.get_interface_by_name("broker") - common_config = broker_config["common"] - - kwargs["broker_host"] = common_config["host"] - kwargs["broker_port"] = common_config["port"] + common_config = broker_config.get("common", dict()) + kwargs["host"] = common_config.get("host") + kwargs["port"] = common_config.get("port") return cls(**kwargs) async def _setup(self) -> None: @@ -115,4 +139,4 @@ def _build_client(self) -> AIOKafkaProducer: @property def _bootstrap_servers(self): - return f"{self.broker_host}:{self.broker_port}" + return f"{self.host}:{self.port}" diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py index 7373d110f..36937e54c 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py @@ -61,19 +61,56 @@ class KafkaBrokerSubscriber(BrokerSubscriber, KafkaCircuitBreakerMixin): def __init__( self, topics: Iterable[str], - broker_host: str, - broker_port: int, + host: Optional[str] = None, + port: Optional[int] = None, group_id: Optional[str] = None, remove_topics_on_destroy: bool = False, **kwargs, ): super().__init__(topics, **kwargs) + if host is None: + host = "localhost" - self.broker_host = broker_host - self.broker_port = broker_port - self.group_id = group_id + if port is None: + port = 9092 - self.remove_topics_on_destroy = remove_topics_on_destroy + self._host = host + self._port = port + self._group_id = group_id + + self._remove_topics_on_destroy = remove_topics_on_destroy + + @property + def host(self) -> str: + """The host of kafka. + + :return: A ``str`` value. + """ + return self._host + + @property + def port(self) -> int: + """The port of kafka. + + :return: A ``int`` value. + """ + return self._port + + @property + def group_id(self) -> Optional[str]: + """The id of kafka's group. + + :return: An ``Optional[str]``` value. + """ + return self._group_id + + @property + def remove_topics_on_destroy(self) -> int: + """Flag to check if topics should be removed on destroy. + + :return: A ``bool`` value. + """ + return self._remove_topics_on_destroy async def _setup(self) -> None: await super()._setup() @@ -128,7 +165,7 @@ def admin_client(self): :return: An ``KafkaAdminClient`` instance. """ - return KafkaAdminClient(bootstrap_servers=f"{self.broker_host}:{self.broker_port}") + return KafkaAdminClient(bootstrap_servers=f"{self.host}:{self.port}") async def _receive(self) -> BrokerMessage: record = await self.client.getone() @@ -144,7 +181,7 @@ def client(self) -> AIOKafkaConsumer: """ return AIOKafkaConsumer( *self.topics, - bootstrap_servers=f"{self.broker_host}:{self.broker_port}", + bootstrap_servers=f"{self.host}:{self.port}", group_id=self.group_id, auto_offset_reset="earliest", ) @@ -160,12 +197,12 @@ def with_config(self, config: Config) -> BrokerSubscriberBuilder: :return: This method return the builder instance. """ broker_config = config.get_interface_by_name("broker") - common_config = broker_config["common"] + common_config = broker_config.get("common", dict()) self.kwargs |= { "group_id": config.get_name(), - "broker_host": common_config["host"], - "broker_port": common_config["port"], + "host": common_config.get("host"), + "port": common_config.get("port"), } return self diff --git a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_publisher.py b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_publisher.py index 63aba848f..4d3d5a8e5 100644 --- a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_publisher.py +++ b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_publisher.py @@ -35,6 +35,11 @@ class TestKafkaBrokerPublisher(unittest.IsolatedAsyncioTestCase): def test_is_subclass(self): self.assertTrue(issubclass(KafkaBrokerPublisher, BrokerPublisher)) + def test_constructor(self): + publisher = KafkaBrokerPublisher() + self.assertEqual("localhost", publisher.host) + self.assertEqual(9092, publisher.port) + def test_from_config(self): config = Config(CONFIG_FILE_PATH) broker_config = config.get_interface_by_name("broker")["common"] @@ -42,8 +47,8 @@ def test_from_config(self): publisher = KafkaBrokerPublisher.from_config(config) self.assertIsInstance(publisher, KafkaBrokerPublisher) - self.assertEqual(broker_config["host"], publisher.broker_host) - self.assertEqual(broker_config["port"], publisher.broker_port) + self.assertEqual(broker_config["host"], publisher.host) + self.assertEqual(broker_config["port"], publisher.port) async def test_client(self): publisher = KafkaBrokerPublisher.from_config(CONFIG_FILE_PATH) diff --git a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py index 1f3929ade..05d32cf7e 100644 --- a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py +++ b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py @@ -45,12 +45,19 @@ class TestKafkaBrokerSubscriber(unittest.IsolatedAsyncioTestCase): def test_is_subclass(self): self.assertTrue(issubclass(KafkaBrokerSubscriber, BrokerSubscriber)) + def test_constructor(self): + subscriber = KafkaBrokerSubscriber(["foo", "bar"]) + self.assertEqual({"foo", "bar"}, subscriber.topics) + self.assertEqual("localhost", subscriber.host) + self.assertEqual(9092, subscriber.port) + self.assertEqual(None, subscriber.group_id) + async def test_from_config(self): config = Config(CONFIG_FILE_PATH) broker_config = config.get_interface_by_name("broker")["common"] async with KafkaBrokerSubscriber.from_config(config, topics={"foo", "bar"}) as subscriber: - self.assertEqual(broker_config["host"], subscriber.broker_host) - self.assertEqual(broker_config["port"], subscriber.broker_port) + self.assertEqual(broker_config["host"], subscriber.host) + self.assertEqual(broker_config["port"], subscriber.port) self.assertEqual(config.get_name(), subscriber.group_id) self.assertEqual(False, subscriber.remove_topics_on_destroy) self.assertEqual({"foo", "bar"}, subscriber.topics) @@ -209,8 +216,8 @@ def test_with_config(self): expected = { "group_id": self.config.get_name(), - "broker_host": common_config["host"], - "broker_port": common_config["port"], + "host": common_config["host"], + "port": common_config["port"], } self.assertEqual(expected, builder.kwargs) @@ -221,8 +228,8 @@ def test_build(self): self.assertIsInstance(subscriber, KafkaBrokerSubscriber) self.assertEqual({"one", "two"}, subscriber.topics) - self.assertEqual(common_config["host"], subscriber.broker_host) - self.assertEqual(common_config["port"], subscriber.broker_port) + self.assertEqual(common_config["host"], subscriber.host) + self.assertEqual(common_config["port"], subscriber.port) class TestPostgreSqlQueuedKafkaBrokerSubscriberBuilder(unittest.TestCase): diff --git a/packages/plugins/minos-discovery-minos/minos/plugins/minos_discovery/client.py b/packages/plugins/minos-discovery-minos/minos/plugins/minos_discovery/client.py index bad9555d6..7e8fb0ec8 100644 --- a/packages/plugins/minos-discovery-minos/minos/plugins/minos_discovery/client.py +++ b/packages/plugins/minos-discovery-minos/minos/plugins/minos_discovery/client.py @@ -5,6 +5,9 @@ from functools import ( partial, ) +from typing import ( + Optional, +) from aiohttp import ( ClientError, @@ -24,8 +27,18 @@ class MinosDiscoveryClient(DiscoveryClient, CircuitBreakerMixin): """Minos Discovery Client class.""" - def __init__(self, *args, circuit_breaker_exceptions: Iterable[type] = tuple(), **kwargs): - super().__init__(*args, circuit_breaker_exceptions=(ClientError, *circuit_breaker_exceptions), **kwargs) + def __init__( + self, + host: Optional[str] = None, + port: Optional[int] = None, + circuit_breaker_exceptions: Iterable[type] = tuple(), + **kwargs, + ): + if host is None: + host = "localhost" + if port is None: + port = 5567 + super().__init__(host, port, circuit_breaker_exceptions=(ClientError, *circuit_breaker_exceptions), **kwargs) async def subscribe( self, host: str, port: int, name: str, endpoints: list[dict[str, str]], *args, **kwargs diff --git a/packages/plugins/minos-discovery-minos/tests/test_minos_discovery/test_client.py b/packages/plugins/minos-discovery-minos/tests/test_minos_discovery/test_client.py index dc7b1555f..2d860c174 100644 --- a/packages/plugins/minos-discovery-minos/tests/test_minos_discovery/test_client.py +++ b/packages/plugins/minos-discovery-minos/tests/test_minos_discovery/test_client.py @@ -28,6 +28,11 @@ class TestMinosDiscoveryClient(unittest.IsolatedAsyncioTestCase): def setUp(self) -> None: self.client = MinosDiscoveryClient("123.456.123.1", 1234, circuit_breaker_time=0.1) + def test_constructor(self): + client = MinosDiscoveryClient() + self.assertEqual("localhost", client.host) + self.assertEqual(5567, client.port) + def test_route(self): # noinspection HttpUrlsUsage self.assertEqual("http://123.456.123.1:1234", self.client.route)