From ef3caaa1b46fdaa6354c99e70c6012baa140d5d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Tue, 29 Mar 2022 12:23:22 +0200 Subject: [PATCH 1/6] ISSUE #334 * Fix bug related with incomprehensible error messages related with failures on setup. --- .../minos/common/launchers.py | 15 +++-- .../tests/test_common/test_launchers.py | 65 ++++++++++++++----- 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/packages/core/minos-microservice-common/minos/common/launchers.py b/packages/core/minos-microservice-common/minos/common/launchers.py index 201cc10e7..67c709b8d 100644 --- a/packages/core/minos-microservice-common/minos/common/launchers.py +++ b/packages/core/minos-microservice-common/minos/common/launchers.py @@ -134,6 +134,7 @@ def launch(self) -> NoReturn: logger.info("Stopping microservice...") exception = exc except Exception as exc: # pragma: no cover + logger.exception("Stopping microservice due to an unhandled exception...") exception = exc finally: self.graceful_shutdown(exception) @@ -143,14 +144,14 @@ def graceful_launch(self) -> None: :return: This method does not return anything. """ - self.loop.run_until_complete(gather(self.setup(), self.entrypoint.__aenter__())) + self.loop.run_until_complete(self.setup()) def graceful_shutdown(self, err: Exception = None) -> None: """Shutdown the execution gracefully. :return: This method does not return anything. """ - self.loop.run_until_complete(gather(self.entrypoint.__aexit__(None, err, None), self.destroy())) + self.loop.run_until_complete(self.destroy()) @cached_property def entrypoint(self) -> Entrypoint: @@ -199,9 +200,10 @@ async def _setup(self) -> None: :return: This method does not return anything. """ - await self.injector.wire_and_setup_injections( - modules=self._external_modules + self._internal_modules, packages=self._external_packages - ) + modules = self._external_modules + self._internal_modules + packages = self._external_packages + self.injector.wire_injections(modules=modules, packages=packages) + await gather(self.injector.setup_injections(), self.entrypoint.__aenter__()) @property def _internal_modules(self) -> list[ModuleType]: @@ -212,7 +214,8 @@ async def _destroy(self) -> None: :return: This method does not return anything. """ - await self.injector.unwire_and_destroy_injections() + await gather(self.entrypoint.__aexit__(None, None, None), self.injector.destroy_injections()) + self.injector.unwire_injections() @property def injections(self) -> dict[str, InjectableMixin]: diff --git a/packages/core/minos-microservice-common/tests/test_common/test_launchers.py b/packages/core/minos-microservice-common/tests/test_common/test_launchers.py index 402f611fe..14a42a639 100644 --- a/packages/core/minos-microservice-common/tests/test_common/test_launchers.py +++ b/packages/core/minos-microservice-common/tests/test_common/test_launchers.py @@ -2,6 +2,7 @@ import warnings from unittest.mock import ( AsyncMock, + MagicMock, call, patch, ) @@ -106,37 +107,67 @@ async def test_loop(self): self.assertEqual(call(), mock_loop.call_args) async def test_setup(self): - mock = AsyncMock() - self.launcher.injector.wire_and_setup_injections = mock - await self.launcher.setup() + wire_mock = MagicMock() + setup_mock = AsyncMock() + mock_entrypoint_aenter = AsyncMock() + + self.launcher.injector.wire_injections = wire_mock + self.launcher.injector.setup_injections = setup_mock + + with patch("minos.common.launchers._create_loop") as mock_loop: + loop = FakeLoop() + mock_loop.return_value = loop + with patch("minos.common.launchers._create_entrypoint") as mock_entrypoint: + entrypoint = FakeEntrypoint() + mock_entrypoint.return_value = entrypoint + + entrypoint.__aenter__ = mock_entrypoint_aenter + + await self.launcher.setup() - self.assertEqual(1, mock.call_count) + self.assertEqual(1, wire_mock.call_count) import tests from minos import ( common, ) - self.assertEqual(0, len(mock.call_args.args)) - self.assertEqual(2, len(mock.call_args.kwargs)) - observed = mock.call_args.kwargs["modules"] + self.assertEqual(0, len(wire_mock.call_args.args)) + self.assertEqual(2, len(wire_mock.call_args.kwargs)) + observed = wire_mock.call_args.kwargs["modules"] self.assertIn(tests, observed) self.assertIn(common, observed) - self.assertEqual(["tests"], mock.call_args.kwargs["packages"]) + self.assertEqual(["tests"], wire_mock.call_args.kwargs["packages"]) - await self.launcher.destroy() + self.assertEqual(1, setup_mock.call_count) + self.assertEqual(1, mock_entrypoint_aenter.call_count) async def test_destroy(self): - self.launcher.injector.wire_and_setup_injections = AsyncMock() + self.launcher._setup = AsyncMock() await self.launcher.setup() - mock = AsyncMock() - self.launcher.injector.unwire_and_destroy_injections = mock - await self.launcher.destroy() + destroy_mock = AsyncMock() + unwire_mock = MagicMock() + mock_entrypoint_aexit = AsyncMock() + + self.launcher.injector.destroy_injections = destroy_mock + self.launcher.injector.unwire_injections = unwire_mock + + with patch("minos.common.launchers._create_loop") as mock_loop: + loop = FakeLoop() + mock_loop.return_value = loop + with patch("minos.common.launchers._create_entrypoint") as mock_entrypoint: + entrypoint = FakeEntrypoint() + mock_entrypoint.return_value = entrypoint + + entrypoint.__aexit__ = mock_entrypoint_aexit + + await self.launcher.destroy() - self.assertEqual(1, mock.call_count) - self.assertEqual(call(), mock.call_args) + self.assertEqual(1, unwire_mock.call_count) + self.assertEqual(1, destroy_mock.call_count) + self.assertEqual(1, mock_entrypoint_aexit.call_count) def test_launch(self): mock_setup = AsyncMock() @@ -157,8 +188,8 @@ def test_launch(self): self.launcher.launch() - self.assertEqual(1, mock_entrypoint.call_count) - self.assertEqual(1, mock_loop.call_count) + self.assertEqual(1, mock_setup.call_count) + self.assertEqual(1, mock_destroy.call_count) class TestEntryPointLauncherLoop(unittest.TestCase): From 28a8628ec13970d07c853ca9a087fc06410e27dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Wed, 30 Mar 2022 09:02:45 +0200 Subject: [PATCH 2/6] ISSUE #336 * Raise a `StopAsyncIteration` exception when `AIOKafkaConsumer` raises a `ConsumerStoppedError`. --- .../minos-broker-kafka/minos/plugins/kafka/subscriber.py | 6 +++++- .../tests/test_kafka/test_subscriber.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py index f63e20ef0..0cae46bd3 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py @@ -22,6 +22,7 @@ from aiokafka import ( AIOKafkaConsumer, + ConsumerStoppedError, ) from cached_property import ( cached_property, @@ -166,7 +167,10 @@ def admin_client(self): return KafkaAdminClient(bootstrap_servers=f"{self.host}:{self.port}") async def _receive(self) -> BrokerMessage: - record = await self.client.getone() + try: + record = await self.client.getone() + except ConsumerStoppedError: + raise StopAsyncIteration bytes_ = record.value message = BrokerMessage.from_avro_bytes(bytes_) return message diff --git a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py index 47a4b1900..0bcc1785f 100644 --- a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py +++ b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py @@ -10,6 +10,7 @@ from aiokafka import ( AIOKafkaConsumer, + ConsumerStoppedError, ) from kafka import ( KafkaAdminClient, @@ -206,6 +207,14 @@ async def test_receive(self): self.assertEqual(messages[0], await subscriber.receive()) self.assertEqual(messages[1], await subscriber.receive()) + async def test_receive_stopped(self): + async with KafkaBrokerSubscriber.from_config(CONFIG_FILE_PATH, topics={"foo", "bar"}) as subscriber: + get_mock = AsyncMock(side_effect=ConsumerStoppedError) + subscriber.client.getone = get_mock + + with self.assertRaises(StopAsyncIteration): + await subscriber.receive() + class TestKafkaBrokerSubscriberBuilder(unittest.TestCase): def setUp(self) -> None: From d685943d3c838fa14ad85c666aa69a88e9113f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Fri, 1 Apr 2022 08:51:36 +0200 Subject: [PATCH 3/6] ISSUE #336 * Minor improvement --- .../minos/plugins/kafka/subscriber.py | 6 ++++-- .../tests/test_kafka/test_subscriber.py | 10 +++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py index 0cae46bd3..6f43045d6 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py @@ -169,8 +169,10 @@ def admin_client(self): async def _receive(self) -> BrokerMessage: try: record = await self.client.getone() - except ConsumerStoppedError: - raise StopAsyncIteration + except ConsumerStoppedError as exc: + if self.already_destroyed: + raise StopAsyncIteration + raise exc bytes_ = record.value message = BrokerMessage.from_avro_bytes(bytes_) return message diff --git a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py index 0bcc1785f..edf432502 100644 --- a/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py +++ b/packages/plugins/minos-broker-kafka/tests/test_kafka/test_subscriber.py @@ -207,12 +207,20 @@ async def test_receive(self): self.assertEqual(messages[0], await subscriber.receive()) self.assertEqual(messages[1], await subscriber.receive()) + async def test_receive_already_stopped_raises(self): + subscriber = KafkaBrokerSubscriber.from_config(CONFIG_FILE_PATH, topics={"foo", "bar"}) + get_mock = AsyncMock(side_effect=ConsumerStoppedError) + subscriber.client.getone = get_mock + + with self.assertRaises(StopAsyncIteration): + await subscriber.receive() + async def test_receive_stopped(self): async with KafkaBrokerSubscriber.from_config(CONFIG_FILE_PATH, topics={"foo", "bar"}) as subscriber: get_mock = AsyncMock(side_effect=ConsumerStoppedError) subscriber.client.getone = get_mock - with self.assertRaises(StopAsyncIteration): + with self.assertRaises(ConsumerStoppedError): await subscriber.receive() From 618d7efde06a129264142f723c9bdd40c6194038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Fri, 1 Apr 2022 09:15:26 +0200 Subject: [PATCH 4/6] * Add `Update Guide`. --- .../core/minos-microservice-common/HISTORY.md | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/packages/core/minos-microservice-common/HISTORY.md b/packages/core/minos-microservice-common/HISTORY.md index d549e8617..3a1ead169 100644 --- a/packages/core/minos-microservice-common/HISTORY.md +++ b/packages/core/minos-microservice-common/HISTORY.md @@ -308,4 +308,38 @@ History * Add `Object` base class with the purpose to avoid issues related with multi-inheritance and mixins. * Add `Port` base class as the base class for ports. * Add `CircuitBreakerMixin` class to provide circuit breaker functionalities. -* Add `SetupMixin` class as a replacement of the `MinosSetup` class. \ No newline at end of file +* Add `SetupMixin` class as a replacement of the `MinosSetup` class. + +### Update Guide (from 0.5.x to 0.6.0) +* Add `@Injectable` decorator to classes that injections: +```python +@Injectable("THE_INJECTION_NAME") +class MyInjectableClass: + ... +``` +* Add `minos-http-aiohttp` package: +```shell +poetry add minos-http-aiohttp@^0.6 +``` +* Update `config.yml` file: + * Add ``: +```yaml +... +service: + injections: + http_connector: minos.plugins.aiohttp.AioHttpConnector + ... + ... +... +``` + * Add `routers` section: +```yaml +... +routers: + - minos.networks.BrokerRouter + - minos.networks.PeriodicRouter + - minos.networks.RestHttpRouter +... +``` +* Update `Config` usages according to the new provided API: + * Most common issues come from calls like `config.query_repository._asdict()`, that must be transformed to `config.get_database_by_name("query")` \ No newline at end of file From 3d39fffab34ecd32106d06495fed85a18ab72a34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Fri, 1 Apr 2022 09:18:27 +0200 Subject: [PATCH 5/6] * Minor improvement. --- .../core/minos-microservice-common/HISTORY.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/core/minos-microservice-common/HISTORY.md b/packages/core/minos-microservice-common/HISTORY.md index 3a1ead169..0bb899e17 100644 --- a/packages/core/minos-microservice-common/HISTORY.md +++ b/packages/core/minos-microservice-common/HISTORY.md @@ -311,18 +311,27 @@ History * Add `SetupMixin` class as a replacement of the `MinosSetup` class. ### Update Guide (from 0.5.x to 0.6.0) + * Add `@Injectable` decorator to classes that injections: + ```python +from minos.common import Injectable + + @Injectable("THE_INJECTION_NAME") class MyInjectableClass: - ... + ... ``` + * Add `minos-http-aiohttp` package: + ```shell poetry add minos-http-aiohttp@^0.6 ``` + * Update `config.yml` file: * Add ``: + ```yaml ... service: @@ -332,7 +341,9 @@ service: ... ... ``` - * Add `routers` section: + +* Add `routers` section: + ```yaml ... routers: @@ -341,5 +352,6 @@ routers: - minos.networks.RestHttpRouter ... ``` -* Update `Config` usages according to the new provided API: + +* Update `minos.common.Config` usages according to the new provided API: * Most common issues come from calls like `config.query_repository._asdict()`, that must be transformed to `config.get_database_by_name("query")` \ No newline at end of file From b86750eed5e3d03ec426767d266d08e0d601d8cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Garc=C3=ADa=20Prado?= Date: Fri, 1 Apr 2022 09:31:13 +0200 Subject: [PATCH 6/6] v0.6.1 --- packages/core/minos-microservice-common/HISTORY.md | 12 ++++++++---- .../minos/common/__init__.py | 2 +- .../core/minos-microservice-common/pyproject.toml | 2 +- packages/plugins/minos-broker-kafka/HISTORY.md | 6 +++++- .../minos/plugins/kafka/__init__.py | 2 +- packages/plugins/minos-broker-kafka/pyproject.toml | 2 +- 6 files changed, 17 insertions(+), 9 deletions(-) diff --git a/packages/core/minos-microservice-common/HISTORY.md b/packages/core/minos-microservice-common/HISTORY.md index 0bb899e17..b33fc6301 100644 --- a/packages/core/minos-microservice-common/HISTORY.md +++ b/packages/core/minos-microservice-common/HISTORY.md @@ -329,8 +329,7 @@ class MyInjectableClass: poetry add minos-http-aiohttp@^0.6 ``` -* Update `config.yml` file: - * Add ``: +* Add `HttpConnector` instance to `service.injections` section of `config.yml` file: ```yaml ... @@ -342,7 +341,7 @@ service: ... ``` -* Add `routers` section: +* Add `routers` section to `config.yml` file: ```yaml ... @@ -354,4 +353,9 @@ routers: ``` * Update `minos.common.Config` usages according to the new provided API: - * Most common issues come from calls like `config.query_repository._asdict()`, that must be transformed to `config.get_database_by_name("query")` \ No newline at end of file + * Most common issues come from calls like `config.query_repository._asdict()`, that must be transformed to `config.get_database_by_name("query")` + +0.6.1 (2022-04-01) +------------------ + +* Fix bug that didn't show the correct exception traceback when microservice failures occurred. \ No newline at end of file diff --git a/packages/core/minos-microservice-common/minos/common/__init__.py b/packages/core/minos-microservice-common/minos/common/__init__.py index 853fce3f1..8cf33b005 100644 --- a/packages/core/minos-microservice-common/minos/common/__init__.py +++ b/packages/core/minos-microservice-common/minos/common/__init__.py @@ -1,6 +1,6 @@ __author__ = "Minos Framework Devs" __email__ = "hey@minos.run" -__version__ = "0.6.0" +__version__ = "0.6.1" from .builders import ( BuildableMixin, diff --git a/packages/core/minos-microservice-common/pyproject.toml b/packages/core/minos-microservice-common/pyproject.toml index 22b569afe..d453cf3ef 100644 --- a/packages/core/minos-microservice-common/pyproject.toml +++ b/packages/core/minos-microservice-common/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "minos-microservice-common" -version = "0.6.0" +version = "0.6.1" description = "The common core of the Minos Framework" readme = "README.md" repository = "https://github.com/minos-framework/minos-python" diff --git a/packages/plugins/minos-broker-kafka/HISTORY.md b/packages/plugins/minos-broker-kafka/HISTORY.md index 21d6e0ee4..3f99566f9 100644 --- a/packages/plugins/minos-broker-kafka/HISTORY.md +++ b/packages/plugins/minos-broker-kafka/HISTORY.md @@ -18,4 +18,8 @@ ## 0.6.0 (2022-03-28) * Add `KafkaCircuitBreakerMixin` to integrate `minos.common.CircuitBreakerMixin` into the `KafkaBrokerPublisher` and `KafkaBrokerSubscriber` classes to be tolerant to connection failures to `kafka`. -* Add `KafkaBrokerPublisherBuilder` and `KafkaBrokerBuilderMixin` classes to ease the building process. \ No newline at end of file +* Add `KafkaBrokerPublisherBuilder` and `KafkaBrokerBuilderMixin` classes to ease the building process. + +## 0.6.1 (2022-04-01) + +* Improve `KafkaBrokerSubscriber`'s destroying process. \ No newline at end of file diff --git a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/__init__.py b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/__init__.py index 98e5b4a0b..d9d454e41 100644 --- a/packages/plugins/minos-broker-kafka/minos/plugins/kafka/__init__.py +++ b/packages/plugins/minos-broker-kafka/minos/plugins/kafka/__init__.py @@ -1,6 +1,6 @@ __author__ = "Minos Framework Devs" __email__ = "hey@minos.run" -__version__ = "0.6.0" +__version__ = "0.6.1" from .common import ( KafkaBrokerBuilderMixin, diff --git a/packages/plugins/minos-broker-kafka/pyproject.toml b/packages/plugins/minos-broker-kafka/pyproject.toml index 5b8fa8e37..d9ce95801 100644 --- a/packages/plugins/minos-broker-kafka/pyproject.toml +++ b/packages/plugins/minos-broker-kafka/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "minos-broker-kafka" -version = "0.6.0" +version = "0.6.1" description = "The kafka plugin of the Minos Framework" readme = "README.md" repository = "https://github.com/minos-framework/minos-python"